Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::{Path, PathBuf},
5    sync::{Arc, Mutex, RwLock},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20    error_codes,
21    manifest::{
22        validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24        ProviderRole,
25    },
26    scope::{
27        ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29    },
30    session::{
31        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33        MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34    },
35    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36    Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42    capability_requirements::{
43        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45        RuntimeModule,
46    },
47    daemon_config::RestartRequiredSection,
48    forwarding::{
49        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52    },
53    observability::{
54        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55    },
56    provenance::{
57        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58    },
59    registry::{ChannelState, ConnectionId, Registry, RegistryError},
60    router::{RouteCtx, RouterError},
61    scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63    stderr_tail::{CaptureState, TailEntry},
64    supervise::{
65        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66        SupervisorHandle, SwapHelloAdmission,
67    },
68    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71/// Lowest envelope version this subc build will negotiate.
72///
73/// Module HELLO negotiation is exact: peers must use the daemon's locked
74/// protocol version. Older and newer peers receive `version_unsupported` and
75/// are not registered.
76pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85    ops::SERVER_DESCRIBE,
86    ops::CATALOG_LIST,
87    ops::ROUTE_OPEN,
88    ops::ROUTE_POLL,
89    ops::ROUTE_CLOSING,
90    ops::ROUTE_CLOSED,
91    ops::SUPERVISOR_LIST,
92    ops::SUPERVISOR_RESTART,
93    ops::SUPERVISOR_SWAP,
94    ops::SUPERVISOR_RELOAD,
95    ops::SUPERVISOR_RESCAN,
96    ops::SUPERVISOR_RELEASE_RESERVED,
97    ops::SUPERVISOR_SET_ENABLED,
98    ops::SUPERVISOR_HEALTH_PROBE,
99    ops::SUPERVISOR_HEALTH,
100    ops::SUPERVISOR_STDERR_TAIL,
101    ops::SUPERVISOR_TERMINALS,
102    ops::SUPERVISOR_ROUTES,
103    ops::SUPERVISOR_PROVENANCE,
104    ops::SUPERVISOR_SPAWN_SNAPSHOT,
105    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110    "supervisor.live_roots",
111    SCOPE_SYNC_OP,
112    SCOPE_DESCRIBE_OP,
113];
114
115/// Module-originated ops the daemon answers but does not advertise in
116/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
117/// belongs to is incomplete, so no module is told it works before it does.
118const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122/// How long subc waits for a module to ack a relayed route.bind before returning
123/// `module_timeout`. The ack waits on the module's own configure, which for AFT
124/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
125/// and DB-open work — on a cold page cache or a large repo that legitimately
126/// exceeds a couple of seconds. The default is generous because rejecting a VALID
127/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
128/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
129pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131/// How many CONSECUTIVE full-budget relay timeouts against one target module
132/// open that module's bind-relay breaker.
133///
134/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
135/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
136/// so three consecutive full-budget timeouts take ~36s to observe: every client
137/// whose open contributed to opening the breaker had already given up on its
138/// own. That is what makes opening the breaker unable to turn a call that would
139/// have succeeded into a refusal — it can only make an already-failing module
140/// fail faster.
141///
142/// Two would be reachable inside one default deadline. One would convict a
143/// module on a single cold-cache bind, which is exactly the valid-but-slow case
144/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
145pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147/// How long a module's bind-relay breaker stays open before exactly one
148/// `route.open` is let through as a probe.
149///
150/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
151/// re-pays a full-budget stall almost continuously, and the breaker stops being
152/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
153/// deadline: a client that starts retrying after the module recovers has to get
154/// a probe opportunity inside its own deadline, or the breaker converts a
155/// recovered module into a failed call — the failure it exists to prevent,
156/// pointed the other way.
157///
158/// 20s sits between those with room on both sides, and it caps what a wedged
159/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
160/// rather than one per `route.open` per connection. The stall that motivated
161/// this, with its measurements, is written up in
162/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
163/// waited the whole budget out.
164pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170    configured: &Path,
171    spawned_from: Option<&Path>,
172    image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174    let path = match spawned_from {
175        Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176        Some(spawned_from) => ReloadPathAgreement::Mismatch {
177            configured: configured.to_path_buf(),
178            spawned_from: spawned_from.to_path_buf(),
179        },
180        None => ReloadPathAgreement::Unavailable {
181            reason: if matches!(
182                image,
183                subc_control::RunningImageAgreement::Unavailable {
184                    reason: subc_control::RunningImageUnavailableReason::NotRunning
185                }
186            ) {
187                ReloadPathUnavailableReason::NotRunning
188            } else {
189                ReloadPathUnavailableReason::SpawnedPathUnavailable
190            },
191        },
192    };
193    PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198    build: DaemonBuildProvenance,
199    pid: Option<u32>,
200    started_at_ms: Option<u64>,
201    start_clock: Option<crate::clock::StartClock>,
202    executable_path: Option<PathBuf>,
203    executable_identity: Option<SpawnedFileIdentity>,
204    process_start_time: Option<u64>,
205    probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209    fn default() -> Self {
210        Self {
211            build: DaemonBuildProvenance {
212                build_git_sha: None,
213                build_lock_digest: None,
214            },
215            pid: None,
216            started_at_ms: None,
217            start_clock: None,
218            executable_path: None,
219            executable_identity: None,
220            process_start_time: None,
221            probe: ExecutableIdentityProbe::default(),
222        }
223    }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228    supervisor: Supervisor,
229    config_path: PathBuf,
230    configured_port: Option<u16>,
231    storage_config: Option<crate::daemon_config::StorageConfig>,
232    admission_facts_carrier_module_id: Option<String>,
233    admission_facts_targets: Option<Vec<String>>,
234    scope_authority_owners: Vec<String>,
235}
236
237/// Refusal labels passed to `observe_route_open_refusal` that mean the target
238/// module is not serving right now, and so open or extend an outage in the
239/// route outage tracker. Every one of them is only reachable after the target
240/// was found in the registry, which is what keeps an arbitrary client-chosen
241/// id from ever creating tracker state.
242///
243/// Deliberately absent: `not_registered` and `removed` (the id may be
244/// anything a client sent, and a removed module is gone on purpose),
245/// `protocol_none` (such a module never serves routes, so nothing is out),
246/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
247/// capability and admission-facts refusals (they refuse the caller, not a
248/// module outage), and `relay_reservation_failed` (its code ranges over
249/// capacity limits as well as a vanished connection). Capacity, breaker,
250/// relay-timeout and module-rejection refusals do not pass through that
251/// function at all; the breaker logs its own transitions.
252///
253/// The two not-serving refusals that bypass that function record themselves
254/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
255/// `required_capability_unprovided` is not tracked: the module itself is up,
256/// and the outage belongs to the missing provider.
257const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258    "reloading",
259    "supervisor_not_live",
260    "registration_not_active",
261    "no_forwarding_connection",
262    "relay_send_failed",
263];
264
265/// Real channel-0 control handler for subc itself.
266#[derive(Clone)]
267pub struct ControlHandler {
268    registry: Arc<Registry>,
269    forwarding: Arc<ForwardingTable>,
270    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271    supervisor: SupervisorHandle,
272    subc_capabilities: Arc<[String]>,
273    /// Daemon-wide route.bind relay budget. Used as the fallback when the
274    /// target module has no per-module override in
275    /// `route_bind_relay_timeouts`.
276    route_bind_relay_timeout: Duration,
277    /// Per-module route.bind relay budget overrides, keyed by module id. When
278    /// `handle_route_open` resolves the deadline for a target module, a
279    /// per-module entry wins over the daemon-wide value above.
280    route_bind_relay_timeouts: BTreeMap<String, Duration>,
281    /// Per-target-module bind-relay breaker state. Shared with the forwarding
282    /// table, which is where a new module connection resets it.
283    route_bind_breakers: RouteBindBreakers,
284    /// Live relay admissions keyed by target module. Shared through the
285    /// forwarding table so cloned or separately built handlers enforce one cap.
286    route_bind_concurrency: RouteBindConcurrency,
287    /// Start and end of each module's not-serving period as seen by
288    /// `route.open`, so an outage gets one line at each edge instead of only
289    /// the per-refusal INFO lines. Taken from the forwarding table, so every
290    /// handler built over one table shares it.
291    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292    /// Consecutive relay timeouts that open a module's breaker.
293    route_bind_breaker_threshold: u32,
294    /// How long a breaker stays open before one probe is admitted.
295    route_bind_breaker_cooldown: Duration,
296    health_probe_timeout: Duration,
297    /// Central storage policy. When set, each registering module receives its
298    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
299    storage_config: Option<crate::daemon_config::StorageConfig>,
300    /// The machine id established at boot, served on every HELLO_ACK and on
301    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
302    /// changes the file, never this value. `None` serves no id.
303    machine_id: Option<crate::machine_id::MachineId>,
304    admission_facts_carrier_module_id: Option<String>,
305    admission_facts_targets: Option<Vec<String>>,
306    /// Scope records with their sync authorities and tombstones; see
307    /// `crate::scopes`. Shared by clones of this handler, so every connection
308    /// reads and writes one table.
309    scopes: Arc<RwLock<ScopeTable>>,
310    /// The configured `scope_authority_owners`, kept so a rescan can report a
311    /// changed value as needing a daemon restart; rescan never applies it.
312    scope_authority_owners: Vec<String>,
313    /// The launch nonce each module connection presented at HELLO, which is how
314    /// a `scope.sync` is matched to the owner's current launch.
315    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316    rescan: Option<SupervisorRescanContext>,
317    connected_clients: ConnectedClients,
318    counters: DaemonCounters,
319    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320    daemon_provenance: DaemonProvenanceFacts,
321    #[cfg(test)]
322    control_dispatch_delay: Option<Duration>,
323    #[cfg(test)]
324    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325}
326
327impl fmt::Debug for ControlHandler {
328    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329        f.debug_struct("ControlHandler")
330            .field("registry", &self.registry)
331            .field("forwarding", &self.forwarding)
332            .field("process_liveness", &self.process_liveness.is_some())
333            .field("supervisor", &self.supervisor)
334            .field("subc_capabilities", &self.subc_capabilities)
335            .finish()
336    }
337}
338
339struct RouteOpenRequest {
340    target: RouteTarget,
341    identity: BindIdentity,
342    consumer_identity: Option<ConsumerIdentity>,
343    consumer_capabilities: Option<Vec<String>>,
344    role_versions: Option<BTreeMap<String, String>>,
345    admission_facts: Option<serde_json::Value>,
346    scope: Option<ScopeSelector>,
347}
348
349struct RouteBindReservationGuard {
350    forwarding: Arc<ForwardingTable>,
351    endpoint: ModuleEndpointId,
352    relay_corr: u64,
353    armed: bool,
354}
355
356struct ModuleControlRpcGuard {
357    forwarding: Arc<ForwardingTable>,
358    endpoint: ModuleEndpointId,
359    corr: u64,
360    armed: bool,
361}
362
363impl ModuleControlRpcGuard {
364    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
365        Self {
366            forwarding,
367            endpoint,
368            corr,
369            armed: true,
370        }
371    }
372
373    fn disarm(&mut self) {
374        self.armed = false;
375    }
376}
377
378impl Drop for ModuleControlRpcGuard {
379    fn drop(&mut self) {
380        if self.armed {
381            let _ = self
382                .forwarding
383                .cancel_module_control_rpc(self.endpoint, self.corr);
384        }
385    }
386}
387
388impl RouteBindReservationGuard {
389    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
390        Self {
391            forwarding,
392            endpoint,
393            relay_corr,
394            armed: true,
395        }
396    }
397
398    fn release_and_disarm(&mut self) {
399        if !self.armed {
400            return;
401        }
402        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
403            self.endpoint,
404            self.relay_corr,
405            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
406        ) {
407            send_goodbye_target_best_effort(
408                &self.forwarding.counters(),
409                &target,
410                "canceled route.bind",
411            );
412        }
413        self.armed = false;
414    }
415
416    fn disarm(&mut self) {
417        self.armed = false;
418    }
419}
420
421impl Drop for RouteBindReservationGuard {
422    fn drop(&mut self) {
423        self.release_and_disarm();
424    }
425}
426
427/// Per-target-module circuit breaker around the `route.bind` relay.
428///
429/// The connection reader is serial per connection, so a module whose `on_bind`
430/// sits on the ack blocks every LATER frame on the connections that call it,
431/// including calls to unrelated modules. This does not make any module's bind
432/// fast; it stops the daemon paying the full budget again and again for a
433/// condition it has already observed.
434///
435/// State is keyed by TARGET MODULE and shared by every connection: a wedged
436/// module wedges everyone, so what one connection learned should protect the
437/// rest.
438///
439/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
440/// relay to that module has actually timed out, and is removed again when a
441/// relay is accepted or the module reconnects, so it cannot grow with traffic
442/// or with modules that behave.
443///
444/// # Why a `std` mutex here is not the head-of-line defect again
445///
446/// Acquisition never awaits. The critical section is a hash lookup plus a few
447/// integer updates, with no I/O and no `.await` inside it, so a reader task
448/// cannot be descheduled behind it the way it can behind
449/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
450/// primitive, held for the same kind of work, as the refusal counter this very
451/// path already increments.
452///
453/// It is also NOT on the data-plane splice path: only `route.open` and module
454/// registration touch it, so bound-route frames gain no state check and no
455/// contention.
456#[derive(Debug, Clone, Default)]
457pub(crate) struct RouteBindBreakers {
458    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
459}
460
461#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindConcurrency {
463    modules: Arc<Mutex<HashMap<String, usize>>>,
464}
465
466struct RouteBindConcurrencyGuard {
467    concurrency: RouteBindConcurrency,
468    module_id: String,
469}
470
471impl RouteBindConcurrency {
472    /// Admit without waiting. Waiting here would move the bind stall from the
473    /// module reply to a semaphore and restore reader head-of-line blocking.
474    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
475        let mut modules = self
476            .modules
477            .lock()
478            .expect("route.bind concurrency mutex poisoned");
479        let in_flight = modules.entry(module_id.to_string()).or_default();
480        if *in_flight >= limit {
481            return Err(*in_flight);
482        }
483        *in_flight += 1;
484        Ok(RouteBindConcurrencyGuard {
485            concurrency: self.clone(),
486            module_id: module_id.to_string(),
487        })
488    }
489}
490
491impl Drop for RouteBindConcurrencyGuard {
492    fn drop(&mut self) {
493        let mut modules = self
494            .concurrency
495            .modules
496            .lock()
497            .expect("route.bind concurrency mutex poisoned");
498        let remove = {
499            let in_flight = modules
500                .get_mut(&self.module_id)
501                .expect("admitted route.bind has a concurrency entry");
502            *in_flight -= 1;
503            *in_flight == 0
504        };
505        if remove {
506            modules.remove(&self.module_id);
507        }
508    }
509}
510
511#[derive(Debug, Default)]
512struct ModuleBreakerState {
513    /// Relay timeouts observed with no accepted relay in between.
514    consecutive_timeouts: u32,
515    /// `Some` while the breaker is open: the instant the cooldown expires and
516    /// the next arrival may probe. `None` means closed.
517    cooldown_until: Option<Instant>,
518    /// A half-open probe has been admitted and has not settled yet. This is
519    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
520    /// that read the cooldown, so concurrent opens arriving at the moment the
521    /// cooldown expires cannot all decide that they are the probe.
522    probe_in_flight: Option<Arc<()>>,
523}
524
525/// What the breaker decided for one `route.open`, before any relay work.
526enum RouteBindAdmission<'a> {
527    Admitted {
528        guard: RouteBindBreakerGuard<'a>,
529        /// This open is the single half-open probe, so the transition is worth
530        /// one log line.
531        probe: bool,
532    },
533    Refused {
534        consecutive_timeouts: u32,
535        /// What is left of the cooldown. Zero when the refusal is because the
536        /// one probe is already in flight rather than because the cooldown has
537        /// not elapsed.
538        retry_in: Duration,
539        probe_in_flight: bool,
540    },
541}
542
543/// An outstanding admission, which must be told how its relay settled.
544///
545/// `Drop` settles it as inconclusive, so an early return between admission and
546/// the relay -- or the whole handler being cancelled when the client
547/// disconnects -- releases a half-open probe slot instead of leaving the
548/// breaker wedged half-open with no further probes.
549struct RouteBindBreakerGuard<'a> {
550    breakers: RouteBindBreakers,
551    module_id: &'a str,
552    probe_token: Option<Arc<()>>,
553    settled: bool,
554}
555
556impl RouteBindBreakerGuard<'_> {
557    /// The module answered within the budget and took the bind. THE ONLY
558    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
559    /// breaker, which is a transition worth logging.
560    fn record_accepted(&mut self) -> bool {
561        self.settled = true;
562        self.breakers.record_accepted(self.module_id)
563    }
564
565    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
566    /// COUNTS TOWARD OPENING.
567    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
568        self.settled = true;
569        self.breakers.record_timeout(
570            self.module_id,
571            self.probe_token.as_ref(),
572            threshold,
573            cooldown,
574        )
575    }
576
577    /// Everything else: the module REJECTED the bind, its connection went away
578    /// mid-relay, or the waiter was cancelled.
579    ///
580    /// None of these is evidence that a module is slow, and each already has
581    /// its own refusal with its own code. A module that rejects a bind in
582    /// microseconds is healthy and must never be convicted for it; a module
583    /// that died has said nothing about the module that replaces it. So these
584    /// neither increment nor reset the count -- they only release a probe slot.
585    fn record_inconclusive(&mut self) {
586        self.settled = true;
587        self.breakers
588            .record_inconclusive(self.module_id, self.probe_token.as_ref());
589    }
590}
591
592impl Drop for RouteBindBreakerGuard<'_> {
593    fn drop(&mut self) {
594        if !self.settled {
595            self.breakers
596                .record_inconclusive(self.module_id, self.probe_token.as_ref());
597        }
598    }
599}
600
601/// The breaker moved to open, reported so the caller can log it outside the
602/// lock. Opening is rare and load-bearing; the refusals that follow are
603/// frequent and are counted rather than logged.
604struct BreakerOpened {
605    consecutive_timeouts: u32,
606    /// True when a failed probe re-opened an already-open breaker, which reads
607    /// very differently in a log from a first opening.
608    reopened_after_probe: bool,
609}
610
611impl RouteBindBreakers {
612    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
613        self.modules
614            .lock()
615            .expect("route.bind breaker mutex poisoned")
616    }
617
618    /// Decide whether this `route.open` may attempt its relay. Takes the map
619    /// lock and nothing else, and never awaits.
620    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
621        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
622            probe: probe_token.is_some(),
623            guard: RouteBindBreakerGuard {
624                breakers: self.clone(),
625                module_id,
626                probe_token,
627                settled: false,
628            },
629        };
630
631        let mut modules = self.lock();
632        let Some(state) = modules.get_mut(module_id) else {
633            return admitted(None);
634        };
635        let Some(cooldown_until) = state.cooldown_until else {
636            return admitted(None);
637        };
638        if state.probe_in_flight.is_some() {
639            return RouteBindAdmission::Refused {
640                consecutive_timeouts: state.consecutive_timeouts,
641                retry_in: Duration::ZERO,
642                probe_in_flight: true,
643            };
644        }
645        let now = Instant::now();
646        if now < cooldown_until {
647            return RouteBindAdmission::Refused {
648                consecutive_timeouts: state.consecutive_timeouts,
649                retry_in: cooldown_until - now,
650                probe_in_flight: false,
651            };
652        }
653        let token = Arc::new(());
654        state.probe_in_flight = Some(Arc::clone(&token));
655        admitted(Some(token))
656    }
657
658    fn record_accepted(&self, module_id: &str) -> bool {
659        self.lock()
660            .remove(module_id)
661            .is_some_and(|state| state.cooldown_until.is_some())
662    }
663
664    fn record_timeout(
665        &self,
666        module_id: &str,
667        probe_token: Option<&Arc<()>>,
668        threshold: u32,
669        cooldown: Duration,
670    ) -> Option<BreakerOpened> {
671        let mut modules = self.lock();
672        let state = modules.entry(module_id.to_string()).or_default();
673        let was_open = state.cooldown_until.is_some();
674        let was_probe = Self::owns_probe(state, probe_token);
675        if was_probe {
676            state.probe_in_flight = None;
677        }
678        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
679        if state.consecutive_timeouts < threshold {
680            return None;
681        }
682        state.cooldown_until = Some(Instant::now() + cooldown);
683        Some(BreakerOpened {
684            consecutive_timeouts: state.consecutive_timeouts,
685            reopened_after_probe: was_open && was_probe,
686        })
687    }
688
689    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
690        // A relay can finish after the breaker was reset or after it opened
691        // again and started a new probe. Only the guard whose token matches the
692        // active probe may release it, so a late relay never frees a newer probe.
693        state
694            .probe_in_flight
695            .as_ref()
696            .zip(token)
697            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
698    }
699
700    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
701        if let Some(state) = self.lock().get_mut(module_id) {
702            if Self::owns_probe(state, probe_token) {
703                state.probe_in_flight = None;
704            }
705        }
706    }
707
708    /// Discard what was learned about a module, because the process it was
709    /// learned about is gone. Returns the discarded count when it was non-zero.
710    ///
711    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
712    /// `module_id` is a configuration identity that outlives any particular
713    /// child; what the breaker observed was the process behind the module
714    /// connection of the moment. When a new connection registers under that id
715    /// the verdict's subject no longer exists, so the verdict is stale by
716    /// construction rather than merely likely to be wrong. Keeping it would
717    /// apply a dead process's record to a live one, which is the same defect
718    /// class this breaker exists to stop the daemon committing.
719    ///
720    /// A half-open probe in flight is discarded with the rest: it was a
721    /// question about the old process.
722    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
723        self.lock()
724            .remove(module_id)
725            .map(|state| state.consecutive_timeouts)
726            .filter(|discarded| *discarded > 0)
727    }
728
729    /// Open breakers, for the `server.describe` counters object. `None` when
730    /// none is open, so the key stays absent rather than present-and-empty.
731    ///
732    /// This is the operator's answer to "is this module refusing instantly or
733    /// is it fine?", which look identical from a client that retries and then
734    /// succeeds.
735    fn open_snapshot(&self) -> Option<serde_json::Value> {
736        let now = Instant::now();
737        let modules = self.lock();
738        let open = modules
739            .iter()
740            .filter_map(|(module_id, state)| {
741                let cooldown_until = state.cooldown_until?;
742                Some((
743                    module_id.clone(),
744                    serde_json::json!({
745                        "consecutive_timeouts": state.consecutive_timeouts,
746                        "cooldown_remaining_ms":
747                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
748                        "probe_in_flight": state.probe_in_flight.is_some(),
749                    }),
750                ))
751            })
752            .collect::<serde_json::Map<String, serde_json::Value>>();
753        (!open.is_empty()).then_some(serde_json::Value::Object(open))
754    }
755}
756
757impl ControlHandler {
758    pub fn new(registry: Arc<Registry>) -> Self {
759        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
760    }
761
762    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
763        let counters = forwarding.counters();
764        // Taken from the forwarding table rather than created here, so that the
765        // breaker a `route.open` consults is the same one a module's
766        // registration resets, however many handlers are built over one table.
767        let route_bind_breakers = forwarding.route_bind_breakers();
768        let route_bind_concurrency = forwarding.route_bind_concurrency();
769        let route_outages = forwarding.route_outages();
770        Self {
771            registry,
772            forwarding,
773            process_liveness: None,
774            supervisor: SupervisorHandle::new(),
775            subc_capabilities: Arc::from([
776                CAP_MANIFEST_REGISTRATION.to_string(),
777                CAP_CHANNEL_LIFECYCLE.to_string(),
778                CAP_PING_PONG.to_string(),
779                CAP_SESSION_ATTACH.to_string(),
780                CAP_ADMISSION_FACTS_RELAY.to_string(),
781                CAP_SCOPES_V1.to_string(),
782                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
783            ]),
784            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
785            route_bind_relay_timeouts: BTreeMap::new(),
786            route_bind_breakers,
787            route_bind_concurrency,
788            route_outages,
789            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
790            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
791            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
792            storage_config: None,
793            machine_id: None,
794            admission_facts_carrier_module_id: None,
795            admission_facts_targets: None,
796            scopes: Arc::new(RwLock::new(ScopeTable::new(
797                crate::daemon_config::default_scope_authority_owners(),
798            ))),
799            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
800            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
801            rescan: None,
802            connected_clients: ConnectedClients::new(),
803            counters,
804            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
805            daemon_provenance: DaemonProvenanceFacts::default(),
806            #[cfg(test)]
807            control_dispatch_delay: None,
808            #[cfg(test)]
809            provenance_probe_override: None,
810        }
811    }
812
813    /// Set the central storage policy: registering modules then receive their
814    /// resolved storage descriptor in HELLO_ACK.
815    pub fn with_storage_config(
816        mut self,
817        storage_config: Option<crate::daemon_config::StorageConfig>,
818    ) -> Self {
819        self.storage_config = storage_config;
820        self
821    }
822
823    /// Set the machine id served to every registering module (HELLO_ACK) and on
824    /// `server.describe`.
825    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
826        self.machine_id = machine_id;
827        self
828    }
829
830    /// Configure the exact reserved module and target ids permitted to relay
831    /// opaque admission facts. Config-file loading validates this authority;
832    /// this builder keeps the same policy available to embedded test daemons.
833    pub fn with_admission_facts_config(
834        mut self,
835        carrier_module_id: Option<String>,
836        targets: Option<Vec<String>>,
837    ) -> Self {
838        self.admission_facts_carrier_module_id = carrier_module_id;
839        self.admission_facts_targets = targets;
840        self
841    }
842
843    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
844    /// Replaces the scope table with an empty one under the new list, so call it
845    /// while building the handler, before any module can sync.
846    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
847        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
848        self.scope_authority_owners = owners;
849        self
850    }
851
852    /// Override the route.bind relay timeout. Used by tests that assert the
853    /// timeout path so they don't block on the production-safe default.
854    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
855        self.route_bind_relay_timeout = timeout;
856        self
857    }
858
859    /// Install per-module route.bind relay budget overrides. A module id
860    /// listed here wins over the daemon-wide default set via
861    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
862    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
863    /// the same `Duration` the bind path will use.
864    pub fn with_route_bind_relay_timeouts(
865        mut self,
866        timeouts: impl IntoIterator<Item = (String, Duration)>,
867    ) -> Self {
868        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
869        self
870    }
871
872    /// Resolve the route.bind relay budget for a specific target module id.
873    /// Per-module overrides win; the daemon-wide value (set via
874    /// `with_route_bind_relay_timeout` or the built-in default) is the
875    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
876    /// the same resolution `handle_route_open` will use.
877    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
878        self.route_bind_relay_timeouts
879            .get(module_id)
880            .copied()
881            .unwrap_or(self.route_bind_relay_timeout)
882    }
883
884    /// Override the per-module bind-relay breaker policy.
885    ///
886    /// Used by tests, which cannot spend three production budgets opening a
887    /// breaker or twenty seconds waiting for its cooldown. The production
888    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
889    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
890    /// reasoning for the numbers.
891    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
892        self.route_bind_breaker_threshold = threshold.max(1);
893        self.route_bind_breaker_cooldown = cooldown;
894        self
895    }
896
897    #[cfg(test)]
898    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
899        self.health_probe_timeout = timeout;
900        self
901    }
902
903    #[cfg(test)]
904    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
905        self.control_dispatch_delay = Some(delay);
906        self
907    }
908
909    pub fn with_process_liveness(
910        mut self,
911        process_liveness: Arc<dyn ModuleProcessLiveness>,
912    ) -> Self {
913        self.process_liveness = Some(process_liveness);
914        self
915    }
916
917    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
918        self.supervisor = supervisor;
919        self
920    }
921
922    pub fn with_daemon_provenance(
923        mut self,
924        pid: u32,
925        started_at_ms: u64,
926        executable_path: Option<PathBuf>,
927        build_git_sha: Option<String>,
928        build_lock_digest: Option<String>,
929    ) -> Self {
930        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
931        let process_start_time = process_start_time(pid);
932        self.daemon_provenance = DaemonProvenanceFacts {
933            build: DaemonBuildProvenance {
934                build_git_sha,
935                build_lock_digest,
936            },
937            pid: Some(pid),
938            started_at_ms: Some(started_at_ms),
939            start_clock: None,
940            executable_path,
941            executable_identity,
942            process_start_time,
943            probe: ExecutableIdentityProbe::default(),
944        };
945        self
946    }
947
948    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
949        self.daemon_provenance.start_clock = Some(clock);
950        self
951    }
952
953    #[cfg(test)]
954    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
955        self.provenance_probe_override = Some(result);
956        self
957    }
958
959    /// Install the configured module set and its reserved capability bindings.
960    /// Bindings are configuration-scoped and may point at a provider that has not
961    /// been installed yet, so this does not require the bound module to exist.
962    pub fn with_capability_config(
963        self,
964        modules: impl IntoIterator<Item = (String, bool)>,
965        reserved_capabilities: BTreeMap<String, String>,
966    ) -> Self {
967        self.capability_evaluator
968            .configure(modules, reserved_capabilities);
969        self
970    }
971
972    pub fn with_supervisor_rescan(
973        mut self,
974        supervisor: Supervisor,
975        config_path: impl Into<PathBuf>,
976        configured_port: Option<u16>,
977    ) -> Self {
978        self.rescan = Some(SupervisorRescanContext {
979            supervisor,
980            config_path: config_path.into(),
981            configured_port,
982            storage_config: self.storage_config.clone(),
983            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
984            admission_facts_targets: self.admission_facts_targets.clone(),
985            scope_authority_owners: self.scope_authority_owners.clone(),
986        });
987        self
988    }
989
990    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
991        self.connected_clients = connected_clients;
992        self
993    }
994
995    pub fn forwarding(&self) -> Arc<ForwardingTable> {
996        Arc::clone(&self.forwarding)
997    }
998
999    pub(crate) fn counters(&self) -> DaemonCounters {
1000        self.counters.clone()
1001    }
1002
1003    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1004    /// requirement event without depending on an operator polling a status command.
1005    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1006        tokio::spawn(async move {
1007            loop {
1008                self.capability_evaluator
1009                    .wait_for_change_or_deadline()
1010                    .await;
1011                self.refresh_capability_requirements();
1012            }
1013        });
1014    }
1015
1016    fn runtime_capability_snapshot(
1017        &self,
1018    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1019        let runtime = self
1020            .supervisor
1021            .list()
1022            .into_iter()
1023            .map(|module| {
1024                let status = module.status().map_err(|err| {
1025                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1026                })?;
1027                Ok(RuntimeModule {
1028                    module_id: status.module_id,
1029                    state: status.state,
1030                    enabled: status.enabled,
1031                })
1032            })
1033            .collect::<Result<Vec<_>, RouterError>>()?;
1034        let (_, registrations) = self.registry.list_modules().map_err(|err| {
1035            RouterError::backend(
1036                0,
1037                0,
1038                format!("failed to list capability registrations: {err}"),
1039            )
1040        })?;
1041        let registrations = registrations
1042            .into_iter()
1043            .map(|registration| RegisteredModule {
1044                module_id: registration.manifest.module_id,
1045                module_version: registration.manifest.module_version,
1046                capabilities: registration.manifest.capabilities,
1047            })
1048            .collect();
1049        Ok((runtime, registrations))
1050    }
1051
1052    /// The capability side effects of a module becoming the active registration
1053    /// for its id: cache its manifest (warning if its claims drifted), run the
1054    /// deny census when its declarations call for one, and recompute the
1055    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1056    /// candidate's does not, and the supervisor does it at promotion instead,
1057    /// through [`crate::supervise::SwapPromotionObserver`].
1058    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1059        let cached_registration = RegisteredModule {
1060            module_id: registration.manifest.module_id.clone(),
1061            module_version: registration.manifest.module_version.clone(),
1062            capabilities: registration.manifest.capabilities.clone(),
1063        };
1064        if self.capability_evaluator.record_hello(&cached_registration) {
1065            warn!(
1066                module_id = %cached_registration.module_id,
1067                "capability claims drifted from the cached manifest"
1068            );
1069        }
1070        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1071            self.enforce_capability_denies();
1072        }
1073        self.refresh_capability_requirements();
1074    }
1075
1076    /// Point the shared supervisor handle at this handler for swap promotions.
1077    /// Called wherever a handler is put behind the `Arc` the router serves, so
1078    /// it can be held weakly.
1079    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1080        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1081            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1082        self.supervisor.set_swap_promotion_observer(observer);
1083    }
1084
1085    pub fn refresh_capability_requirements(&self) {
1086        match self.runtime_capability_snapshot() {
1087            Ok((runtime, registrations)) => {
1088                log_requirement_events(
1089                    self.capability_evaluator
1090                        .evaluate_now(&runtime, &registrations),
1091                );
1092            }
1093            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1094        }
1095    }
1096
1097    /// Reconcile only live, attested route bindings after a capability deny edge
1098    /// or target claim was added. This is deliberately a control-plane census:
1099    /// the opaque forwarding hot path must not grow a per-frame capability check.
1100    fn enforce_capability_denies(&self) {
1101        let (_, registrations) = match self.registry.list_modules() {
1102            Ok(snapshot) => snapshot,
1103            Err(err) => {
1104                warn!(error = %err, "failed to read registrations for capability deny census");
1105                return;
1106            }
1107        };
1108        let manifests = registrations
1109            .into_iter()
1110            .map(|registration| {
1111                (
1112                    registration.manifest.module_id.clone(),
1113                    registration.manifest,
1114                )
1115            })
1116            .collect::<BTreeMap<_, _>>();
1117        let census = match self.forwarding.route_census(None) {
1118            Ok(census) => census,
1119            Err(err) => {
1120                warn!(error = %err, "failed to read route census for capability deny enforcement");
1121                return;
1122            }
1123        };
1124
1125        for (target_module_id, routes) in census {
1126            let Some(target_manifest) = manifests.get(&target_module_id) else {
1127                continue;
1128            };
1129            let mut closed_routes = Vec::new();
1130            let mut module_goodbyes = Vec::new();
1131            for route in routes {
1132                let Principal::Reserved {
1133                    module_id: opening_module_id,
1134                } = &route.principal
1135                else {
1136                    continue;
1137                };
1138                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1139                    continue;
1140                };
1141                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1142                    continue;
1143                };
1144
1145                match self.forwarding.release_client_route(
1146                    route.goodbye_target.connection_id,
1147                    route.goodbye_target.channel,
1148                    route.goodbye_target.epoch,
1149                ) {
1150                    Ok(RouteRelease::Removed(module_goodbye)) => {
1151                        warn!(
1152                            opening_module_id,
1153                            target_module_id,
1154                            capability,
1155                            "force-closing route because an attested capability deny edge now matches"
1156                        );
1157                        closed_routes.push(route);
1158                        module_goodbyes.push(module_goodbye);
1159                    }
1160                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1161                    Err(err) => warn!(
1162                        opening_module_id,
1163                        target_module_id,
1164                        capability,
1165                        error = %err,
1166                        "failed to force-close capability-denied route"
1167                    ),
1168                }
1169            }
1170
1171            if closed_routes.is_empty() {
1172                continue;
1173            }
1174            send_route_control_pushes(
1175                &self.forwarding,
1176                closed_routes,
1177                ClientControlPush::RouteClosed {
1178                    module_id: target_module_id,
1179                    channels: Vec::new(),
1180                    reason: RouteCloseReason::CapabilityDenied,
1181                    drained: false,
1182                    abandoned: 0,
1183                    excluded_subscriptions: 0,
1184                    terminal: Some(false),
1185                },
1186            );
1187            self.emit_route_goodbyes(module_goodbyes);
1188        }
1189    }
1190
1191    /// Why a registered module is not accepting new route binds, or `None` when
1192    /// it is. This is the module's effective readiness: its declared readiness
1193    /// first, then every `need: required` capability it declares evaluating to
1194    /// `provided`. `route.open` and `catalog.list` both read it here so the
1195    /// catalog never reports a module routable that `route.open` would refuse.
1196    fn not_ready_reason(
1197        &self,
1198        registration: &crate::registry::ModuleRegistration,
1199    ) -> Option<NotReadyReason> {
1200        if !registration.ready {
1201            return Some(NotReadyReason {
1202                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1203                capability: None,
1204            });
1205        }
1206        self.first_unprovided_required_capability(registration)
1207            .map(|capability| NotReadyReason {
1208                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1209                capability: Some(capability),
1210            })
1211    }
1212
1213    /// The lexicographically first capability this registration declares
1214    /// `need: required` whose evaluator verdict is not `provided`.
1215    ///
1216    /// The verdicts are the capability evaluator's own; nothing here decides
1217    /// what "provided" means. The evaluator counts a capability provided as
1218    /// soon as a module claiming it has REGISTERED, not once that module is
1219    /// ready. That distinction is what keeps two modules that require each
1220    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1221    /// is ready", each would wait for the other to become ready first and
1222    /// neither ever would. Do not tighten it to readiness.
1223    ///
1224    /// A required capability with no verdict at all means this registration's
1225    /// HELLO or catalog.update landed after the last recompute; recompute once
1226    /// rather than let a missing verdict read as either answer. If it is still
1227    /// missing (the recompute itself failed) the capability counts as
1228    /// unprovided: the refusal is retryable, and routing a module whose
1229    /// required provider is unknown is the outcome this check exists to stop.
1230    fn first_unprovided_required_capability(
1231        &self,
1232        registration: &crate::registry::ModuleRegistration,
1233    ) -> Option<String> {
1234        let required = registration
1235            .manifest
1236            .capabilities
1237            .iter()
1238            .flat_map(|declarations| declarations.requires.iter())
1239            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1240            .map(|requirement| requirement.capability.as_str())
1241            .collect::<BTreeSet<_>>();
1242        if required.is_empty() {
1243            return None;
1244        }
1245        let module_id = registration.manifest.module_id.as_str();
1246        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1247        if required
1248            .iter()
1249            .any(|capability| verdict(capability).is_none())
1250        {
1251            self.refresh_capability_requirements();
1252        }
1253        required
1254            .into_iter()
1255            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1256            .map(str::to_string)
1257    }
1258
1259    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1260        self.capability_evaluator
1261            .statuses()
1262            .into_iter()
1263            .map(capability_requirement_status)
1264            .collect()
1265    }
1266
1267    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1268    /// registration-release watch. The signal is what the supervisor waits on
1269    /// before spawning a replacement, so it must only fire once forwarding
1270    /// teardown is also done (see [`Self::cleanup_connection`] /
1271    /// [`Self::handle_goodbye`]). Used directly only where there is no forwarding
1272    /// state to tear down (a HELLO that failed before module registration).
1273    fn deregister_connection(
1274        &self,
1275        connection_id: ConnectionId,
1276    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1277        self.registry.deregister_connection(connection_id)
1278    }
1279
1280    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1281        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1282            return None;
1283        }
1284        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1285            parse_client_control_request(&frame.body)
1286        else {
1287            return None;
1288        };
1289        Some(target_module_id(&target).to_string())
1290    }
1291
1292    pub(crate) fn route_open_capacity_refusal(
1293        &self,
1294        ctx: &RouteCtx,
1295        frame: &Frame,
1296        target_module_id: &str,
1297        in_flight: usize,
1298        limit: usize,
1299    ) -> Result<Frame, RouterError> {
1300        self.route_open_admission_refusal_frame(
1301            ctx,
1302            frame,
1303            target_module_id,
1304            "open_admission_full",
1305            (in_flight, limit),
1306            format!(
1307                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1308            ),
1309        )
1310    }
1311
1312    fn route_open_target_capacity_refusal(
1313        &self,
1314        ctx: &RouteCtx,
1315        frame: &Frame,
1316        target_module_id: &str,
1317        in_flight: usize,
1318    ) -> Result<Frame, RouterError> {
1319        self.route_open_admission_refusal_frame(
1320            ctx,
1321            frame,
1322            target_module_id,
1323            "target_binds_full",
1324            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1325            format!(
1326                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1327            ),
1328        )
1329    }
1330
1331    /// Admission pressure clears as existing binds settle, so its refusal must
1332    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1333    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1334    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1335    /// cannot currently reach its target; `module_timeout` would falsely claim
1336    /// that a wait expired. A new, cleaner code would be terminal to deployed
1337    /// clients, so it requires a client-tolerance rollout before daemon emission.
1338    fn route_open_admission_refusal_frame(
1339        &self,
1340        ctx: &RouteCtx,
1341        frame: &Frame,
1342        target_module_id: &str,
1343        reason: &'static str,
1344        (in_flight, limit): (usize, usize),
1345        message: impl Into<String>,
1346    ) -> Result<Frame, RouterError> {
1347        let code = error_codes::TARGET_UNAVAILABLE;
1348        self.counters.increment_route_open_refused(code);
1349        info!(
1350            target: "control",
1351            code,
1352            reason,
1353            module_id = ?target_module_id,
1354            connection_id = ctx.connection_id.get(),
1355            in_flight,
1356            limit,
1357            "route.open refused"
1358        );
1359        control_error_frame(frame, code, message.into())
1360    }
1361
1362    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1363    ///
1364    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1365    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1366    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1367    #[cfg(test)]
1368    pub fn handle_control(
1369        &self,
1370        connection_id: ConnectionId,
1371        frame: Frame,
1372    ) -> Result<Vec<Frame>, RouterError> {
1373        match frame.header.ty {
1374            FrameType::Ping => Ok(vec![pong(&frame)?]),
1375            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1376            FrameType::Goodbye => self.handle_goodbye(connection_id),
1377            ty => Ok(vec![control_error_frame(
1378                &frame,
1379                "unsupported_control_frame",
1380                format!("unsupported channel-0 frame {ty:?}"),
1381            )?]),
1382        }
1383    }
1384
1385    pub async fn handle_control_frame(
1386        &self,
1387        ctx: &RouteCtx,
1388        frame: Frame,
1389    ) -> Result<Vec<Frame>, RouterError> {
1390        self.handle_control_frame_timed(ctx, frame, None).await
1391    }
1392
1393    pub(crate) async fn handle_control_frame_timed(
1394        &self,
1395        ctx: &RouteCtx,
1396        frame: Frame,
1397        dispatch_started_at: Option<StdInstant>,
1398    ) -> Result<Vec<Frame>, RouterError> {
1399        match frame.header.ty {
1400            FrameType::Ping => Ok(vec![pong(&frame)?]),
1401            FrameType::Hello => {
1402                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1403            }
1404            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1405            FrameType::Cancel => {
1406                if self
1407                    .supervisor
1408                    .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1409                {
1410                    Ok(Vec::new())
1411                } else {
1412                    Ok(vec![control_error_frame(
1413                        &frame,
1414                        "unknown_subscription",
1415                        "no supervisor spawn subscription has this correlation id",
1416                    )?])
1417                }
1418            }
1419            FrameType::Request => {
1420                if self
1421                    .forwarding
1422                    .module_endpoint_for_connection(ctx.connection_id)
1423                    .map_err(RouterError::Forwarding)?
1424                    .is_some()
1425                {
1426                    if !is_known_module_request_op(&frame.body) {
1427                        return Ok(vec![control_error_frame(
1428                            &frame,
1429                            "unsupported_control_frame",
1430                            "module-originated channel-0 REQUEST is not supported",
1431                        )?]);
1432                    }
1433                    let request = match parse_module_control_request_from_module(&frame.body) {
1434                        Ok(request) => request,
1435                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1436                            return Ok(vec![control_error_frame(
1437                                &frame,
1438                                "unsupported_control_frame",
1439                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1440                            )?])
1441                        }
1442                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1443                            return Ok(vec![control_error_frame(
1444                                &frame,
1445                                "invalid_control_body",
1446                                format!("malformed module control body: {err}"),
1447                            )?])
1448                        }
1449                    };
1450                    let op = module_control_request_op(&request);
1451                    let corr = frame.header.corr;
1452                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1453                    let result =
1454                        self.handle_module_control_request(ctx.connection_id, frame, request);
1455                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1456                    return result;
1457                }
1458
1459                if is_known_module_request_op(&frame.body) {
1460                    return Ok(vec![control_error_frame(
1461                        &frame,
1462                        "not_registered",
1463                        "catalog.update requires an active module registration owned by this connection",
1464                    )?]);
1465                }
1466
1467                let request = match parse_client_control_request(&frame.body) {
1468                    Ok(request) => request,
1469                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1470                        return Ok(vec![control_error_frame(
1471                            &frame,
1472                            "unknown_control_op",
1473                            format!("unknown client control op: {err}"),
1474                        )?])
1475                    }
1476                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1477                        return Ok(vec![control_error_frame(
1478                            &frame,
1479                            "invalid_control_body",
1480                            format!("malformed client control body: {err}"),
1481                        )?])
1482                    }
1483                };
1484                let op = client_control_request_op(&request);
1485                let corr = frame.header.corr;
1486                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1487                #[cfg(test)]
1488                if let Some(delay) = self.control_dispatch_delay {
1489                    tokio::time::sleep(delay).await;
1490                }
1491                let result = self
1492                    .handle_client_control_request(ctx, frame, request)
1493                    .await;
1494                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1495                result
1496            }
1497            FrameType::Push => {
1498                let Some(endpoint) = self
1499                    .forwarding
1500                    .module_endpoint_for_connection(ctx.connection_id)
1501                    .map_err(RouterError::Forwarding)?
1502                else {
1503                    return Ok(vec![control_error_frame(
1504                        &frame,
1505                        "unsupported_control_frame",
1506                        "client-originated channel-0 PUSH is not supported",
1507                    )?]);
1508                };
1509                self.handle_status_update(endpoint, frame)
1510            }
1511            FrameType::Response | FrameType::Error
1512                if self
1513                    .forwarding
1514                    .module_endpoint_for_connection(ctx.connection_id)
1515                    .map_err(RouterError::Forwarding)?
1516                    .is_some() =>
1517            {
1518                self.handle_module_relay_response(ctx.connection_id, frame)
1519            }
1520            ty => Ok(vec![control_error_frame(
1521                &frame,
1522                "unsupported_control_frame",
1523                format!("unsupported channel-0 frame {ty:?}"),
1524            )?]),
1525        }
1526    }
1527
1528    pub fn cleanup_connection(
1529        &self,
1530        connection_id: ConnectionId,
1531    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1532        let crash_closed = self
1533            .registry
1534            .get_module_by_connection(connection_id)?
1535            .and_then(|registration| {
1536                self.forwarding
1537                    .module_endpoint_for_connection(connection_id)
1538                    .ok()
1539                    .flatten()
1540                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1541                    .map(|routes| (registration.manifest.module_id, routes))
1542            });
1543        let crash_closed = crash_closed.map(|(module_id, routes)| {
1544            let terminal = match self.supervisor.get(&module_id) {
1545                None => false,
1546                Some(module) => match module.will_recover_after_connection_loss() {
1547                    Ok(will_recover) => !will_recover,
1548                    Err(err) => {
1549                        warn!(
1550                            %module_id,
1551                            error = %err,
1552                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1553                        );
1554                        false
1555                    }
1556                },
1557            };
1558            // The forwarding table gates all providers at the start of daemon
1559            // shutdown, before their connections are closed. An ordinary
1560            // module disconnect still reports crash if that gate is not set.
1561            let reason = match self.forwarding.is_daemon_draining() {
1562                Ok(true) => RouteCloseReason::Restart,
1563                Ok(false) => RouteCloseReason::Crash,
1564                Err(err) => {
1565                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1566                    RouteCloseReason::Crash
1567                }
1568            };
1569            (module_id, routes, reason, terminal)
1570        });
1571        let registrations = self.deregister_connection(connection_id);
1572        let cleanup = if crash_closed.is_some() {
1573            self.forwarding.cleanup_connection_counted(connection_id)
1574        } else {
1575            // A module connection's teardown needs the count of abandoned
1576            // route.bind relays for its route.closed notice below. Any other
1577            // connection, such as a client's, sends no such notice and needs
1578            // only its routes released, so it uses the route-only wrapper and
1579            // reports zero.
1580            self.forwarding
1581                .cleanup_connection(connection_id)
1582                .map(|released| crate::forwarding::ConnectionCleanup {
1583                    released,
1584                    abandoned_relays: 0,
1585                })
1586        };
1587        // The route.closed push waits for forwarding teardown because only
1588        // teardown knows how many pending route.bind relays it aborted. It still
1589        // goes out before the GOODBYEs for the released routes, and its targets
1590        // were captured above, before teardown removed those routes.
1591        if let Some((module_id, routes, reason, terminal)) = crash_closed {
1592            let abandoned = cleanup
1593                .as_ref()
1594                .map_or(0, |cleanup| cleanup.abandoned_relays);
1595            send_route_control_pushes(
1596                &self.forwarding,
1597                routes,
1598                ClientControlPush::RouteClosed {
1599                    module_id,
1600                    channels: Vec::new(),
1601                    reason,
1602                    drained: false,
1603                    abandoned,
1604                    excluded_subscriptions: 0,
1605                    terminal: Some(terminal),
1606                },
1607            );
1608        }
1609        if let Ok(cleanup) = cleanup {
1610            self.emit_route_goodbyes(cleanup.released);
1611        }
1612        // Signal the registration-release watch only now that BOTH registry and
1613        // forwarding teardown are done, so a supervisor waiting to spawn a
1614        // replacement never observes release while old routes still exist.
1615        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1616            crate::supervise::notify_registration_release();
1617            self.capability_evaluator.wake_deadline_loop();
1618            self.refresh_capability_requirements();
1619        }
1620        self.supervisor.remove_spawn_subscribers(connection_id);
1621        // Sync authority dies with its connection, so the owner's next
1622        // connection can take it; the owner's scopes stay as they are.
1623        self.hello_launch_nonces
1624            .lock()
1625            .unwrap_or_else(|poisoned| poisoned.into_inner())
1626            .forget(connection_id);
1627        self.scopes
1628            .write()
1629            .unwrap_or_else(|poisoned| poisoned.into_inner())
1630            .release_connection(connection_id);
1631        registrations
1632    }
1633
1634    pub(crate) fn handle_route_goodbye(
1635        &self,
1636        connection_id: ConnectionId,
1637        route_channel: u16,
1638        route_epoch: u32,
1639    ) -> Result<bool, RouterError> {
1640        debug!(
1641            connection_id = connection_id.get(),
1642            route_channel, route_epoch, "handling route GOODBYE"
1643        );
1644        let RouteRelease::Removed(released_route) = self
1645            .forwarding
1646            .release_client_route(connection_id, route_channel, route_epoch)
1647            .map_err(RouterError::Forwarding)?
1648        else {
1649            return Ok(false);
1650        };
1651        self.emit_route_goodbyes(vec![released_route]);
1652        Ok(true)
1653    }
1654
1655    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1656        for released in released_routes {
1657            let frame = match Frame::build_with_version(
1658                released.negotiated_ver,
1659                FrameType::Goodbye,
1660                control_flags(),
1661                released.channel,
1662                released.epoch,
1663                0,
1664                Vec::new(),
1665            ) {
1666                Ok(frame) => frame,
1667                Err(err) => {
1668                    warn!(
1669                        route_channel = released.channel,
1670                        error = %err,
1671                        "failed to build route GOODBYE frame"
1672                    );
1673                    continue;
1674                }
1675            };
1676            if !released.close_on_delivery_failure() {
1677                crate::forwarding::send_module_route_goodbye(
1678                    &self.counters,
1679                    &released.sink,
1680                    frame,
1681                    released.module_id.as_deref(),
1682                    "client route released",
1683                );
1684                continue;
1685            }
1686            if let Err(err) = released.sink.try_send(frame) {
1687                warn!(
1688                    target_connection_id = released.connection_id.get(),
1689                    route_channel = released.channel,
1690                    error = %err,
1691                    "route GOODBYE was not delivered to client; closing target connection"
1692                );
1693                if self
1694                    .forwarding
1695                    .escalate_client_delivery_failure(
1696                        released.connection_id,
1697                        released.channel,
1698                        released.epoch,
1699                        CloseReason::new(
1700                            "route_goodbye_delivery_failed",
1701                            format!(
1702                                "failed to enqueue route GOODBYE for channel {}: {err}",
1703                                released.channel
1704                            ),
1705                        ),
1706                        crate::forwarding::UndeliveredFrame {
1707                            module_id: released.module_id.as_deref(),
1708                            sink: &released.sink,
1709                        },
1710                    )
1711                    .unwrap_or(false)
1712                {
1713                    self.counters.increment_goodbye_relay_client_failed();
1714                }
1715            }
1716        }
1717    }
1718
1719    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1720    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1721    /// own commit failed after the module had already accepted). Without this, a
1722    /// module that accepts late keeps a binding subc has torn down, so a later
1723    /// frame on that module channel could misdeliver if the channel is reused.
1724    ///
1725    /// Never closes the shared module connection on failure: a dropped notification
1726    /// only wastes a bounded amount of warm module-side state, which the module's
1727    /// own idle reaper reclaims. Only call this once the route.bind relay was
1728    /// actually enqueued to the module — if the relay send itself failed, the
1729    /// module never created a binding and there is nothing to tear down.
1730    fn send_abandoned_route_bind_goodbye(
1731        &self,
1732        module_sink: &crate::FrameSink,
1733        negotiated_ver: u8,
1734        module_channel: u16,
1735        module_epoch: u32,
1736    ) {
1737        let frame = match Frame::build_with_version(
1738            negotiated_ver,
1739            FrameType::Goodbye,
1740            control_flags(),
1741            module_channel,
1742            module_epoch,
1743            0,
1744            Vec::new(),
1745        ) {
1746            Ok(frame) => frame,
1747            Err(err) => {
1748                warn!(
1749                    route_channel = module_channel,
1750                    error = %err,
1751                    "failed to build GOODBYE for abandoned route.bind"
1752                );
1753                return;
1754            }
1755        };
1756        crate::forwarding::send_module_route_goodbye(
1757            &self.counters,
1758            module_sink,
1759            frame,
1760            None,
1761            "abandoned route.bind",
1762        );
1763    }
1764
1765    fn handle_hello(
1766        &self,
1767        connection_id: ConnectionId,
1768        sink: Option<crate::FrameSink>,
1769        frame: Frame,
1770    ) -> Result<Vec<Frame>, RouterError> {
1771        debug!(
1772            connection_id = connection_id.get(),
1773            corr = frame.header.corr,
1774            "handling HELLO"
1775        );
1776        // A module connection has one identity for its entire lifetime. A second
1777        // registration would leave the old registry owner behind while replacing
1778        // its forwarding endpoint and launch nonce.
1779        if self
1780            .registry
1781            .get_module_by_connection(connection_id)
1782            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1783            .is_some()
1784        {
1785            return Ok(vec![control_error_frame(
1786                &frame,
1787                "invalid_hello",
1788                "connection is already registered as a module",
1789            )?]);
1790        }
1791        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1792            Ok(value) => value,
1793            Err(err) => {
1794                return Ok(vec![control_error_frame(
1795                    &frame,
1796                    "invalid_hello",
1797                    format!("malformed HELLO body: {err}"),
1798                )?])
1799            }
1800        };
1801        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1802            return Ok(vec![control_error_frame(
1803                &frame,
1804                "invalid_capability_grammar",
1805                err.to_string(),
1806            )?]);
1807        }
1808        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1809            return Ok(vec![control_error_frame(
1810                &frame,
1811                "invalid_manifest",
1812                err.to_string(),
1813            )?]);
1814        }
1815        if let Some(provenance) = hello_value
1816            .get("manifest")
1817            .and_then(|manifest| manifest.get("provenance"))
1818        {
1819            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1820                return Ok(vec![control_error_frame(
1821                    &frame,
1822                    "invalid_manifest",
1823                    format!("malformed manifest provenance: {err}"),
1824                )?]);
1825            }
1826        }
1827        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1828            Ok(hello) => hello,
1829            Err(err) => {
1830                return Ok(vec![control_error_frame(
1831                    &frame,
1832                    "invalid_hello",
1833                    format!("malformed HELLO body: {err}"),
1834                )?])
1835            }
1836        };
1837
1838        if hello.protocol_ver != hello.manifest.protocol_ver {
1839            return Ok(vec![control_error_frame(
1840                &frame,
1841                "invalid_manifest",
1842                format!(
1843                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1844                    hello.protocol_ver, hello.manifest.protocol_ver
1845                ),
1846            )?]);
1847        }
1848
1849        if hello.manifest.module_id.trim().is_empty() {
1850            return Ok(vec![control_error_frame(
1851                &frame,
1852                "invalid_manifest",
1853                "manifest module_id must not be empty",
1854            )?]);
1855        }
1856
1857        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1858            Ok(negotiated_ver) => negotiated_ver,
1859            Err(message) => {
1860                return Ok(vec![control_error_frame(
1861                    &frame,
1862                    "version_unsupported",
1863                    message,
1864                )?])
1865            }
1866        };
1867
1868        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
1869        // swap is open for this id, the only HELLO admitted as a second process
1870        // is the one carrying the candidate's launch nonce (the swap token), and
1871        // it registers into the candidate slot rather than being refused as a
1872        // duplicate. Run after the reserved gate, a reserved module's candidate
1873        // would be refused `reserved_module` for presenting a nonce that gate
1874        // does not know. See `SupervisorHandle::swap_hello_admission`.
1875        let swap_admission = self
1876            .supervisor
1877            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1878        if swap_admission == SwapHelloAdmission::Refused {
1879            warn!(
1880                module_id = %hello.manifest.module_id,
1881                connection_id = connection_id.get(),
1882                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1883            );
1884            return Ok(vec![control_error_frame(
1885                &frame,
1886                "swap_token_invalid",
1887                format!(
1888                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1889                    hello.manifest.module_id
1890                ),
1891            )?]);
1892        }
1893        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1894
1895        // Reserved-module identity gate: a module_id configured `reserved` may be
1896        // registered ONLY by the process subc spawned for it, proven by echoing the
1897        // one-time launch nonce subc injected. A non-reserved id has no recorded
1898        // nonce and always passes. This blocks a key-holder from impersonating a
1899        // security-boundary module (e.g. the credential vault) while the real one is
1900        // down/restarting and its registration slot is momentarily free. A swap
1901        // candidate has already proven the same thing with its own nonce above.
1902        if let Some(rejection) = (!swap_candidate)
1903            .then(|| {
1904                self.supervisor.reserved_hello_rejection(
1905                    &hello.manifest.module_id,
1906                    hello.launch_nonce.as_deref(),
1907                )
1908            })
1909            .flatten()
1910        {
1911            let message = match rejection {
1912                ReservedHelloRejection::Exact { module_id } => format!(
1913                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1914                ),
1915                ReservedHelloRejection::Prefix {
1916                    prefix,
1917                    owner_module_id,
1918                } => format!(
1919                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1920                    hello.manifest.module_id
1921                ),
1922            };
1923            return Ok(vec![control_error_frame(
1924                &frame,
1925                "reserved_module",
1926                message,
1927            )?]);
1928        }
1929
1930        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1931            &hello.manifest.module_id,
1932            hello.manifest.capabilities.as_ref(),
1933        );
1934        if let Some(refusal) = reserved_capability_refusals.first() {
1935            let capability = refusal.capability.clone();
1936            let bound_module = refusal.claimants[0].clone();
1937            log_duplicate_claim_events(reserved_capability_refusals);
1938            return Ok(vec![control_error_frame(
1939                &frame,
1940                "reserved_capability",
1941                format!(
1942                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1943                    capability, bound_module, hello.manifest.module_id
1944                ),
1945            )?]);
1946        }
1947
1948        // A connection that already opened client routes must not also register as
1949        // a module: cleanup would then release only one side and leak the other.
1950        if self
1951            .forwarding
1952            .connection_has_client_routes(connection_id)
1953            .map_err(RouterError::Forwarding)?
1954        {
1955            return Ok(vec![control_error_frame(
1956                &frame,
1957                "invalid_hello",
1958                "connection has open client routes and cannot also register as a module",
1959            )?]);
1960        }
1961
1962        // Kept for scope sync authority, which goes only to the connection that
1963        // presented the module's current launch nonce. Recorded before the
1964        // registration is attempted: a connection whose registration then fails
1965        // has no registration, so it cannot sync anyway, and cleanup forgets it.
1966        self.hello_launch_nonces
1967            .lock()
1968            .unwrap_or_else(|poisoned| poisoned.into_inner())
1969            .record(connection_id, hello.launch_nonce.as_deref());
1970        let control_ops = effective_module_control_ops(hello.control_ops);
1971        // Built before anything is registered so an encoding failure leaves no
1972        // registry or forwarding state behind.
1973        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1974        if swap_candidate {
1975            return self.register_swap_candidate(
1976                connection_id,
1977                sink,
1978                &frame,
1979                hello.manifest,
1980                negotiated_ver,
1981                control_ops,
1982                hello_ack,
1983            );
1984        }
1985        let registration = match self.registry.register_with_control_ops(
1986            hello.manifest,
1987            negotiated_ver,
1988            connection_id,
1989            control_ops,
1990        ) {
1991            Ok(registration) => registration,
1992            Err(RegistryError::DuplicateModuleId { module_id }) => {
1993                return Ok(vec![control_error_frame(
1994                    &frame,
1995                    "duplicate_module_id",
1996                    format!(
1997                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1998                    ),
1999                )?])
2000            }
2001            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2002                return Ok(vec![control_error_frame(
2003                    &frame,
2004                    "invalid_module_id",
2005                    err.to_string(),
2006                )?])
2007            }
2008            Err(err) => {
2009                return Ok(vec![control_error_frame(
2010                    &frame,
2011                    "registry_error",
2012                    err.to_string(),
2013                )?])
2014            }
2015        };
2016
2017        let reply = if let Some(sink) = sink {
2018            // The forwarding table's module store is also the daemon-to-module
2019            // control-RPC lane, so every HELLO gets a live endpoint even when the
2020            // manifest has no routable provider role. Non-routable modules still
2021            // cannot receive route.bind in production: `handle_route_open` checks
2022            // the registry manifest with `target_has_required_role` before the
2023            // only production call to `begin_route_bind_relay_for` below that
2024            // route.open path. The remaining direct relay callers are unit tests
2025            // and benchmark harnesses that construct forwarding state explicitly.
2026            //
2027            // The HELLO_ACK is queued by the forwarding table itself, before the
2028            // endpoint becomes visible, and is NOT returned as a reply. A module
2029            // reads HELLO_ACK first and exits on anything else; a reply is only
2030            // written after this handler returns, by which time a route.open on
2031            // another connection could already have queued a route.bind request
2032            // for this module ahead of it.
2033            let concurrency = manifest_concurrency(&registration.manifest);
2034            if let Err(err) = self.forwarding.register_module_connection_acked(
2035                connection_id,
2036                registration.manifest.module_id.clone(),
2037                negotiated_ver,
2038                concurrency,
2039                sink,
2040                hello_ack,
2041            ) {
2042                // Forwarding registration failed, so there is no forwarding
2043                // state to tear down. Remove the registry entry and signal the
2044                // release watch directly.
2045                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2046                    crate::supervise::notify_registration_release();
2047                }
2048                return Ok(vec![control_error_frame(
2049                    &frame,
2050                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2051                        "invalid_hello"
2052                    } else {
2053                        forwarding_error_code(&err)
2054                    },
2055                    err.to_string(),
2056                )?]);
2057            }
2058            Vec::new()
2059        } else {
2060            // No sink means no forwarding endpoint, so nothing can be routed
2061            // ahead of the ack; it goes out as the reply.
2062            vec![hello_ack]
2063        };
2064
2065        // Exposure over assumption: Concurrency's serde default is pinned to the
2066        // pre-field behavior (ModuleManaged), so a management surface that is
2067        // genuinely Serial and just never declared it inherits concurrent
2068        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2069        // turns "no module has been bitten yet" into the checkable claim "no
2070        // module is exposed" -- one read of the boot log instead of a fleet
2071        // audit. Detected from the raw HELLO bytes because the serde default
2072        // deliberately erases the absent/declared distinction from the type.
2073        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2074            info!(
2075                module_id = %registration.manifest.module_id,
2076                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2077            );
2078        }
2079
2080        self.apply_registration_capabilities(&registration);
2081
2082        info!(
2083            module_id = %registration.manifest.module_id,
2084            module_version = %registration.manifest.module_version,
2085            negotiated_ver,
2086            routable_provider = manifest_provides_routable_role(&registration.manifest),
2087            connection_id = connection_id.get(),
2088            "module registered"
2089        );
2090
2091        Ok(reply)
2092    }
2093
2094    /// Register a HELLO the swap gate admitted into the candidate slot of the
2095    /// registry and of forwarding, where it is reachable over its own
2096    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2097    /// nothing routes to it until the supervisor cuts over.
2098    ///
2099    /// Registry first, then forwarding, the same order as an ordinary HELLO;
2100    /// a forwarding failure removes the registry entry again. The capability
2101    /// census is not run: it describes routable modules, and this one is not
2102    /// routable until promotion.
2103    #[allow(clippy::too_many_arguments)]
2104    fn register_swap_candidate(
2105        &self,
2106        connection_id: ConnectionId,
2107        sink: Option<crate::FrameSink>,
2108        frame: &Frame,
2109        manifest: ModuleManifest,
2110        negotiated_ver: u8,
2111        control_ops: Vec<String>,
2112        hello_ack: Frame,
2113    ) -> Result<Vec<Frame>, RouterError> {
2114        let module_id = manifest.module_id.clone();
2115        let registration = match self.registry.register_candidate_with_control_ops(
2116            manifest,
2117            negotiated_ver,
2118            connection_id,
2119            control_ops,
2120        ) {
2121            Ok(registration) => registration,
2122            Err(RegistryError::DuplicateModuleId { module_id }) => {
2123                return Ok(vec![control_error_frame(
2124                    frame,
2125                    "duplicate_module_id",
2126                    format!(
2127                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2128                    ),
2129                )?])
2130            }
2131            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2132                return Ok(vec![control_error_frame(
2133                    frame,
2134                    "invalid_module_id",
2135                    err.to_string(),
2136                )?])
2137            }
2138            Err(err) => {
2139                return Ok(vec![control_error_frame(
2140                    frame,
2141                    "registry_error",
2142                    err.to_string(),
2143                )?])
2144            }
2145        };
2146        let reply = if let Some(sink) = sink {
2147            // Same ordering as an ordinary HELLO: the forwarding table queues
2148            // the HELLO_ACK before the candidate endpoint is inserted, because
2149            // a module exits if its first frame after HELLO is anything else.
2150            let concurrency = manifest_concurrency(&registration.manifest);
2151            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2152                connection_id,
2153                module_id.clone(),
2154                negotiated_ver,
2155                concurrency,
2156                sink,
2157                hello_ack,
2158            ) {
2159                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2160                    crate::supervise::notify_registration_release();
2161                }
2162                return Ok(vec![control_error_frame(
2163                    frame,
2164                    forwarding_error_code(&err),
2165                    err.to_string(),
2166                )?]);
2167            }
2168            Vec::new()
2169        } else {
2170            vec![hello_ack]
2171        };
2172        self.supervisor.mark_swap_candidate_admitted(&module_id);
2173        info!(
2174            module_id = %module_id,
2175            module_version = %registration.manifest.module_version,
2176            negotiated_ver,
2177            ready = registration.ready,
2178            connection_id = connection_id.get(),
2179            "swap candidate registered; not routable until cutover"
2180        );
2181        Ok(reply)
2182    }
2183
2184    fn build_hello_ack(
2185        &self,
2186        frame: &Frame,
2187        negotiated_ver: u8,
2188        module_id: &str,
2189    ) -> Result<Frame, RouterError> {
2190        let ack = ModuleHelloAckBody {
2191            negotiated_ver,
2192            subc_ops: module_subc_ops(),
2193            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2194            storage: self
2195                .storage_config
2196                .as_ref()
2197                .map(|cfg| cfg.descriptor_for(module_id)),
2198            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2199        };
2200        let body = serde_json::to_vec(&ack).map_err(|err| {
2201            RouterError::backend(
2202                0,
2203                frame.header.corr,
2204                format!("failed to encode HELLO_ACK: {err}"),
2205            )
2206        })?;
2207
2208        Frame::build_with_version(
2209            negotiated_ver,
2210            FrameType::HelloAck,
2211            control_flags(),
2212            0,
2213            0,
2214            frame.header.corr,
2215            body,
2216        )
2217        .map_err(RouterError::FrameBuild)
2218    }
2219
2220    async fn handle_client_control_request(
2221        &self,
2222        ctx: &RouteCtx,
2223        frame: Frame,
2224        request: ClientControlRequest,
2225    ) -> Result<Vec<Frame>, RouterError> {
2226        match request {
2227            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2228            ClientControlRequest::CatalogList { module_id } => {
2229                self.handle_catalog_list(frame, module_id)
2230            }
2231            ClientControlRequest::RouteOpen {
2232                target,
2233                identity,
2234                consumer_identity,
2235                consumer_capabilities,
2236                role_versions,
2237                admission_facts,
2238                scope,
2239            } => {
2240                self.handle_route_open(
2241                    ctx,
2242                    frame,
2243                    RouteOpenRequest {
2244                        target,
2245                        identity,
2246                        consumer_identity,
2247                        consumer_capabilities,
2248                        role_versions,
2249                        admission_facts,
2250                        scope,
2251                    },
2252                )
2253                .await
2254            }
2255            ClientControlRequest::RoutePoll {
2256                route_channel,
2257                route_epoch,
2258                kind,
2259            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2260            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2261            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2262                self.handle_supervisor_spawn_snapshot(frame)
2263            }
2264            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2265                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2266            }
2267            ClientControlRequest::SupervisorRestart {
2268                module_id,
2269                drain_timeout_ms,
2270            } => {
2271                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2272                    .await
2273            }
2274            ClientControlRequest::SupervisorSwap {
2275                module_id,
2276                ready_timeout_ms,
2277            } => {
2278                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2279                    .await
2280            }
2281            ClientControlRequest::SupervisorReload { module_id } => {
2282                self.handle_supervisor_reload(frame, module_id).await
2283            }
2284            ClientControlRequest::SupervisorRescan { preview } => {
2285                self.handle_supervisor_rescan(frame, preview).await
2286            }
2287            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2288                self.handle_supervisor_release_reserved(frame, module_id)
2289                    .await
2290            }
2291            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2292                self.handle_supervisor_set_enabled(frame, module_id, enabled)
2293                    .await
2294            }
2295            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2296                self.handle_supervisor_health_probe(frame, module_id).await
2297            }
2298            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2299            ClientControlRequest::SupervisorRoutes { module_id } => {
2300                self.handle_supervisor_routes(frame, module_id)
2301            }
2302            ClientControlRequest::SupervisorProvenance { module_id } => {
2303                self.handle_supervisor_provenance(frame, module_id).await
2304            }
2305            ClientControlRequest::SupervisorStderrTail {
2306                module_id,
2307                max_lines,
2308                max_bytes,
2309            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2310            ClientControlRequest::SupervisorTerminals { module_id } => {
2311                self.handle_supervisor_terminals(frame, module_id).await
2312            }
2313        }
2314    }
2315
2316    fn handle_module_control_request(
2317        &self,
2318        connection_id: ConnectionId,
2319        frame: Frame,
2320        request: ModuleControlRequestFromModule,
2321    ) -> Result<Vec<Frame>, RouterError> {
2322        match request {
2323            ModuleControlRequestFromModule::CatalogUpdate {
2324                provides,
2325                capabilities,
2326                ready,
2327            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2328            ModuleControlRequestFromModule::LiveRoots {} => {
2329                let registered = self
2330                    .registry
2331                    .get_module_by_connection(connection_id)
2332                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2333                let Some(registration) = registered else {
2334                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2335                };
2336                let response = self
2337                    .forwarding
2338                    .live_roots(&registration.manifest.module_id)
2339                    .map_err(RouterError::Forwarding)?;
2340                Ok(vec![control_response_body_frame(
2341                    &frame,
2342                    &response,
2343                    "ModuleControlResponseToModule::LiveRoots",
2344                )?])
2345            }
2346            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2347                self.handle_scope_sync(connection_id, frame, generation, scopes)
2348            }
2349            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2350                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2351            }
2352        }
2353    }
2354
2355    /// `scope.sync`: the owner is the module registered on this connection.
2356    /// A connection with no registration (every client connection, `direct`
2357    /// included) is refused `not_registered` before the table is consulted.
2358    fn handle_scope_sync(
2359        &self,
2360        connection_id: ConnectionId,
2361        frame: Frame,
2362        generation: u64,
2363        scopes: Vec<ScopeRecord>,
2364    ) -> Result<Vec<Frame>, RouterError> {
2365        let Some(registration) = self
2366            .registry
2367            .get_module_by_connection(connection_id)
2368            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2369        else {
2370            return Ok(vec![control_error_frame(
2371                &frame,
2372                "not_registered",
2373                "scope.sync requires an active module registration owned by this connection",
2374            )?]);
2375        };
2376        let owner = registration.manifest.module_id;
2377        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2378        let is_current_launch = |connection: ConnectionId| {
2379            self.hello_launch_nonces
2380                .lock()
2381                .unwrap_or_else(|poisoned| poisoned.into_inner())
2382                .presented(connection, current_nonce.as_deref())
2383        };
2384        // Lock order is the scope table, then the forwarding table: the new
2385        // tags are published, and the routes the change closes are selected,
2386        // while the scope table is still write-locked, so no admission can read
2387        // a record whose tag is not yet published.
2388        let mut table = self
2389            .scopes
2390            .write()
2391            .unwrap_or_else(|poisoned| poisoned.into_inner());
2392        let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2393        let drained = match &outcome {
2394            Ok(applied) => self
2395                .forwarding
2396                .publish_scope_changes(&applied.tag_changes)
2397                .map_err(RouterError::Forwarding)?,
2398            Err(_) => Vec::new(),
2399        };
2400        drop(table);
2401        match outcome {
2402            Ok(applied) => {
2403                let counts = ScopeOutcomeCounts::of(&applied.results);
2404                info!(
2405                    owner = %owner,
2406                    generation,
2407                    records = applied.results.len(),
2408                    created = counts.created,
2409                    replaced = counts.replaced,
2410                    updated = counts.updated,
2411                    unchanged = counts.unchanged,
2412                    refused = counts.refused,
2413                    ended = applied.ended.len(),
2414                    tag_changes = applied.tag_changes.len(),
2415                    routes_closed = drained.len(),
2416                    "scope sync accepted"
2417                );
2418                // An accepted sync can still refuse individual records, and the
2419                // owner is the only party that sees the reply. Name them here so
2420                // an operator can tell a refused session from a missing one
2421                // without the owner's logs. Capped so a sync that refuses
2422                // thousands cannot flood the log; the count above is complete.
2423                for refused in applied
2424                    .results
2425                    .iter()
2426                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2427                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2428                {
2429                    warn!(
2430                        owner = %owner,
2431                        generation,
2432                        scope_ref = %refused.scope_ref,
2433                        scope_epoch = refused.scope_epoch,
2434                        code = refused.code.as_deref().unwrap_or(""),
2435                        "scope record refused"
2436                    );
2437                }
2438                self.close_scope_drained_routes(drained);
2439                let response = ModuleControlResponseToModule::ScopeSync {
2440                    generation,
2441                    results: applied.results,
2442                    ended: applied.ended,
2443                };
2444                Ok(vec![control_response_body_frame(
2445                    &frame,
2446                    &response,
2447                    "ModuleControlResponseToModule::ScopeSync",
2448                )?])
2449            }
2450            Err(refusal) => {
2451                info!(
2452                    owner = %owner,
2453                    generation,
2454                    code = refusal.code,
2455                    "scope sync refused"
2456                );
2457                Ok(vec![control_error_frame(
2458                    &frame,
2459                    refusal.code,
2460                    refusal.message,
2461                )?])
2462            }
2463        }
2464    }
2465
2466    /// Tell both ends of each route a scope change closed. The module gets a
2467    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
2468    /// the client's route handle. The client also gets `route.closed` with the
2469    /// scope reason, one push per module and reason, so it can tell a revoked
2470    /// route from an ordinary close and not reopen it.
2471    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2472        if drained.is_empty() {
2473            return;
2474        }
2475        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2476            BTreeMap::new();
2477        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2478        for route in drained {
2479            warn!(
2480                module_id = %route.module_id,
2481                reason = ?route.reason,
2482                client_connection_id = route.client.connection_id.get(),
2483                route_channel = route.client.channel,
2484                "closing route because its scope changed"
2485            );
2486            pushes
2487                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2488                .or_insert_with(|| (route.reason, Vec::new()))
2489                .1
2490                .push(EndpointRoute {
2491                    goodbye_target: route.client.clone(),
2492                    principal: Principal::Unverified,
2493                    bound_at: Instant::now(),
2494                    draining: false,
2495                    drain_reason: None,
2496                });
2497            goodbyes.push(route.module);
2498            goodbyes.push(route.client);
2499        }
2500        for ((module_id, _), (reason, routes)) in pushes {
2501            send_route_control_pushes(
2502                &self.forwarding,
2503                routes,
2504                ClientControlPush::RouteClosed {
2505                    module_id,
2506                    channels: Vec::new(),
2507                    reason,
2508                    drained: false,
2509                    abandoned: 0,
2510                    excluded_subscriptions: 0,
2511                    terminal: Some(false),
2512                },
2513            );
2514        }
2515        self.emit_route_goodbyes(goodbyes);
2516    }
2517
2518    /// `scope.describe`: any registered module may read any scope, because a
2519    /// provider must read the scope a route it serves is stamped with.
2520    fn handle_scope_describe(
2521        &self,
2522        connection_id: ConnectionId,
2523        frame: Frame,
2524        owner: Principal,
2525        scope_ref: String,
2526    ) -> Result<Vec<Frame>, RouterError> {
2527        let registered = self
2528            .registry
2529            .get_module_by_connection(connection_id)
2530            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2531        if registered.is_none() {
2532            return Ok(vec![control_error_frame(
2533                &frame,
2534                "not_registered",
2535                "scope.describe requires an active module registration owned by this connection",
2536            )?]);
2537        }
2538        let description = self
2539            .scopes
2540            .read()
2541            .unwrap_or_else(|poisoned| poisoned.into_inner())
2542            .describe(&owner, &scope_ref);
2543        let owner_configured = match &owner {
2544            // Ask whether the owner is configured (`is_configured`), not
2545            // whether it is on the roster (`get(..).is_some()`): a supervised
2546            // module's process can register and describe a scope before the
2547            // supervisor has put it on the roster.
2548            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2549            _ => false,
2550        };
2551        let response = ModuleControlResponseToModule::ScopeDescribe {
2552            status: description.status,
2553            scope_epoch: description.scope_epoch,
2554            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2555            owner_synced: description.owner_synced,
2556            owner_configured,
2557            scope: description.stamp,
2558        };
2559        Ok(vec![control_response_body_frame(
2560            &frame,
2561            &response,
2562            "ModuleControlResponseToModule::ScopeDescribe",
2563        )?])
2564    }
2565
2566    fn handle_catalog_update(
2567        &self,
2568        connection_id: ConnectionId,
2569        frame: Frame,
2570        provides: Vec<ProviderRole>,
2571        capabilities: Option<CapabilityDeclarations>,
2572        ready: Option<bool>,
2573    ) -> Result<Vec<Frame>, RouterError> {
2574        self.refresh_capability_requirements();
2575        let Some(registration) = self
2576            .registry
2577            .get_module_by_connection(connection_id)
2578            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2579        else {
2580            return Ok(vec![control_error_frame(
2581                &frame,
2582                "not_registered",
2583                "catalog.update requires an active module registration owned by this connection",
2584            )?]);
2585        };
2586
2587        if let Some(message) =
2588            catalog_update_frozen_field_message(&registration.manifest, &provides)
2589        {
2590            return Ok(vec![control_error_frame(
2591                &frame,
2592                "catalog_update_frozen_field",
2593                message,
2594            )?]);
2595        }
2596
2597        let mut candidate = registration.manifest.clone();
2598        candidate.provides = provides.clone();
2599        candidate.capabilities = capabilities
2600            .clone()
2601            .or_else(|| registration.manifest.capabilities.clone());
2602        if let Err(err) = candidate.validate_capability_grammar() {
2603            return Ok(vec![control_error_frame(
2604                &frame,
2605                "invalid_capability_grammar",
2606                err.to_string(),
2607            )?]);
2608        }
2609
2610        // Updates must honor the same reserved owner as initial registration;
2611        // otherwise an empty HELLO could acquire the claim after admission.
2612        let mut conflicts = self
2613            .capability_evaluator
2614            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2615        if let Some(conflict) = conflicts.first() {
2616            let message = format!(
2617                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2618                conflict.capability, conflict.claimants[0], candidate.module_id
2619            );
2620            for conflict in &mut conflicts {
2621                conflict.source = DuplicateClaimSource::CatalogUpdate;
2622            }
2623            log_duplicate_claim_events(conflicts);
2624            return Ok(vec![control_error_frame(
2625                &frame,
2626                "reserved_capability",
2627                message,
2628            )?]);
2629        }
2630
2631        let updated = self
2632            .registry
2633            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2634            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2635        if updated.is_none() {
2636            return Ok(vec![control_error_frame(
2637                &frame,
2638                "not_registered",
2639                "catalog.update requires an active module registration owned by this connection",
2640            )?]);
2641        }
2642        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2643            log_duplicate_claim_events(
2644                self.capability_evaluator
2645                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
2646            );
2647        }
2648        if capability_census_trigger(
2649            registration.manifest.capabilities.as_ref(),
2650            updated
2651                .as_ref()
2652                .and_then(|entry| entry.manifest.capabilities.as_ref()),
2653        ) {
2654            self.enforce_capability_denies();
2655        }
2656        self.refresh_capability_requirements();
2657
2658        let response = ModuleControlResponseToModule::CatalogUpdate {};
2659        control_response_body_frame(
2660            &frame,
2661            &response,
2662            "ModuleControlResponseToModule::CatalogUpdate",
2663        )
2664        .map(|frame| vec![frame])
2665    }
2666
2667    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2668        self.refresh_capability_requirements();
2669        // A bare connection count is ambiguous between many clients holding a
2670        // route each and one client accumulating hundreds, so publish the
2671        // concentration alongside it. Route state is best-effort here: a
2672        // diagnostic endpoint must still answer if the forwarding lock is
2673        // contended.
2674        let mut counters = self.counters.snapshot();
2675        if let (Ok((connections_with_routes, max)), Some(obj)) = (
2676            self.forwarding.client_route_concentration(),
2677            counters.as_object_mut(),
2678        ) {
2679            obj.insert(
2680                "client_connections_with_routes".into(),
2681                connections_with_routes.into(),
2682            );
2683            obj.insert("max_routes_on_one_connection".into(), max.into());
2684        }
2685        // A module that is being fast-refused and a module that is fine look
2686        // identical from a client that retries and succeeds, so name the open
2687        // breakers here. This rides the existing free-form counters object
2688        // rather than a new wire field, so no sibling that deserializes
2689        // `ServerDescribe` has to be rebuilt to keep reading it.
2690        if let (Some(open_breakers), Some(obj)) = (
2691            self.route_bind_breakers.open_snapshot(),
2692            counters.as_object_mut(),
2693        ) {
2694            obj.insert("route_bind_breakers_open".into(), open_breakers);
2695        }
2696        let response = ClientControlResponse::ServerDescribe {
2697            protocol_ver: PROTOCOL_VERSION,
2698            subc_ops: subc_ops(),
2699            capabilities: self.subc_capabilities.as_ref().to_vec(),
2700            connected_clients: self.connected_clients.count(),
2701            counters: Some(counters),
2702            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2703            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2704            capability_requirements: self.capability_requirement_statuses(),
2705            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2706        };
2707        Ok(vec![control_response_body_frame(
2708            &frame,
2709            &response,
2710            "ClientControlResponse::ServerDescribe",
2711        )?])
2712    }
2713
2714    fn handle_catalog_list(
2715        &self,
2716        frame: Frame,
2717        module_id: Option<String>,
2718    ) -> Result<Vec<Frame>, RouterError> {
2719        let (generation, modules) = self.registry.list_modules().map_err(|err| {
2720            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2721        })?;
2722        let entries = modules
2723            .into_iter()
2724            .filter(|registration| {
2725                module_id
2726                    .as_deref()
2727                    .map(|wanted| registration.manifest.module_id == wanted)
2728                    .unwrap_or(true)
2729            })
2730            .map(|registration| {
2731                let not_ready = self.not_ready_reason(&registration);
2732                let roles = registration.manifest.provides;
2733                CatalogEntry {
2734                    module_id: registration.manifest.module_id,
2735                    ready: not_ready.is_none(),
2736                    not_ready,
2737                    module_version: Some(registration.manifest.module_version),
2738                    roles,
2739                    control_ops: registration.control_ops,
2740                    capabilities: registration.manifest.capabilities,
2741                    self_signals: registration.manifest.self_signals,
2742                }
2743            })
2744            .collect();
2745        let response = ClientControlResponse::CatalogList {
2746            generation,
2747            modules: entries,
2748            subc_ops: subc_ops(),
2749        };
2750        Ok(vec![control_response_body_frame(
2751            &frame,
2752            &response,
2753            "ClientControlResponse::CatalogList",
2754        )?])
2755    }
2756
2757    fn route_open_principal(
2758        &self,
2759        frame: &Frame,
2760        consumer_identity: Option<ConsumerIdentity>,
2761    ) -> Result<Result<Principal, Frame>, RouterError> {
2762        let Some(consumer_identity) = consumer_identity else {
2763            return Ok(Ok(Principal::Direct));
2764        };
2765
2766        if self.supervisor.spawned_consumer_authorized(
2767            &consumer_identity.module_id,
2768            &consumer_identity.launch_nonce,
2769        ) {
2770            return Ok(Ok(Principal::Reserved {
2771                module_id: consumer_identity.module_id,
2772            }));
2773        }
2774
2775        Ok(Err(control_error_frame(
2776            frame,
2777            "bad_consumer_identity",
2778            format!(
2779                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2780                consumer_identity.module_id
2781            ),
2782        )?))
2783    }
2784
2785    /// Ordinary `route.open` refusals go through here; admission and breaker
2786    /// refusals log separately with their capacity or breaker state. The daemon can
2787    /// attest which code it sent: without the event, a client's "the daemon
2788    /// refused me" and the daemon's own view could only be reconciled by
2789    /// argument. Malformed input (`invalid_project_root`) does not come here;
2790    /// rejecting a request that was never a valid open is not a refusal of one.
2791    fn route_open_refusal_frame(
2792        &self,
2793        ctx: &RouteCtx,
2794        frame: &Frame,
2795        module_id: &str,
2796        reason: &'static str,
2797        code: &'static str,
2798        message: impl Into<String>,
2799    ) -> Result<Frame, RouterError> {
2800        self.observe_route_open_refusal(ctx, module_id, reason, code);
2801        control_error_frame(frame, code, message.into())
2802    }
2803
2804    /// Refuse a `route.open` because the target module's bind-relay breaker is
2805    /// open, without attempting the relay.
2806    ///
2807    /// The wire code is `module_timeout`, which is the truth (the module has
2808    /// not been answering binds) and which both SDKs already classify as
2809    /// retryable with capped backoff. Reusing it is what keeps this change out
2810    /// of both SDKs; the daemon-side distinction lives in the counter key
2811    /// instead.
2812    ///
2813    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
2814    /// While a breaker is open this fires on every open to that module, and the
2815    /// stall written up in `docs/designs/route-open-head-of-line.md` already
2816    /// produced 261 lines about a single module inside 3000 lines of daemon
2817    /// log. The rare transitions are logged at warn/info instead and the volume
2818    /// is carried by the counter, so the evidence survives without the flood.
2819    /// The debug line keeps a per-refusal record reachable for whoever turns
2820    /// the level up.
2821    fn route_open_breaker_refusal_frame(
2822        &self,
2823        ctx: &RouteCtx,
2824        frame: &Frame,
2825        module_id: &str,
2826        consecutive_timeouts: u32,
2827        retry_in: Duration,
2828        probe_in_flight: bool,
2829    ) -> Result<Frame, RouterError> {
2830        self.counters
2831            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2832        debug!(
2833            target: "control",
2834            code = "module_timeout",
2835            module_id = ?module_id,
2836            connection_id = ctx.connection_id.get(),
2837            consecutive_timeouts,
2838            retry_in_ms = retry_in.as_millis() as u64,
2839            probe_in_flight,
2840            "route.open refused by open bind-relay breaker"
2841        );
2842        // Say what a caller can act on. An open bind-relay breaker means the
2843        // module timed out accepting several new routes in a row. The module
2844        // is still running and its established routes keep working; only new
2845        // route.open requests are refused until the cooldown ends and one
2846        // test route (the probe) gets through. A message that only counts
2847        // failed relays reads as "the module is down" to a worker that sees it.
2848        let detail = if probe_in_flight {
2849            "one test route is already being tried; retry once it settles".to_string()
2850        } else {
2851            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
2852        };
2853        control_error_frame(
2854            frame,
2855            "module_timeout",
2856            format!(
2857                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
2858                 timed out in a row); {detail}; its established routes are unaffected"
2859            ),
2860        )
2861    }
2862
2863    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
2864    /// requester's bytes (an unknown target is whatever the client sent) and
2865    /// is Debug-formatted so control characters land in the log escaped
2866    /// rather than as terminal sequences for whoever tails it.
2867    ///
2868    /// `reason` names the check that refused, because one wire code has
2869    /// several senders: after a module registers, `target_unavailable` can
2870    /// come from a missing role, an inactive registration, a supervisor that
2871    /// has not marked the process live, a missing forwarding connection, or a
2872    /// failed relay, and a log that records only the code cannot say which of
2873    /// them fired. It is a static, daemon-chosen label per branch, so it is
2874    /// safe to print plainly and stays a closed set.
2875    fn observe_route_open_refusal(
2876        &self,
2877        ctx: &RouteCtx,
2878        module_id: &str,
2879        reason: &'static str,
2880        code: &'static str,
2881    ) {
2882        self.counters.increment_route_open_refused(code);
2883        info!(
2884            target: "control",
2885            code,
2886            reason,
2887            module_id = ?module_id,
2888            connection_id = ctx.connection_id.get(),
2889            "route.open refused"
2890        );
2891        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2892            self.route_outages.record_not_serving(module_id, reason);
2893        }
2894    }
2895
2896    /// Record an ACCEPTED route.open.
2897    ///
2898    /// Refusals have been logged and counted since the attestation work; accepts
2899    /// were invisible, so the daemon knew every principal it stamped and wrote
2900    /// none of them down. The party that attests the identity was the only party
2901    /// not recording it, which left a credential vault unable to name the sender
2902    /// of a call that reached it (claustrum #43) and left the launch-nonce
2903    /// concurrency question unanswerable from the outside.
2904    ///
2905    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
2906    /// `code`/`module_id`/`connection_id` returns both directions of the same
2907    /// decision rather than two shapes a reader has to join by hand.
2908    ///
2909    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
2910    /// and the difference carries information rather than being an
2911    /// inconsistency. This line is only reachable after a successful bind to a
2912    /// REGISTERED module, so the value has already passed HELLO validation
2913    /// including the path-hazard refusal and cannot contain control bytes. A
2914    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
2915    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
2916    ///
2917    /// Bare is also what every other daemon line already emits (`module
2918    /// registered`, `configured module supervised`). Shipping `?module_id` here
2919    /// made this instrument the only one in the file whose ids did not answer
2920    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
2921    /// line whose whole purpose is being grepped beside its sibling.
2922    ///
2923    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
2924    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
2925    /// forwards every field type through it and a bare `&str` and a `?`-escaped
2926    /// one are recorded identically. A test written against that harness passes
2927    /// either way -- I wrote one, measured it, and deleted it rather than ship a
2928    /// green assertion that cannot fail. The same limit applies to the escaping
2929    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
2930    /// it reads as a guard on the Debug escaping and cannot detect its removal.
2931    /// Fencing either needs the real formatter, not the capture layer.
2932    ///
2933    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
2934    /// unix-socket options and subc is loopback TCP, so there is no peer identity
2935    /// to record. The ephemeral port would decay within minutes and answer only a
2936    /// live question. The identity question is instead answered by counting
2937    /// distinct live connections presenting one module's `consumer_identity` --
2938    /// "is anyone else holding this secret" rather than "is this the right
2939    /// process".
2940    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2941        self.route_outages.record_accepted(module_id);
2942        self.counters.increment_route_open_accepted(principal);
2943        info!(
2944            target: "control",
2945            principal,
2946            module_id,
2947            connection_id = ctx.connection_id.get(),
2948            "route.open accepted"
2949        );
2950    }
2951
2952    fn supervised_absent_route_open_refusal_frame(
2953        &self,
2954        ctx: &RouteCtx,
2955        frame: &Frame,
2956        module_id: &str,
2957        code: &'static str,
2958        status: &crate::supervise::ModuleStatus,
2959    ) -> Result<Frame, RouterError> {
2960        self.counters.increment_route_open_refused(code);
2961        info!(
2962            target: "control",
2963            code,
2964            reason = "supervised_not_registered",
2965            module_id = ?module_id,
2966            connection_id = ctx.connection_id.get(),
2967            state = %status.state,
2968            enabled = status.enabled,
2969            live = status.live,
2970            "route.open refused"
2971        );
2972        // A supervised module whose process has not registered is not
2973        // serving, whatever the reason; the supervisor knows this id, so it is
2974        // safe to track.
2975        self.route_outages
2976            .record_not_serving(module_id, "supervised_not_registered");
2977        control_error_frame(
2978            frame,
2979            code,
2980            format!(
2981                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2982                status.state, status.enabled, status.live
2983            ),
2984        )
2985    }
2986
2987    async fn handle_route_open(
2988        &self,
2989        ctx: &RouteCtx,
2990        frame: Frame,
2991        request: RouteOpenRequest,
2992    ) -> Result<Vec<Frame>, RouterError> {
2993        let RouteOpenRequest {
2994            target,
2995            mut identity,
2996            consumer_identity,
2997            consumer_capabilities,
2998            role_versions,
2999            admission_facts,
3000            scope,
3001        } = request;
3002        let target_module_id = target_module_id(&target).to_string();
3003        if self
3004            .registry
3005            .get_module_by_connection(ctx.connection_id)
3006            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3007            .is_some()
3008        {
3009            return Ok(vec![control_error_frame(
3010                &frame,
3011                "invalid_request",
3012                "module connections cannot open client routes",
3013            )?]);
3014        }
3015        debug!(
3016            connection_id = ctx.connection_id.get(),
3017            corr = frame.header.corr,
3018            module_id = %target_module_id,
3019            "handling route.open"
3020        );
3021
3022        // A malformed declaration is refused first, before anything about the
3023        // target is looked up: the same body would be refused against any
3024        // module, so the caller learns nothing by retrying or waiting. An empty
3025        // map declares nothing and travels as no field at all, so a provider
3026        // only ever sees a missing field or a non-empty one.
3027        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3028        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3029            self.observe_route_open_refusal(
3030                ctx,
3031                &target_module_id,
3032                "invalid_role_versions",
3033                error_codes::INVALID_REQUEST,
3034            );
3035            return Ok(vec![control_error_body_frame(
3036                &frame,
3037                ErrorBody {
3038                    code: error_codes::INVALID_REQUEST.to_string(),
3039                    message: error.to_string(),
3040                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3041                },
3042            )?]);
3043        }
3044
3045        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
3046        // opposite. Below, a caller learns whether a module is unregistered,
3047        // supervised-but-down (with state/enabled/live), or registered without the
3048        // requested role. Elsewhere that is an enumeration leak: a probe learning
3049        // the shape of a fleet it cannot otherwise see.
3050        //
3051        // It is not one here, and the reason is the ACCESS MODEL rather than
3052        // anything about these errors. Reaching route.open requires the
3053        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
3054        // connection file, so any caller who completes it already runs as this
3055        // user -- and can read subc.jsonc for the module list and `ck module
3056        // status` for live state. The reply discloses nothing the caller cannot
3057        // read more easily from disk, while the precision is load-bearing:
3058        // `unknown_module` is retryable and a missing role is not.
3059        //
3060        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
3061        // remote transport, a sandboxed caller, a shared-host mode -- THAT
3062        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
3063        let Some(registration) = self
3064            .registry
3065            .get_module(&target_module_id)
3066            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3067        else {
3068            if let Some((status, warming)) =
3069                self.supervisor_status(&target_module_id, frame.header.corr)?
3070            {
3071                // BEFORE the two availability codes below, because for a module
3072                // that speaks no subc wire both of them are false comfort: they
3073                // say "not right now" and are retried, and this module will
3074                // never register no matter how long the caller waits. The
3075                // absence here is the declaration being honoured, not a module
3076                // that is late.
3077                if status.protocol == ModuleProtocol::None {
3078                    return Ok(vec![self.route_open_refusal_frame(
3079                        ctx,
3080                        &frame,
3081                        &target_module_id,
3082                        "protocol_none",
3083                        error_codes::MODULE_NO_PROTOCOL,
3084                        format!(
3085                            "module_id '{target_module_id}' is declared protocol: none; \
3086                             it speaks no subc wire and serves no routes"
3087                        ),
3088                    )?]);
3089                }
3090                let code = if warming {
3091                    "module_warming"
3092                } else {
3093                    "target_unavailable"
3094                };
3095                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3096                    ctx,
3097                    &frame,
3098                    &target_module_id,
3099                    code,
3100                    &status,
3101                )?]);
3102            }
3103            if let Some(removed_ago_ms) =
3104                self.supervisor.removal_tombstone_age_ms(&target_module_id)
3105            {
3106                return Ok(vec![self.route_open_refusal_frame(
3107                    ctx,
3108                    &frame,
3109                    &target_module_id,
3110                    "removed",
3111                    error_codes::MODULE_REMOVED,
3112                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3113                )?]);
3114            }
3115            return Ok(vec![self.route_open_refusal_frame(
3116                ctx,
3117                &frame,
3118                &target_module_id,
3119                "not_registered",
3120                error_codes::UNKNOWN_MODULE,
3121                format!("module_id '{target_module_id}' is not registered"),
3122            )?]);
3123        };
3124
3125        // Best-effort only: registry readiness and forwarding reservation use
3126        // different locks, so a module can flip readiness between this read and
3127        // the relay. Modules must still tolerate an `on_bind` while not ready.
3128        if !registration.ready {
3129            self.counters
3130                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3131            info!(
3132                target: "control",
3133                code = error_codes::MODULE_WARMING,
3134                module_id = ?target_module_id,
3135                connection_id = ctx.connection_id.get(),
3136                reason = "declared_not_ready",
3137                "route.open refused"
3138            );
3139            // The module is registered but says it cannot take work, which is
3140            // an outage from the caller's side even though its process is up.
3141            self.route_outages
3142                .record_not_serving(&target_module_id, "declared_not_ready");
3143            return Ok(vec![control_error_body_frame(
3144                &frame,
3145                ErrorBody {
3146                    code: error_codes::MODULE_WARMING.to_string(),
3147                    message: format!(
3148                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3149                    ),
3150                    detail: Some(serde_json::json!({
3151                        "reason": "declared_not_ready"
3152                    })),
3153                },
3154            )?]);
3155        }
3156
3157        // Effective readiness, second half: a module that declares a capability
3158        // `need: required` is not routable while that capability has no
3159        // registered provider. It is enforced HERE, as a retryable routing
3160        // refusal, and deliberately not as spawn ordering or a boot block. The
3161        // module is still started and registered and can make its own calls;
3162        // spawn ordering is a promise that cannot be kept once a provider
3163        // crashes at runtime, and refusing to boot would stop the whole
3164        // machine, including the tools needed to fix its configuration.
3165        //
3166        // "Provided" is the evaluator's verdict, which counts a provider as
3167        // soon as it has REGISTERED, not once it is ready. Two modules that
3168        // require each other's capabilities are therefore both routable once
3169        // both register; counting readiness instead would deadlock them.
3170        //
3171        // Only new opens are refused. Routes already bound when a provider
3172        // goes away stay bound: nothing here tears them down, and the module
3173        // answers them as it can. Like the readiness read above this is
3174        // best-effort against a provider registering or leaving concurrently.
3175        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
3176            self.counters
3177                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3178            info!(
3179                target: "control",
3180                code = error_codes::MODULE_WARMING,
3181                module_id = ?target_module_id,
3182                connection_id = ctx.connection_id.get(),
3183                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3184                capability = %capability,
3185                "route.open refused"
3186            );
3187            return Ok(vec![control_error_body_frame(
3188                &frame,
3189                ErrorBody {
3190                    code: error_codes::MODULE_WARMING.to_string(),
3191                    message: format!(
3192                        "module_id '{target_module_id}' requires capability '{capability}', \
3193                         which no registered module provides; retry"
3194                    ),
3195                    detail: Some(serde_json::json!({
3196                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3197                        "capability": capability,
3198                    })),
3199                },
3200            )?]);
3201        }
3202
3203        if !target_has_required_role(&target, &registration.manifest.provides) {
3204            return Ok(vec![self.route_open_refusal_frame(
3205                ctx,
3206                &frame,
3207                &target_module_id,
3208                "role_not_provided",
3209                "target_unavailable",
3210                format!("module_id '{target_module_id}' does not provide the requested target"),
3211            )?]);
3212        }
3213
3214        if registration.state != ChannelState::Active {
3215            return Ok(vec![self.route_open_refusal_frame(
3216                ctx,
3217                &frame,
3218                &target_module_id,
3219                "registration_not_active",
3220                "target_unavailable",
3221                format!("module_id '{target_module_id}' is not active"),
3222            )?]);
3223        }
3224
3225        if self
3226            .forwarding
3227            .module_is_draining(&target_module_id)
3228            .map_err(RouterError::Forwarding)?
3229        {
3230            return Ok(vec![self.route_open_refusal_frame(
3231                ctx,
3232                &frame,
3233                &target_module_id,
3234                "reloading",
3235                "module_reloading",
3236                format!("module_id '{target_module_id}' is reloading"),
3237            )?]);
3238        }
3239
3240        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3241            process_liveness.process_live(&target_module_id) == Some(false)
3242        }) {
3243            // A module the supervisor is restarting or reloading can still hold
3244            // a registration: the old process before its connection closes, or
3245            // a new one that registered while the supervisor was draining. The
3246            // forwarding table does not see that as draining, but the consumer
3247            // should still be told to retry soon, exactly as for the drain
3248            // above, rather than that the target is unavailable.
3249            if process_liveness.process_replacing(&target_module_id) {
3250                return Ok(vec![self.route_open_refusal_frame(
3251                    ctx,
3252                    &frame,
3253                    &target_module_id,
3254                    "reloading",
3255                    "module_reloading",
3256                    format!("module_id '{target_module_id}' is reloading"),
3257                )?]);
3258            }
3259            return Ok(vec![self.route_open_refusal_frame(
3260                ctx,
3261                &frame,
3262                &target_module_id,
3263                "supervisor_not_live",
3264                "target_unavailable",
3265                format!("module_id '{target_module_id}' is not live"),
3266            )?]);
3267        }
3268
3269        if !self
3270            .forwarding
3271            .has_live_module_connection(&target_module_id)
3272            .map_err(RouterError::Forwarding)?
3273        {
3274            return Ok(vec![self.route_open_refusal_frame(
3275                ctx,
3276                &frame,
3277                &target_module_id,
3278                "no_forwarding_connection",
3279                "target_unavailable",
3280                format!("module_id '{target_module_id}' has no live forwarding connection"),
3281            )?]);
3282        }
3283
3284        if let Some(error) =
3285            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3286        {
3287            self.observe_route_open_refusal(
3288                ctx,
3289                &target_module_id,
3290                "op_not_allowed",
3291                "op_not_allowed",
3292            );
3293            return Ok(vec![error]);
3294        }
3295
3296        let principal = match self.route_open_principal(&frame, consumer_identity)? {
3297            Ok(principal) => principal,
3298            Err(error) => {
3299                self.observe_route_open_refusal(
3300                    ctx,
3301                    &target_module_id,
3302                    "bad_consumer_identity",
3303                    "bad_consumer_identity",
3304                );
3305                return Ok(vec![error]);
3306            }
3307        };
3308
3309        // This is attested, control-plane policy for supervised module origins.
3310        // Keep it before route reservation and out of the opaque forwarding hot
3311        // path: data frames must never acquire a per-frame capability check.
3312        if let Principal::Reserved {
3313            module_id: opening_module_id,
3314        } = &principal
3315        {
3316            if let Some(opening_registration) = self
3317                .registry
3318                .get_module(opening_module_id)
3319                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3320            {
3321                if let Some(capability) =
3322                    denied_capability(&opening_registration.manifest, &registration.manifest)
3323                {
3324                    warn!(
3325                        opening_module_id,
3326                        target_module_id,
3327                        capability,
3328                        "refusing route.open because an attested capability deny edge matches"
3329                    );
3330                    return Ok(vec![self.route_open_refusal_frame(
3331                        ctx,
3332                        &frame,
3333                        &target_module_id,
3334                        "capability_deny_edge",
3335                        "capability_forbidden",
3336                        format!(
3337                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3338                        ),
3339                    )?]);
3340                }
3341            }
3342        }
3343
3344        if admission_facts.is_some() {
3345            let carrier_matches = matches!(
3346                &principal,
3347                Principal::Reserved { module_id }
3348                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3349            );
3350            if !carrier_matches {
3351                return Ok(vec![self.route_open_refusal_frame(
3352                    ctx,
3353                    &frame,
3354                    &target_module_id,
3355                    "admission_facts_carrier_not_permitted",
3356                    "admission_facts_not_permitted",
3357                    "admission facts may only be carried by the configured reserved module",
3358                )?]);
3359            }
3360
3361            let target_allowed = self
3362                .admission_facts_targets
3363                .as_ref()
3364                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3365            if !target_allowed {
3366                return Ok(vec![self.route_open_refusal_frame(
3367                    ctx,
3368                    &frame,
3369                    &target_module_id,
3370                    "admission_facts_target_not_listed",
3371                    "admission_facts_target_not_allowed",
3372                    format!(
3373                        "admission facts are not permitted for target module_id '{target_module_id}'"
3374                    ),
3375                )?]);
3376            }
3377
3378            // Keep the value opaque to subc. The downstream admission validator owns
3379            // schema and semantic checks; this daemon only enforces carrier authority
3380            // and the configured destination allowlist.
3381        }
3382
3383        // Scope admission, on the attested principal above and never on the
3384        // request body. The tag read here travels with the pending bind and is
3385        // compared with the published one at commit, so a sync between here
3386        // and the module's ack refuses the open instead of binding a stamp
3387        // that is no longer true.
3388        let (bound_scope, scope_stamp) = match scope {
3389            None => (None, None),
3390            Some(selector) => {
3391                let owner_configured = match &selector.owner {
3392                    // A reserved owner counts as configured from before its
3393                    // process is spawned (see `SupervisorHandle::is_configured`).
3394                    // So an owner that has not synced its scopes yet is refused
3395                    // as retryable (`scope_not_synced`), not as one that will
3396                    // never sync.
3397                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3398                    _ => false,
3399                };
3400                let admitted = self
3401                    .scopes
3402                    .read()
3403                    .unwrap_or_else(|poisoned| poisoned.into_inner())
3404                    .admit(&principal, &target_module_id, &selector, owner_configured);
3405                match admitted {
3406                    Ok(admission) => (
3407                        Some(BoundScope {
3408                            owner: admission.owner,
3409                            scope_ref: admission.stamp.scope_ref.clone(),
3410                            tag: admission.tag,
3411                        }),
3412                        Some(admission.stamp),
3413                    ),
3414                    Err(refusal) => {
3415                        return Ok(vec![self.route_open_refusal_frame(
3416                            ctx,
3417                            &frame,
3418                            &target_module_id,
3419                            refusal.code,
3420                            refusal.code,
3421                            refusal.message,
3422                        )?]);
3423                    }
3424                }
3425            }
3426        };
3427
3428        // Bind admits a root that no longer exists on disk, because refusing here
3429        // closes the only exit from a paused run: cancel needs a bound route, and a
3430        // renamed or reclaimed directory makes that route unopenable forever. The
3431        // run itself is intact and still addressable by its recorded identity.
3432        //
3433        // This does NOT relax the rule the strict constructor protects. That rule is
3434        // that no root is ever aliased into NEW durable state -- a missing component
3435        // can reappear as a symlink elsewhere, which would move the identity and
3436        // split a session's history across two of them. The engine now refuses the
3437        // two operations that create such state (send and import) at admission,
3438        // which is a narrower way to hold the same invariant: reads and terminations
3439        // are admitted, writes are not. That refusal had to ship before this line
3440        // changed, or there is an interval where a send commits under a provisional
3441        // identity -- the exact failure the original policy existed to prevent.
3442        //
3443        // Resolution follows realpath rather than lexical cleanup: the longest
3444        // existing ancestor is canonicalized and the missing tail re-appended, so a
3445        // live root is unchanged and a vanished leaf keeps the identity it was
3446        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
3447        // same caller the moment the directory vanished, which strands the run more
3448        // quietly than refusing it.
3449        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3450            Ok(project_root) => project_root,
3451            Err(err) => {
3452                return Ok(vec![control_error_frame(
3453                    &frame,
3454                    "invalid_project_root",
3455                    err.to_string(),
3456                )?])
3457            }
3458        };
3459        identity.project_root = project_root.as_path().to_path_buf();
3460
3461        // Last gate before any relay work, and deliberately after the cheap
3462        // registry and availability checks above: those name a more precise
3463        // condition (unknown, removed, reloading) and a caller is better served
3464        // by the precise code than by this one.
3465        //
3466        // Everything below this point costs an egress permit, a reserved handle
3467        // pair and, if the module does not answer, the whole relay budget. The
3468        // reader no longer waits for that budget, so cap each target explicitly;
3469        // serial dispatch used to provide the accidental cap of one relay per
3470        // connection. Admission is a mutex-protected count and never waits.
3471        let _concurrency_guard = match self
3472            .route_bind_concurrency
3473            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3474        {
3475            Ok(guard) => guard,
3476            Err(in_flight) => {
3477                return Ok(vec![self.route_open_target_capacity_refusal(
3478                    ctx,
3479                    &frame,
3480                    &target_module_id,
3481                    in_flight,
3482                )?]);
3483            }
3484        };
3485
3486        // A module that has already burned the whole budget `threshold` times
3487        // in a row does not get to charge it again until a probe says it recovered.
3488        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3489            RouteBindAdmission::Admitted { guard, probe } => {
3490                if probe {
3491                    info!(
3492                        module_id = %target_module_id,
3493                        connection_id = ctx.connection_id.get(),
3494                        "route.bind breaker half-open: admitting one probe"
3495                    );
3496                }
3497                guard
3498            }
3499            RouteBindAdmission::Refused {
3500                consecutive_timeouts,
3501                retry_in,
3502                probe_in_flight,
3503            } => {
3504                return Ok(vec![self.route_open_breaker_refusal_frame(
3505                    ctx,
3506                    &frame,
3507                    &target_module_id,
3508                    consecutive_timeouts,
3509                    retry_in,
3510                    probe_in_flight,
3511                )?]);
3512            }
3513        };
3514
3515        // Resolve the per-module budget here so the wait matches the operator's
3516        // intent for this specific target. A per-module override in
3517        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
3518        // daemons) wins over the daemon-wide default.
3519        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3520        let relay_deadline = Instant::now() + route_bind_relay_timeout;
3521        let pending = match self
3522            .forwarding
3523            .begin_route_bind_relay_for(
3524                ctx.connection_id,
3525                ctx.egress.clone(),
3526                response_version(&frame),
3527                frame.header.corr,
3528                &target_module_id,
3529                principal.clone(),
3530                bound_scope,
3531                Some(project_root),
3532                relay_deadline,
3533            )
3534            .await
3535        {
3536            Ok(pending) => pending,
3537            Err(err) => {
3538                return Ok(vec![self.route_open_refusal_frame(
3539                    ctx,
3540                    &frame,
3541                    &target_module_id,
3542                    "relay_reservation_failed",
3543                    forwarding_error_code(&err),
3544                    err.to_string(),
3545                )?])
3546            }
3547        };
3548        let crate::forwarding::PendingRouteBindRelay {
3549            endpoint,
3550            module_sink,
3551            negotiated_ver,
3552            client_channel,
3553            client_epoch,
3554            module_channel,
3555            module_epoch,
3556            corr: relay_corr,
3557            receiver,
3558        } = pending;
3559        let mut reservation =
3560            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3561
3562        debug!(
3563            connection_id = ctx.connection_id.get(),
3564            client_channel,
3565            client_epoch,
3566            module_channel,
3567            module_epoch,
3568            "reserved route handle pair"
3569        );
3570        // Rendered BEFORE the move into the relay, because the accept arm below
3571        // is where it is logged and the principal is gone by then.
3572        let principal_label = match &principal {
3573            Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3574            Principal::Direct => "direct".to_string(),
3575            other => format!("{other:?}"),
3576        };
3577        let relay = ModuleControlRequest::RouteBind {
3578            route_channel: module_channel,
3579            epoch: module_epoch,
3580            target,
3581            identity,
3582            principal: Some(principal),
3583            consumer_capabilities,
3584            role_versions,
3585            admission_facts,
3586            scope: scope_stamp,
3587        };
3588        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3589            RouterError::backend(
3590                0,
3591                frame.header.corr,
3592                format!("failed to encode route.bind request: {err}"),
3593            )
3594        })?;
3595        let relay_frame = Frame::build_with_version(
3596            negotiated_ver,
3597            FrameType::Request,
3598            control_flags(),
3599            0,
3600            0,
3601            relay_corr,
3602            relay_body,
3603        )
3604        .map_err(RouterError::FrameBuild)?;
3605
3606        if let Err(err) = module_sink.send(relay_frame).await {
3607            reservation.release_and_disarm();
3608            return Ok(vec![self.route_open_refusal_frame(
3609                ctx,
3610                &frame,
3611                &target_module_id,
3612                "relay_send_failed",
3613                "target_unavailable",
3614                err.to_string(),
3615            )?]);
3616        }
3617
3618        if !self
3619            .forwarding
3620            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3621            .map_err(RouterError::Forwarding)?
3622        {
3623            self.send_abandoned_route_bind_goodbye(
3624                &module_sink,
3625                negotiated_ver,
3626                module_channel,
3627                module_epoch,
3628            );
3629        }
3630
3631        match timeout_at(relay_deadline, receiver).await {
3632            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3633                reservation.disarm();
3634                if breaker.record_accepted() {
3635                    info!(
3636                        module_id = %target_module_id,
3637                        "route.bind breaker closed: the probe was accepted"
3638                    );
3639                }
3640                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3641                Ok(Vec::new())
3642            }
3643            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3644                reservation.release_and_disarm();
3645                // A module that says no in microseconds is healthy. Rejection
3646                // is a different condition with its own refusal and must not
3647                // move the breaker.
3648                breaker.record_inconclusive();
3649                // The daemon's own commit re-check refused the bind because the
3650                // scope ended or changed after admission. The module accepted;
3651                // counting it as a module rejection would blame the module.
3652                let scope_code = match body.code.as_str() {
3653                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3654                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3655                    _ => None,
3656                };
3657                if let Some(code) = scope_code {
3658                    self.observe_route_open_refusal(
3659                        ctx,
3660                        &target_module_id,
3661                        "scope_changed_before_commit",
3662                        code,
3663                    );
3664                    return Ok(vec![control_error_body_frame(&frame, body)?]);
3665                }
3666                self.counters
3667                    .increment_route_open_refused("module_rejected");
3668                info!(
3669                    target: "control",
3670                    code = "module_rejected",
3671                    module_code = ?body.code,
3672                    module_id = ?target_module_id,
3673                    connection_id = ctx.connection_id.get(),
3674                    "route.open refused"
3675                );
3676                Ok(vec![control_error_body_frame(&frame, body)?])
3677            }
3678            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3679                reservation.release_and_disarm();
3680                breaker.record_inconclusive();
3681                // Fires when the module's connection closes while a relayed
3682                // bind is pending -- typically a caller racing a module restart
3683                // whose bind was relayed BEFORE the drain mark went up. Logged
3684                // because the caller sees only its own error and the fleet has
3685                // already spent one diagnosis round unable to tell this arm
3686                // from a relay timeout without daemon-side evidence.
3687                tracing::warn!(
3688                    module_id = %target_module_id,
3689                    "route.bind relay abandoned: {message}"
3690                );
3691                Ok(vec![self.route_open_refusal_frame(
3692                    ctx,
3693                    &frame,
3694                    &target_module_id,
3695                    "relay_abandoned",
3696                    "target_unavailable",
3697                    message,
3698                )?])
3699            }
3700            Ok(Err(_)) => {
3701                reservation.release_and_disarm();
3702                breaker.record_inconclusive();
3703                Ok(vec![self.route_open_refusal_frame(
3704                    ctx,
3705                    &frame,
3706                    &target_module_id,
3707                    "relay_waiter_canceled",
3708                    "target_unavailable",
3709                    "route.bind relay waiter was canceled before the module responded",
3710                )?])
3711            }
3712            Err(_) => {
3713                reservation.release_and_disarm();
3714                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
3715                // answer at all is the one condition a fast refusal can
3716                // usefully stand in for; every other arm already answered.
3717                if let Some(opened) = breaker.record_timeout(
3718                    self.route_bind_breaker_threshold,
3719                    self.route_bind_breaker_cooldown,
3720                ) {
3721                    warn!(
3722                        module_id = %target_module_id,
3723                        consecutive_timeouts = opened.consecutive_timeouts,
3724                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3725                        reopened_after_probe = opened.reopened_after_probe,
3726                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3727                    );
3728                }
3729                // The generous budget just burned to no answer: the module is
3730                // registered and its connection is up, but its bind handler sat
3731                // on the ack for the full budget (warm-on-bind, cold configure,
3732                // or a wedged handler). Every earlier unavailability shape
3733                // fast-refuses BEFORE the relay, so this arm firing means the
3734                // slowness is module-side -- log it so the per-module timeline
3735                // is reconstructable without client audit rows.
3736                tracing::warn!(
3737                    module_id = %target_module_id,
3738                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3739                    "route.bind relay timed out: module did not ack within budget"
3740                );
3741                Ok(vec![self.route_open_refusal_frame(
3742                    ctx,
3743                    &frame,
3744                    &target_module_id,
3745                    "relay_timed_out",
3746                    "module_timeout",
3747                    format!(
3748                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
3749                        route_bind_relay_timeout
3750                    ),
3751                )?])
3752            }
3753        }
3754    }
3755
3756    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3757        let response = ClientControlResponse::SupervisorSpawnSnapshot {
3758            snapshot: self.supervisor.spawn_snapshot(),
3759        };
3760        Ok(vec![control_response_body_frame(
3761            &frame,
3762            &response,
3763            "ClientControlResponse::SupervisorSpawnSnapshot",
3764        )?])
3765    }
3766
3767    fn handle_supervisor_spawn_subscribe(
3768        &self,
3769        ctx: &RouteCtx,
3770        frame: Frame,
3771        since: Option<SpawnCursor>,
3772    ) -> Result<Vec<Frame>, RouterError> {
3773        match self.supervisor.subscribe_spawns(
3774            ctx.connection_id,
3775            frame.header.corr,
3776            response_version(&frame),
3777            since,
3778            ctx.egress.clone(),
3779        ) {
3780            Ok(()) => Ok(Vec::new()),
3781            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3782                Ok(vec![control_error_body_frame(
3783                    &frame,
3784                    ErrorBody {
3785                        code: "spawn_cursor_incarnation_mismatch".to_string(),
3786                        message: "spawn cursor belongs to a different daemon incarnation"
3787                            .to_string(),
3788                        detail: Some(serde_json::json!({
3789                            "current_daemon_incarnation": current
3790                        })),
3791                    },
3792                )?])
3793            }
3794            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3795                &frame,
3796                ErrorBody {
3797                    code: "spawn_cursor_too_old".to_string(),
3798                    message: "spawn cursor predates the retained event ring".to_string(),
3799                    detail: Some(serde_json::json!({
3800                        "oldest_retained_cursor": oldest
3801                    })),
3802                },
3803            )?]),
3804            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3805                0,
3806                frame.header.corr,
3807                format!("failed to open supervisor spawn subscription: {error}"),
3808            )),
3809        }
3810    }
3811
3812    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3813        let generation = self
3814            .registry
3815            .generation()
3816            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3817        let mut modules = Vec::new();
3818        for module in self.supervisor.list() {
3819            let status = module.status_for_control("list").map_err(|err| {
3820                RouterError::backend(
3821                    0,
3822                    frame.header.corr,
3823                    format!("failed to read supervisor status: {err}"),
3824                )
3825            })?;
3826            let (configured, _) = module.configuration().map_err(|err| {
3827                RouterError::backend(
3828                    0,
3829                    frame.header.corr,
3830                    format!("failed to read module configuration: {err}"),
3831                )
3832            })?;
3833            // Status and configuration snapshots release their locks before the image probe awaits.
3834            let image = module.running_image_agreement().await;
3835            // Read per request so the figure is current when the operator asks;
3836            // the daemon samples nothing in between.
3837            let resources = Some(module.child_resource_usage());
3838            let pending_reload = Some(reload_verdict(
3839                &configured.program,
3840                status.spawned_from.as_deref(),
3841                image,
3842            ));
3843            modules.push(SupervisorEntry {
3844                // Keep the retired policy field on the wire for one release so
3845                // existing status consumers still receive the platform policy.
3846                launch_nonce_env: Some(!cfg!(unix)),
3847                module_id: status.module_id,
3848                state: status.state.to_string(),
3849                enabled: status.enabled,
3850                live: status.live,
3851                protocol: status.protocol,
3852                health: status.health.status,
3853                pending_reload,
3854                last_probe_ms: status.health.last_probe_ms,
3855                last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3856                last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3857                last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3858                last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3859                restart_count: Some(status.restart_count),
3860                max_restarts: Some(status.max_restarts),
3861                lifetime_restarts: Some(status.lifetime_restarts),
3862                spawn_generation: Some(status.spawn_generation),
3863                restart_window_secs: Some(status.restart_window.as_secs()),
3864                drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3865                restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3866                restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3867                resources,
3868            });
3869        }
3870        let response = ClientControlResponse::SupervisorList {
3871            generation,
3872            modules,
3873        };
3874        Ok(vec![control_response_body_frame(
3875            &frame,
3876            &response,
3877            "ClientControlResponse::SupervisorList",
3878        )?])
3879    }
3880
3881    fn handle_supervisor_stderr_tail(
3882        &self,
3883        frame: Frame,
3884        module_id: String,
3885        max_lines: Option<u32>,
3886        max_bytes: Option<u32>,
3887    ) -> Result<Vec<Frame>, RouterError> {
3888        let Some(module) = self.supervisor.get(&module_id) else {
3889            return Ok(vec![control_error_frame(
3890                &frame,
3891                "unknown_module",
3892                format!("module_id '{module_id}' is not supervised"),
3893            )?]);
3894        };
3895
3896        let snapshot = module.stderr_tail(
3897            max_lines.map(|value| value as usize),
3898            max_bytes.map(|value| value as usize),
3899        );
3900
3901        let response = ClientControlResponse::SupervisorStderrTail {
3902            module_id,
3903            tail: StderrTail {
3904                capture: match snapshot.capture {
3905                    CaptureState::Captured => StderrCaptureState::Captured,
3906                    CaptureState::Incomplete { reason } => {
3907                        StderrCaptureState::Incomplete { reason }
3908                    }
3909                    CaptureState::NotCaptured { reason } => {
3910                        StderrCaptureState::NotCaptured { reason }
3911                    }
3912                },
3913                entries: snapshot
3914                    .entries
3915                    .into_iter()
3916                    .map(|entry| match entry {
3917                        TailEntry::Line {
3918                            text,
3919                            truncated,
3920                            at_ms,
3921                        } => StderrTailEntry::Line {
3922                            text,
3923                            truncated,
3924                            at_ms,
3925                        },
3926                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3927                    })
3928                    .collect(),
3929                dropped_lines: snapshot.dropped_lines,
3930            },
3931        };
3932        Ok(vec![control_response_body_frame(
3933            &frame,
3934            &response,
3935            "ClientControlResponse::SupervisorStderrTail",
3936        )?])
3937    }
3938
3939    async fn handle_supervisor_terminals(
3940        &self,
3941        frame: Frame,
3942        module_id: String,
3943    ) -> Result<Vec<Frame>, RouterError> {
3944        let Some(module) = self.supervisor.get(&module_id) else {
3945            return Ok(vec![control_error_frame(
3946                &frame,
3947                "unknown_module",
3948                format!("module_id '{module_id}' is not supervised"),
3949            )?]);
3950        };
3951
3952        // The journal read runs on a blocking thread: it can be megabytes of
3953        // file I/O and must not occupy a runtime worker.
3954        let terminals = module
3955            .read_durable_terminal_history()
3956            .await
3957            .map_err(|error| {
3958                RouterError::backend(
3959                    0,
3960                    frame.header.corr,
3961                    format!("failed to read terminal history: {error}"),
3962                )
3963            })?;
3964        let response = ClientControlResponse::SupervisorTerminals {
3965            module_id,
3966            terminals,
3967        };
3968        Ok(vec![control_response_body_frame(
3969            &frame,
3970            &response,
3971            "ClientControlResponse::SupervisorTerminals",
3972        )?])
3973    }
3974
3975    fn handle_supervisor_routes(
3976        &self,
3977        frame: Frame,
3978        module_id: Option<String>,
3979    ) -> Result<Vec<Frame>, RouterError> {
3980        let modules = self
3981            .forwarding
3982            .route_census(module_id.as_deref())
3983            .map_err(RouterError::Forwarding)?
3984            .into_iter()
3985            .map(|(module_id, routes)| SupervisorRouteModule {
3986                module_id,
3987                routes: routes
3988                    .into_iter()
3989                    .map(|route| SupervisorRoute {
3990                        consumer: match route.principal {
3991                            Principal::Reserved { module_id } => {
3992                                SupervisorRouteConsumer::Reserved { module_id }
3993                            }
3994                            Principal::Direct | Principal::Unverified => {
3995                                SupervisorRouteConsumer::Direct {
3996                                    connection_id: route.goodbye_target.connection_id.get(),
3997                                }
3998                            }
3999                        },
4000                        age_ms: Instant::now()
4001                            .saturating_duration_since(route.bound_at)
4002                            .as_millis()
4003                            .try_into()
4004                            .unwrap_or(u64::MAX),
4005                        draining: route.draining,
4006                        drain_reason: route.drain_reason,
4007                    })
4008                    .collect(),
4009            })
4010            .collect();
4011        let response = ClientControlResponse::SupervisorRoutes { modules };
4012        Ok(vec![control_response_body_frame(
4013            &frame,
4014            &response,
4015            "ClientControlResponse::SupervisorRoutes",
4016        )?])
4017    }
4018
4019    async fn handle_supervisor_provenance(
4020        &self,
4021        frame: Frame,
4022        module_id: Option<String>,
4023    ) -> Result<Vec<Frame>, RouterError> {
4024        let mut selected = if let Some(module_id) = module_id {
4025            let Some(module) = self.supervisor.get(&module_id) else {
4026                return Ok(vec![control_error_frame(
4027                    &frame,
4028                    "unknown_module",
4029                    format!("module_id '{module_id}' is not supervised"),
4030                )?]);
4031            };
4032            vec![module]
4033        } else {
4034            self.supervisor.list()
4035        };
4036
4037        let mut modules = Vec::with_capacity(selected.len());
4038        for module in selected.drain(..) {
4039            let status = module.status().map_err(|err| {
4040                RouterError::backend(
4041                    0,
4042                    frame.header.corr,
4043                    format!("failed to read supervisor status: {err}"),
4044                )
4045            })?;
4046            let module_declared = self
4047                .registry
4048                .get_module(&status.module_id)
4049                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4050                .and_then(|registration| registration.manifest.provenance)
4051                .map(|build| ModuleDeclaredProvenance::Reported { build })
4052                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4053            #[cfg(test)]
4054            let running_image = match &self.provenance_probe_override {
4055                Some(result) => result.clone(),
4056                None => module.running_image_agreement().await,
4057            };
4058            #[cfg(not(test))]
4059            let running_image = module.running_image_agreement().await;
4060            modules.push(SupervisorModuleProvenance {
4061                module_id: status.module_id,
4062                module_declared,
4063                daemon_observed: SupervisorObservedProcess {
4064                    pid: status.pid,
4065                    spawned_at_ms: status.spawned_at_ms,
4066                    spawned_from: status.spawned_from,
4067                    running_image,
4068                },
4069            });
4070        }
4071        let daemon = SupervisorDaemonProvenance {
4072            daemon_build: self.daemon_provenance.build.clone(),
4073            daemon_observed: DaemonObservedProcess {
4074                pid: self.daemon_provenance.pid,
4075                started_at_ms: self
4076                    .daemon_provenance
4077                    .start_clock
4078                    .map(|clock| clock.started_at_ms())
4079                    .or(self.daemon_provenance.started_at_ms),
4080                running_image: self
4081                    .daemon_provenance
4082                    .probe
4083                    .observe(
4084                        self.daemon_provenance.pid,
4085                        self.daemon_provenance.executable_path.as_deref(),
4086                        self.daemon_provenance.executable_identity,
4087                        self.daemon_provenance.process_start_time,
4088                    )
4089                    .await,
4090            },
4091        };
4092        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4093        Ok(vec![control_response_body_frame(
4094            &frame,
4095            &response,
4096            "ClientControlResponse::SupervisorProvenance",
4097        )?])
4098    }
4099
4100    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4101        self.refresh_capability_requirements();
4102        let generation = self
4103            .registry
4104            .generation()
4105            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4106        let modules = self
4107            .supervisor
4108            .list()
4109            .into_iter()
4110            .map(|module| {
4111                let status = module.status_for_control("health").map_err(|err| {
4112                    RouterError::backend(
4113                        0,
4114                        frame.header.corr,
4115                        format!("failed to read supervisor health: {err}"),
4116                    )
4117                })?;
4118                let module_id = status.module_id;
4119                let capability_detail = self
4120                    .capability_evaluator
4121                    .required_problem_detail(&module_id);
4122                Ok(SupervisorHealthEntry {
4123                    module_id,
4124                    status: status.health.status,
4125                    detail: append_capability_problem_detail(
4126                        status.health.detail,
4127                        capability_detail,
4128                    ),
4129                    metrics: status.health.metrics,
4130                    consecutive_failures: status.health.consecutive_failures,
4131                    late_answer_count: status.health.late_answer_count,
4132                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4133                    last_action: status.health.last_action,
4134                    last_action_ms: status.health.last_action_ms,
4135                    last_probe_ms: status.health.last_probe_ms,
4136                })
4137            })
4138            .collect::<Result<Vec<_>, RouterError>>()?;
4139        let response = ClientControlResponse::SupervisorHealth {
4140            generation,
4141            modules,
4142        };
4143        Ok(vec![control_response_body_frame(
4144            &frame,
4145            &response,
4146            "ClientControlResponse::SupervisorHealth",
4147        )?])
4148    }
4149
4150    async fn handle_supervisor_restart(
4151        &self,
4152        frame: Frame,
4153        module_id: String,
4154        drain_timeout_ms: Option<u64>,
4155    ) -> Result<Vec<Frame>, RouterError> {
4156        let operation_lock = self.supervisor.operation_lock();
4157        let _operation_guard = operation_lock.lock().await;
4158        let Some(module) = self.supervisor.get(&module_id) else {
4159            return Ok(vec![control_error_frame(
4160                &frame,
4161                "unknown_module",
4162                format!("module_id '{module_id}' is not supervised"),
4163            )?]);
4164        };
4165
4166        self.route_outages.mark_operator_action(&module_id);
4167        if let Err(err) = module.restart(drain_timeout_ms).await {
4168            self.route_outages
4169                .operator_action_ended_unrefused(&module_id);
4170            let (code, message) = match err {
4171                crate::supervise::SuperviseError::Disabled { .. } => {
4172                    ("module_disabled", err.to_string())
4173                }
4174                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4175                    ("swap_in_progress", err.to_string())
4176                }
4177                _ => (
4178                    "target_unavailable",
4179                    format!("failed to restart module_id '{module_id}': {err}"),
4180                ),
4181            };
4182            return Ok(vec![control_error_frame(&frame, code, message)?]);
4183        }
4184
4185        let response = ClientControlResponse::SupervisorAck {
4186            module_id,
4187            applied: true,
4188        };
4189        Ok(vec![control_response_body_frame(
4190            &frame,
4191            &response,
4192            "ClientControlResponse::SupervisorAck",
4193        )?])
4194    }
4195
4196    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
4197    /// when the old process has finished draining: a caller whose own lane
4198    /// rides the old process must get its reply before that drain waits on it.
4199    async fn handle_supervisor_swap(
4200        &self,
4201        frame: Frame,
4202        module_id: String,
4203        ready_timeout_ms: Option<u64>,
4204    ) -> Result<Vec<Frame>, RouterError> {
4205        // The daemon-wide operation lock is held only to resolve the handle,
4206        // not across the swap. The swap can take its whole readiness budget,
4207        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
4208        // holding it here would park an operator's stop behind the swap it is
4209        // meant to abort. A rescan or stop that reaches the module during the
4210        // swap is served by the swap itself (see `supervise_swap`).
4211        let module = {
4212            let operation_lock = self.supervisor.operation_lock();
4213            let _operation_guard = operation_lock.lock().await;
4214            self.supervisor.get(&module_id)
4215        };
4216        let Some(module) = module else {
4217            return Ok(vec![control_error_frame(
4218                &frame,
4219                "unknown_module",
4220                format!("module_id '{module_id}' is not supervised"),
4221            )?]);
4222        };
4223
4224        self.route_outages.mark_operator_action(&module_id);
4225        if let Err(err) = module
4226            .swap(ready_timeout_ms.map(Duration::from_millis))
4227            .await
4228        {
4229            self.route_outages
4230                .operator_action_ended_unrefused(&module_id);
4231            use crate::supervise::SuperviseError;
4232            let message = err.to_string();
4233            let error = match err {
4234                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4235                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4236                    code: "swap_refused".to_string(),
4237                    message,
4238                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4239                },
4240                SuperviseError::SwapFailed {
4241                    arm,
4242                    candidate_exit,
4243                    ..
4244                } => ErrorBody {
4245                    code: "swap_failed".to_string(),
4246                    message,
4247                    detail: Some(serde_json::json!({
4248                        "arm": arm.as_str(),
4249                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4250                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4251                    })),
4252                },
4253                _ => ErrorBody::new(
4254                    "target_unavailable",
4255                    format!("failed to swap module_id '{module_id}': {message}"),
4256                ),
4257            };
4258            return Ok(vec![control_error_body_frame(&frame, error)?]);
4259        }
4260        // A completed swap kept the incumbent serving until cutover, so it
4261        // usually opened no outage; a mark left behind would make the next,
4262        // unrelated outage read as requested.
4263        self.route_outages
4264            .operator_action_ended_unrefused(&module_id);
4265
4266        let response = ClientControlResponse::SupervisorAck {
4267            module_id,
4268            applied: true,
4269        };
4270        Ok(vec![control_response_body_frame(
4271            &frame,
4272            &response,
4273            "ClientControlResponse::SupervisorAck",
4274        )?])
4275    }
4276
4277    async fn handle_supervisor_reload(
4278        &self,
4279        frame: Frame,
4280        module_id: String,
4281    ) -> Result<Vec<Frame>, RouterError> {
4282        let operation_lock = self.supervisor.operation_lock();
4283        let _operation_guard = operation_lock.lock().await;
4284        let Some(module) = self.supervisor.get(&module_id) else {
4285            return Ok(vec![control_error_frame(
4286                &frame,
4287                "unknown_module",
4288                format!("module_id '{module_id}' is not supervised"),
4289            )?]);
4290        };
4291
4292        self.route_outages.mark_operator_action(&module_id);
4293        if let Err(err) = module.reload().await {
4294            self.route_outages
4295                .operator_action_ended_unrefused(&module_id);
4296            let (code, message) = match err {
4297                crate::supervise::SuperviseError::Disabled { .. } => {
4298                    ("module_disabled", err.to_string())
4299                }
4300                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4301                    ("swap_in_progress", err.to_string())
4302                }
4303                _ => (
4304                    "reload_failed",
4305                    format!("failed to reload module_id '{module_id}': {err}"),
4306                ),
4307            };
4308            return Ok(vec![control_error_frame(&frame, code, message)?]);
4309        }
4310
4311        let response = ClientControlResponse::SupervisorAck {
4312            module_id,
4313            applied: true,
4314        };
4315        Ok(vec![control_response_body_frame(
4316            &frame,
4317            &response,
4318            "ClientControlResponse::SupervisorAck",
4319        )?])
4320    }
4321
4322    async fn handle_supervisor_rescan(
4323        &self,
4324        frame: Frame,
4325        preview: bool,
4326    ) -> Result<Vec<Frame>, RouterError> {
4327        let Some(context) = self.rescan.clone() else {
4328            return Ok(vec![control_error_frame(
4329                &frame,
4330                "rescan_unavailable",
4331                "the daemon was not started with a reloadable config path".to_string(),
4332            )?]);
4333        };
4334
4335        let operation_lock = self.supervisor.operation_lock();
4336        let _operation_guard = operation_lock.lock().await;
4337        let loaded = match crate::daemon_config::load(&context.config_path) {
4338            Ok(config) => config,
4339            Err(err) => {
4340                return Ok(vec![control_error_frame(
4341                    &frame,
4342                    "invalid_daemon_config",
4343                    format!("supervisor rescan rejected daemon config: {err}"),
4344                )?])
4345            }
4346        };
4347        // `load` reports a missing file as Ok(None), which is correct at boot
4348        // (no config, nothing to supervise) and catastrophic here: rescan treats
4349        // "not in the config" as "remove it", so an absent file would read as an
4350        // empty module list and retire the entire running fleet. An editor
4351        // writing via write-new-then-rename, or a half-finished edit, is enough
4352        // to open that window. Refuse instead: a config that cannot be read
4353        // carries no instruction to remove anything.
4354        let Some(config) = loaded else {
4355            return Ok(vec![control_error_frame(
4356                &frame,
4357                "invalid_daemon_config",
4358                format!(
4359                    "daemon config not found at {}; refusing to rescan (an absent config would \
4360                     retire every supervised module)",
4361                    context.config_path.display()
4362                ),
4363            )?]);
4364        };
4365        let (
4366            configured_port,
4367            storage_config,
4368            admission_facts_carrier_module_id,
4369            admission_facts_targets,
4370            scope_authority_owners,
4371            modules,
4372            reserved_capabilities,
4373        ) = (
4374            config.port,
4375            config.storage,
4376            config.admission_facts_carrier_module_id,
4377            config.admission_facts_targets,
4378            config.scope_authority_owners,
4379            config.modules,
4380            config.reserved_capabilities,
4381        );
4382
4383        // Collect the sections rescan cannot apply, so the REPLY carries them.
4384        //
4385        // The warning below has always been correct and has always gone only to
4386        // the journal -- addressed to whoever reads logs, while the person who
4387        // just edited the config is looking at the CLI. Naming each section
4388        // individually rather than setting a flag: "something outside modules
4389        // changed" sends the operator back to diffing their own file, which is
4390        // the work this is meant to save.
4391        let mut restart_required = Vec::new();
4392        for section in RestartRequiredSection::ALL {
4393            let changed = match section {
4394                RestartRequiredSection::Port => configured_port != context.configured_port,
4395                RestartRequiredSection::Storage => storage_config != context.storage_config,
4396                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4397                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4398                }
4399                RestartRequiredSection::AdmissionFactsTargets => {
4400                    admission_facts_targets != context.admission_facts_targets
4401                }
4402                RestartRequiredSection::ScopeAuthorityOwners => {
4403                    scope_authority_owners != context.scope_authority_owners
4404                }
4405            };
4406            if changed {
4407                restart_required.push(section.label().to_string());
4408            }
4409        }
4410        if !restart_required.is_empty() {
4411            warn!(
4412                config_path = %context.config_path.display(),
4413                sections = %restart_required.join(", "),
4414                "daemon config changed outside the modules section; restart the daemon to apply those changes"
4415            );
4416        }
4417
4418        for configured in &modules {
4419            if let Err(err) = validate_spec(&configured.module_spec()) {
4420                return Ok(vec![control_error_frame(
4421                    &frame,
4422                    "invalid_daemon_config",
4423                    format!("supervisor rescan rejected daemon config: {err}"),
4424                )?]);
4425            }
4426        }
4427
4428        let configured_capabilities = modules
4429            .iter()
4430            .map(|module| (module.module_id.clone(), module.enabled))
4431            .collect::<Vec<_>>();
4432        let preview_capability_warnings = if preview {
4433            let (_, registrations) = self.runtime_capability_snapshot()?;
4434            let current_modules = self
4435                .supervisor
4436                .list()
4437                .into_iter()
4438                .map(|module| module.module_id().to_string())
4439                .collect::<BTreeSet<_>>();
4440            let resulting_modules = configured_capabilities.clone();
4441            let removed = current_modules
4442                .into_iter()
4443                .filter(|module_id| {
4444                    !resulting_modules
4445                        .iter()
4446                        .any(|(configured_id, _)| configured_id == module_id)
4447                })
4448                .collect::<Vec<_>>();
4449            self.capability_evaluator.preview_removal_warnings(
4450                resulting_modules,
4451                &removed,
4452                &registrations,
4453            )
4454        } else {
4455            Vec::new()
4456        };
4457        let result = match self
4458            .reconcile_supervised_modules(&context.supervisor, modules, preview)
4459            .await
4460        {
4461            Ok(result) => result,
4462            Err(message) => {
4463                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4464            }
4465        };
4466        if !preview {
4467            self.capability_evaluator
4468                .configure(configured_capabilities, reserved_capabilities);
4469            self.capability_evaluator.wake_deadline_loop();
4470            self.refresh_capability_requirements();
4471        }
4472        let mut result = result;
4473        result.restart_required = restart_required;
4474        result.capability_warnings = preview_capability_warnings;
4475        let response = ClientControlResponse::SupervisorRescan { result };
4476        Ok(vec![control_response_body_frame(
4477            &frame,
4478            &response,
4479            "ClientControlResponse::SupervisorRescan",
4480        )?])
4481    }
4482
4483    async fn handle_supervisor_release_reserved(
4484        &self,
4485        frame: Frame,
4486        module_id: String,
4487    ) -> Result<Vec<Frame>, RouterError> {
4488        let Some(context) = self.rescan.clone() else {
4489            return Ok(vec![control_error_frame(
4490                &frame,
4491                "release_unavailable",
4492                "reserved-id release requires a daemon started with a reloadable config path",
4493            )?]);
4494        };
4495        let operation_lock = self.supervisor.operation_lock();
4496        let _operation_guard = operation_lock.lock().await;
4497        let loaded = match crate::daemon_config::load(&context.config_path) {
4498            Ok(Some(config)) => config,
4499            Ok(None) => {
4500                return Ok(vec![control_error_frame(
4501                    &frame,
4502                    "invalid_daemon_config",
4503                    format!(
4504                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4505                        context.config_path.display()
4506                    ),
4507                )?])
4508            }
4509            Err(err) => {
4510                return Ok(vec![control_error_frame(
4511                    &frame,
4512                    "invalid_daemon_config",
4513                    format!("unable to verify reserved-id release against daemon config: {err}"),
4514                )?])
4515            }
4516        };
4517        if loaded
4518            .modules
4519            .iter()
4520            .any(|configured| configured.module_id == module_id)
4521        {
4522            return Ok(vec![control_error_frame(
4523                &frame,
4524                "reserved_module_configured",
4525                format!(
4526                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4527                ),
4528            )?]);
4529        }
4530        if !self.supervisor.release_retained_reserved_gate(&module_id) {
4531            return Ok(vec![control_error_frame(
4532                &frame,
4533                "reserved_gate_not_retained",
4534                format!(
4535                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4536                ),
4537            )?]);
4538        }
4539
4540        let response = ClientControlResponse::SupervisorAck {
4541            module_id,
4542            applied: true,
4543        };
4544        Ok(vec![control_response_body_frame(
4545            &frame,
4546            &response,
4547            "ClientControlResponse::SupervisorAck",
4548        )?])
4549    }
4550
4551    /// Reconcile the running module set against the configured one.
4552    ///
4553    /// With `preview` set, the diff is computed and returned WITHOUT applying any
4554    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
4555    /// deliberately shares this function with the executing path rather than
4556    /// computing the same diff somewhere else -- two implementations of one
4557    /// decision agree until they do not, and the whole value of a preview is that
4558    /// it describes the operation that will actually run.
4559    async fn reconcile_supervised_modules(
4560        &self,
4561        supervisor: &Supervisor,
4562        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4563        preview: bool,
4564    ) -> Result<SupervisorRescanResult, String> {
4565        let mut current = BTreeMap::new();
4566        for module in self.supervisor.list() {
4567            let (spec, health) = module.configuration().map_err(|err| {
4568                format!(
4569                    "failed to read configuration for module_id '{}': {err}",
4570                    module.module_id()
4571                )
4572            })?;
4573            let enabled = module
4574                .status()
4575                .map_err(|err| {
4576                    format!(
4577                        "failed to read status for module_id '{}': {err}",
4578                        module.module_id()
4579                    )
4580                })?
4581                .enabled;
4582            current.insert(
4583                module.module_id().to_string(),
4584                (module, spec, health, enabled),
4585            );
4586        }
4587        let configured = configured_modules
4588            .into_iter()
4589            .map(|module| (module.module_id.clone(), module))
4590            .collect::<BTreeMap<_, _>>();
4591
4592        let added = configured
4593            .keys()
4594            .filter(|module_id| !current.contains_key(*module_id))
4595            .cloned()
4596            .collect::<Vec<_>>();
4597        let removed = current
4598            .keys()
4599            .filter(|module_id| !configured.contains_key(*module_id))
4600            .cloned()
4601            .collect::<Vec<_>>();
4602        let mut changed_pending_reload = Vec::new();
4603        let mut configuration_changes = BTreeSet::new();
4604        let mut enabled_changes = BTreeSet::new();
4605        let mut unchanged = 0_u32;
4606
4607        for (module_id, configured_module) in &configured {
4608            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4609            else {
4610                continue;
4611            };
4612            // Compare the whole launch spec so a future launch field cannot
4613            // accidentally become a live-only policy change. Health is stored
4614            // separately and applies live without replacing the process.
4615            let launch_changed = *current_spec != configured_module.module_spec();
4616            let configuration_changed =
4617                launch_changed || *current_health != configured_module.health;
4618            let enabled_changed = *current_enabled != configured_module.enabled;
4619            if configuration_changed {
4620                configuration_changes.insert(module_id.clone());
4621            }
4622            if launch_changed {
4623                changed_pending_reload.push(module_id.clone());
4624            }
4625            if enabled_changed {
4626                enabled_changes.insert(module_id.clone());
4627            }
4628            if !configuration_changed && !enabled_changed {
4629                unchanged = unchanged.saturating_add(1);
4630            }
4631        }
4632
4633        // Everything above this point is pure computation over two snapshots.
4634        // Everything below MUTATES. The preview returns here so the boundary is a
4635        // single early return rather than a condition repeated at each mutation
4636        // site, where one missed guard would apply part of a change the caller was
4637        // told would not happen.
4638        if preview {
4639            return Ok(SupervisorRescanResult {
4640                added,
4641                removed,
4642                changed_pending_reload,
4643                enabled_changes: enabled_changes.iter().cloned().collect(),
4644                unchanged,
4645                preview: true,
4646                // Filled by the caller on both paths, so the preview reports
4647                // restart-required sections identically to an executed rescan --
4648                // the preview is where an operator is most likely to be looking.
4649                restart_required: Vec::new(),
4650                capability_warnings: Vec::new(),
4651            });
4652        }
4653
4654        for module_id in &removed {
4655            let module = &current
4656                .get(module_id)
4657                .expect("removed module came from current supervisor state")
4658                .0;
4659            module.retire().await.map_err(|err| {
4660                format!("failed to retire module_id '{module_id}' during rescan: {err}")
4661            })?;
4662            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
4663            //
4664            // `handle_route_open` resolves an absent module in three steps:
4665            // registry, then supervisor status, then tombstone. Retiring first
4666            // opens a window where ALL THREE ARE ABSENT -- the registry entry
4667            // went with the teardown above, the supervisor entry went with
4668            // `retire`, and the tombstone does not exist yet -- so a route.open
4669            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
4670            // it") for a module that was deliberately removed and whose caller
4671            // should get `module_removed` (TERMINAL, carrying a removal age).
4672            //
4673            // Writing the tombstone first closes it: during the window the
4674            // supervisor entry still answers, so the caller gets
4675            // `target_unavailable` -- retryable, and TRUE, because the module
4676            // is mid-teardown. After both statements it is `module_removed`.
4677            // No instant remains where a removed module reads as one that
4678            // never existed.
4679            //
4680            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
4681            // the absence of a test beside a fix invites deletion: these are
4682            // two sync statements with no await between them, so reaching the
4683            // window needs a second worker thread to land exactly between them
4684            // and there is no hook to force it. MEASURED: the 25 daemon_config
4685            // tests pass identically with the old order and the new one, so
4686            // the existing suite cannot see this and a green run is not
4687            // evidence either way. What the suite does hold is the
4688            // post-condition -- a removed module answers `module_removed` --
4689            // which this preserves.
4690            //
4691            // Found by an Athena panel reading the shipped tree against a
4692            // design note (2026-09-19), as the one concrete instance of that
4693            // note's class that survived contact with source. Direction is
4694            // benign: retryable where terminal was intended, never the reverse.
4695            self.supervisor.record_rescan_removal(module_id);
4696            self.supervisor.retire(module_id);
4697            self.route_outages.forget(module_id);
4698        }
4699
4700        for module_id in configured.keys() {
4701            let Some((module, _, _, _)) = current.get(module_id) else {
4702                continue;
4703            };
4704            let configured_module = configured
4705                .get(module_id)
4706                .expect("configured module id came from configured map");
4707            if configuration_changes.contains(module_id) {
4708                module
4709                    .update_configuration(
4710                        configured_module.module_spec(),
4711                        configured_module.health.clone(),
4712                        configured_module.drain_timeout_ms,
4713                    )
4714                    .await
4715                    .map_err(|err| {
4716                        format!(
4717                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
4718                        )
4719                    })?;
4720            }
4721            if enabled_changes.contains(module_id) {
4722                // A rescan that starts or stops a module applies an operator's
4723                // edit to the config, so the resulting outage was asked for.
4724                self.route_outages.mark_operator_action(module_id);
4725                module
4726                    .set_enabled(configured_module.enabled)
4727                    .await
4728                    .map_err(|err| {
4729                        self.route_outages.operator_action_ended_unrefused(module_id);
4730                        format!(
4731                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4732                            configured_module.enabled
4733                        )
4734                    })?;
4735            }
4736        }
4737
4738        for module_id in &added {
4739            let configured_module = configured
4740                .get(module_id)
4741                .expect("added module id came from configured map");
4742            supervisor
4743                .supervise_configured_with_health(
4744                    configured_module.module_spec(),
4745                    configured_module.enabled,
4746                    configured_module.health.clone(),
4747                    configured_module.drain_timeout_ms,
4748                    configured_module.restart,
4749                )
4750                .map_err(|err| {
4751                    format!("failed to add module_id '{module_id}' during rescan: {err}")
4752                })?;
4753        }
4754
4755        Ok(SupervisorRescanResult {
4756            added,
4757            removed,
4758            changed_pending_reload,
4759            enabled_changes: enabled_changes.iter().cloned().collect(),
4760            unchanged,
4761            preview: false,
4762            // Filled by the caller, which is the only layer that can see the
4763            // previous config to diff against.
4764            restart_required: Vec::new(),
4765            capability_warnings: Vec::new(),
4766        })
4767    }
4768
4769    async fn handle_supervisor_set_enabled(
4770        &self,
4771        frame: Frame,
4772        module_id: String,
4773        enabled: bool,
4774    ) -> Result<Vec<Frame>, RouterError> {
4775        let operation_lock = self.supervisor.operation_lock();
4776        let _operation_guard = operation_lock.lock().await;
4777        let Some(module) = self.supervisor.get(&module_id) else {
4778            return Ok(vec![control_error_frame(
4779                &frame,
4780                "unknown_module",
4781                format!("module_id '{module_id}' is not supervised"),
4782            )?]);
4783        };
4784
4785        // Enabling counts as well as disabling: a module an operator starts
4786        // is refused until it registers, and that wait was asked for.
4787        self.route_outages.mark_operator_action(&module_id);
4788        let applied = match module.set_enabled(enabled).await {
4789            Ok(applied) => applied,
4790            Err(err) => {
4791                self.route_outages
4792                    .operator_action_ended_unrefused(&module_id);
4793                return Ok(vec![control_error_frame(
4794                    &frame,
4795                    "target_unavailable",
4796                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4797                )?]);
4798            }
4799        };
4800        if !applied {
4801            // Already in the requested state: nothing was made unavailable,
4802            // so the mark must not outlive this request.
4803            self.route_outages
4804                .operator_action_ended_unrefused(&module_id);
4805        }
4806
4807        self.capability_evaluator.wake_deadline_loop();
4808        self.refresh_capability_requirements();
4809        let response = ClientControlResponse::SupervisorAck { module_id, applied };
4810        Ok(vec![control_response_body_frame(
4811            &frame,
4812            &response,
4813            "ClientControlResponse::SupervisorAck",
4814        )?])
4815    }
4816
4817    async fn handle_supervisor_health_probe(
4818        &self,
4819        frame: Frame,
4820        module_id: String,
4821    ) -> Result<Vec<Frame>, RouterError> {
4822        self.refresh_capability_requirements();
4823        let Some(registration) = self
4824            .registry
4825            .get_module(&module_id)
4826            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4827        else {
4828            return Ok(vec![control_error_frame(
4829                &frame,
4830                "unknown_module",
4831                format!("module_id '{module_id}' is not registered"),
4832            )?]);
4833        };
4834
4835        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
4836        // named for it. Making `module_registration_grants_op` return false
4837        // unconditionally reddens five tests, and every one is named for something
4838        // else -- capability relay, probe/bind demultiplexing, supervision-only
4839        // probing. They exercise a successful advertisement check on the way to their
4840        // own subject.
4841        //
4842        // Real protection, fragile in a specific way: narrowing any of those tests to
4843        // focus on its stated subject would silently remove coverage nobody knows
4844        // they are carrying. Recorded here rather than as a sixth test, because the
4845        // useful fact is WHICH tests hold the guard up -- a new test would add
4846        // coverage without telling the next person what the existing ones quietly do.
4847        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4848        {
4849            return Ok(vec![control_error_frame(
4850                &frame,
4851                "health_not_advertised",
4852                format!("module_id '{module_id}' did not advertise health.check"),
4853            )?]);
4854        }
4855
4856        let deadline = Instant::now() + self.health_probe_timeout;
4857        let pending = match self.forwarding.begin_module_control_rpc_for(
4858            &module_id,
4859            MODULE_CONTROL_OP_HEALTH_CHECK,
4860            deadline,
4861        ) {
4862            Ok(pending) => pending,
4863            Err(err) => {
4864                return Ok(vec![control_error_frame(
4865                    &frame,
4866                    forwarding_error_code(&err),
4867                    err.to_string(),
4868                )?])
4869            }
4870        };
4871
4872        let PendingModuleControlRpc {
4873            endpoint,
4874            module_sink,
4875            negotiated_ver,
4876            corr: probe_corr,
4877            receiver,
4878        } = pending;
4879        let mut guard =
4880            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4881        let probe_body =
4882            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4883                RouterError::backend(
4884                    0,
4885                    frame.header.corr,
4886                    format!("failed to encode health.check request: {err}"),
4887                )
4888            })?;
4889        let probe_frame = Frame::build_with_version(
4890            negotiated_ver,
4891            FrameType::Request,
4892            control_flags(),
4893            0,
4894            0,
4895            probe_corr,
4896            probe_body,
4897        )
4898        .map_err(RouterError::FrameBuild)?;
4899
4900        if let Err(err) = module_sink.send(probe_frame).await {
4901            return Ok(vec![control_error_frame(
4902                &frame,
4903                "target_unavailable",
4904                err.to_string(),
4905            )?]);
4906        }
4907
4908        match timeout_at(deadline, receiver).await {
4909            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4910                guard.disarm();
4911                let Some(report) = response.health_report() else {
4912                    return Ok(vec![control_error_frame(
4913                        &frame,
4914                        "invalid_control_body",
4915                        "health.check RPC returned a non-health response",
4916                    )?]);
4917                };
4918                // Metrics go out whole here. The supervisor's cached snapshot
4919                // caps this blob (see truncate_health_metrics), and this path
4920                // exists precisely to answer without that cap -- so applying it
4921                // here would leave no way to see what the cached view drops.
4922                let HealthReport {
4923                    status,
4924                    detail,
4925                    metrics,
4926                } = report;
4927                let capability_detail = self
4928                    .capability_evaluator
4929                    .required_problem_detail(&module_id);
4930                let response = ClientControlResponse::SupervisorHealthProbe {
4931                    module_id,
4932                    status,
4933                    detail: append_capability_problem_detail(detail, capability_detail),
4934                    metrics,
4935                };
4936                Ok(vec![control_response_body_frame(
4937                    &frame,
4938                    &response,
4939                    "ClientControlResponse::SupervisorHealthProbe",
4940                )?])
4941            }
4942            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4943                guard.disarm();
4944                Ok(vec![control_error_body_frame(&frame, body)?])
4945            }
4946            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4947                guard.disarm();
4948                Ok(vec![control_error_frame(
4949                    &frame,
4950                    "target_unavailable",
4951                    message,
4952                )?])
4953            }
4954            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4955                guard.disarm();
4956                Ok(vec![control_error_frame(
4957                    &frame,
4958                    "invalid_control_body",
4959                    message,
4960                )?])
4961            }
4962            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4963                guard.disarm();
4964                Ok(vec![control_error_frame(
4965                    &frame,
4966                    "invalid_control_body",
4967                    format!("expected module-control op '{expected}', got '{actual}'"),
4968                )?])
4969            }
4970            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4971                guard.disarm();
4972                Ok(vec![control_error_frame(
4973                    &frame,
4974                    "module_timeout",
4975                    format!(
4976                        "module_id '{module_id}' answered health.check after {:?}",
4977                        self.health_probe_timeout
4978                    ),
4979                )?])
4980            }
4981            Ok(Err(_)) => Ok(vec![control_error_frame(
4982                &frame,
4983                "target_unavailable",
4984                "health.check waiter was canceled before the module responded",
4985            )?]),
4986            Err(_) => Ok(vec![control_error_frame(
4987                &frame,
4988                "module_timeout",
4989                format!(
4990                    "module_id '{module_id}' did not answer health.check within {:?}",
4991                    self.health_probe_timeout
4992                ),
4993            )?]),
4994        }
4995    }
4996
4997    fn supervisor_status(
4998        &self,
4999        module_id: &str,
5000        corr: u64,
5001    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5002        self.supervisor
5003            .get(module_id)
5004            .map(|module| {
5005                let warming = module.is_warming_for_control("status").map_err(|err| {
5006                    RouterError::backend(
5007                        0,
5008                        corr,
5009                        format!(
5010                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
5011                        ),
5012                    )
5013                })?;
5014                module.status_for_control("status").map_err(|err| {
5015                    RouterError::backend(
5016                        0,
5017                        corr,
5018                        format!(
5019                            "failed to read supervisor status for module_id '{module_id}': {err}"
5020                        ),
5021                    )
5022                }).map(|status| (status, warming))
5023            })
5024            .transpose()
5025    }
5026
5027    fn guard_module_control_op(
5028        &self,
5029        frame: &Frame,
5030        module_id: &str,
5031        op: &str,
5032    ) -> Result<Option<Frame>, RouterError> {
5033        if self.module_grants_op(module_id, op, frame.header.corr)? {
5034            return Ok(None);
5035        }
5036
5037        Ok(Some(control_error_frame(
5038            frame,
5039            "op_not_allowed",
5040            format!("module_id '{module_id}' did not grant control op '{op}'"),
5041        )?))
5042    }
5043
5044    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5045        let Some(registration) = self
5046            .registry
5047            .get_module(module_id)
5048            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5049        else {
5050            return Ok(false);
5051        };
5052        Ok(module_registration_grants_op(&registration.control_ops, op))
5053    }
5054
5055    fn handle_status_update(
5056        &self,
5057        endpoint: ModuleEndpointId,
5058        frame: Frame,
5059    ) -> Result<Vec<Frame>, RouterError> {
5060        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5061            Ok(update) => update,
5062            Err(err) => {
5063                // Forward-compat: a newer module may push a channel-0 op this subc
5064                // version doesn't know. The control contract says unknown push ops
5065                // are IGNORED, never answered with an error. Only a malformed body
5066                // for an op we DO know is a real error worth surfacing.
5067                if is_known_module_push_op(&frame.body) {
5068                    return Ok(vec![control_error_frame(
5069                        &frame,
5070                        "invalid_control_body",
5071                        format!("malformed module control push body: {err}"),
5072                    )?]);
5073                }
5074                return Ok(Vec::new());
5075            }
5076        };
5077
5078        match update {
5079            ModuleControlPush::RouteStatus {
5080                route_channel,
5081                route_epoch,
5082                status,
5083            } => {
5084                self.forwarding
5085                    .cache_status(endpoint, route_channel, route_epoch, status)
5086                    .map_err(RouterError::Forwarding)?;
5087            }
5088        }
5089        Ok(Vec::new())
5090    }
5091
5092    fn handle_route_poll(
5093        &self,
5094        ctx: &RouteCtx,
5095        frame: Frame,
5096        route_channel: u16,
5097        route_epoch: u32,
5098        kind: PollKind,
5099    ) -> Result<Vec<Frame>, RouterError> {
5100        let snapshot = self
5101            .forwarding
5102            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5103            .map_err(RouterError::Forwarding)?;
5104        let response = match (kind, snapshot) {
5105            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5106                ClientControlResponse::RoutePoll {
5107                    route_channel,
5108                    route_epoch,
5109                    status,
5110                    live: None,
5111                }
5112            }
5113            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5114                route_channel,
5115                route_epoch,
5116                status: None,
5117                live: None,
5118            },
5119            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5120                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
5121                // is what makes reporting `true` correct rather than a
5122                // confident guess. `process_live` returns None only when the
5123                // module id has no supervisor snapshot at all -- an
5124                // externally-started module the daemon did not spawn -- and
5125                // for those the supervisor has no opinion to offer, ever. It
5126                // is never None for a supervised module in an unknown state:
5127                // a supervised module always has a snapshot, and the answer
5128                // comes from `state == Running && process_alive`.
5129                //
5130                // The route is Bound, so the module completed a HELLO on a
5131                // live connection; "the process this route points at is
5132                // running" is therefore attested by the binding rather than
5133                // assumed. Reporting `false` for an unsupervised module would
5134                // be the actual lie -- it would tell a client its healthy
5135                // route is dead because the daemon does not manage the
5136                // process.
5137                //
5138                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
5139                // module whose liveness is genuinely unknown, e.g. a snapshot
5140                // that has not been populated yet -- THIS DEFAULT BECOMES
5141                // WRONG and must split: unsupervised stays true, unknown
5142                // becomes null so the client can tell the two apart. The
5143                // response field is already `Option<bool>`, so the wire can
5144                // carry that distinction today.
5145                let live = self
5146                    .process_liveness
5147                    .as_ref()
5148                    .and_then(|source| source.process_live(&module_id))
5149                    .unwrap_or(true);
5150                ClientControlResponse::RoutePoll {
5151                    route_channel,
5152                    route_epoch,
5153                    status: None,
5154                    live: Some(live),
5155                }
5156            }
5157            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5158                route_channel,
5159                route_epoch,
5160                status: None,
5161                live: Some(false),
5162            },
5163        };
5164
5165        Ok(vec![control_response_body_frame(
5166            &frame,
5167            &response,
5168            "ClientControlResponse::RoutePoll",
5169        )?])
5170    }
5171
5172    pub(crate) fn observe_module_control_completion(
5173        &self,
5174        completion: ModuleControlRpcCompletion,
5175    ) -> bool {
5176        match completion {
5177            ModuleControlRpcCompletion::Unknown => false,
5178            ModuleControlRpcCompletion::Settled => true,
5179            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5180                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5181                info!(
5182                    module_id = %module_id,
5183                    latency_ms,
5184                    "late health.check answer proves the module is alive"
5185                );
5186                match self
5187                    .supervisor
5188                    .record_late_health_answer(&module_id, latency_ms)
5189                {
5190                    Ok(true) => {}
5191                    Ok(false) => debug!(
5192                        module_id = %module_id,
5193                        latency_ms,
5194                        "late health.check answer has no active supervisor snapshot"
5195                    ),
5196                    Err(err) => warn!(
5197                        module_id = %module_id,
5198                        latency_ms,
5199                        error = %err,
5200                        "failed to record late health.check answer"
5201                    ),
5202                }
5203                true
5204            }
5205        }
5206    }
5207
5208    /// Decide whether a failure while settling a relayed `route.bind` belongs to
5209    /// the module connection whose frame is being handled, or to the client that
5210    /// relay was opened for.
5211    ///
5212    /// This runs on the MODULE connection's frame handler, where returning `Err`
5213    /// ends that connection -- and a module connection carries every client's
5214    /// routes to that module, so ending it costs the whole fleet its tools.
5215    /// `ConnectionClosing` carries the id of the connection that is closing, and
5216    /// when that id is a CLIENT's, the condition is entirely about that one
5217    /// client's route.open. A client-scoped condition has no authority over a
5218    /// shared module connection, so it is logged and the single relay is dropped:
5219    /// the client is going away, and `complete_pending_relay` already removed the
5220    /// relay before failing, so there is nothing left to settle. Anything that
5221    /// relay still reserved is released by that client's own connection teardown,
5222    /// which is already under way -- that is what "closing" means.
5223    ///
5224    /// Every other failure is a statement about THIS connection and stays fatal:
5225    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
5226    /// id in `ConnectionClosing` all mean this connection cannot keep serving
5227    /// frames correctly.
5228    fn refuse_to_end_module_connection_for_a_client(
5229        &self,
5230        module_connection_id: ConnectionId,
5231        corr: u64,
5232        err: ForwardingError,
5233    ) -> Result<(), RouterError> {
5234        if let ForwardingError::ConnectionClosing { connection_id } = err {
5235            if connection_id != module_connection_id {
5236                warn!(
5237                    module_connection_id = module_connection_id.get(),
5238                    client_connection_id = connection_id.get(),
5239                    corr,
5240                    "dropping a route.bind response for a closing client; the module connection keeps serving"
5241                );
5242                return Ok(());
5243            }
5244        }
5245        Err(RouterError::Forwarding(err))
5246    }
5247
5248    fn handle_module_relay_response(
5249        &self,
5250        connection_id: ConnectionId,
5251        frame: Frame,
5252    ) -> Result<Vec<Frame>, RouterError> {
5253        let mut secondary_error = None;
5254        let outcome = match frame.header.ty {
5255            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5256                Ok(probe) if probe.op == "route.bind" => {
5257                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5258                        Ok(ModuleControlResponse::RouteBindAck {}) => {
5259                            RouteBindRelayOutcome::Accepted
5260                        }
5261                        Ok(other) => {
5262                            let message =
5263                                format!("route.bind response carried unexpected body: {other:?}");
5264                            secondary_error = Some(control_error_frame(
5265                                &frame,
5266                                "invalid_control_body",
5267                                message.clone(),
5268                            )?);
5269                            RouteBindRelayOutcome::ModuleGone(message)
5270                        }
5271                        Err(err) => {
5272                            let message = format!("malformed route.bind response body: {err}");
5273                            secondary_error = Some(control_error_frame(
5274                                &frame,
5275                                "invalid_control_body",
5276                                message.clone(),
5277                            )?);
5278                            RouteBindRelayOutcome::ModuleGone(message)
5279                        }
5280                    }
5281                }
5282                Ok(probe) => {
5283                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5284                    {
5285                        Ok(response) => ModuleControlRpcOutcome::Response(response),
5286                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5287                            "malformed {} response body: {err}",
5288                            probe.op
5289                        )),
5290                    };
5291                    let completion = self
5292                        .forwarding
5293                        .complete_module_control_rpc(
5294                            connection_id,
5295                            frame.header.corr,
5296                            Some(&probe.op),
5297                            outcome,
5298                        )
5299                        .map_err(RouterError::Forwarding)?;
5300                    if !self.observe_module_control_completion(completion) {
5301                        debug!(
5302                            connection_id = connection_id.get(),
5303                            corr = frame.header.corr,
5304                            op = %probe.op,
5305                            "dropping late or unknown module-control RPC response"
5306                        );
5307                    }
5308                    return Ok(Vec::new());
5309                }
5310                Err(err) => {
5311                    if let Some(expected_op) = self
5312                        .forwarding
5313                        .pending_module_control_op(connection_id, frame.header.corr)
5314                        .map_err(RouterError::Forwarding)?
5315                    {
5316                        let completion = self
5317                            .forwarding
5318                            .complete_module_control_rpc(
5319                                connection_id,
5320                                frame.header.corr,
5321                                None,
5322                                ModuleControlRpcOutcome::MalformedResponse(format!(
5323                                    "malformed {expected_op} response body: {err}"
5324                                )),
5325                            )
5326                            .map_err(RouterError::Forwarding)?;
5327                        if !self.observe_module_control_completion(completion) {
5328                            debug!(
5329                                connection_id = connection_id.get(),
5330                                corr = frame.header.corr,
5331                                "dropping late malformed module-control RPC response"
5332                            );
5333                        }
5334                        return Ok(Vec::new());
5335                    }
5336                    let message = format!("malformed route.bind response body: {err}");
5337                    secondary_error = Some(control_error_frame(
5338                        &frame,
5339                        "invalid_control_body",
5340                        message.clone(),
5341                    )?);
5342                    RouteBindRelayOutcome::ModuleGone(message)
5343                }
5344            },
5345            FrameType::Error => {
5346                if self
5347                    .forwarding
5348                    .pending_module_control_op(connection_id, frame.header.corr)
5349                    .map_err(RouterError::Forwarding)?
5350                    .is_some()
5351                {
5352                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5353                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5354                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5355                            "malformed module-control ERROR body: {err}"
5356                        )),
5357                    };
5358                    let completion = self
5359                        .forwarding
5360                        .complete_module_control_rpc(
5361                            connection_id,
5362                            frame.header.corr,
5363                            None,
5364                            outcome,
5365                        )
5366                        .map_err(RouterError::Forwarding)?;
5367                    if !self.observe_module_control_completion(completion) {
5368                        debug!(
5369                            connection_id = connection_id.get(),
5370                            corr = frame.header.corr,
5371                            "dropping late or unknown module-control RPC error"
5372                        );
5373                    }
5374                    return Ok(Vec::new());
5375                }
5376                match serde_json::from_slice::<ErrorBody>(&frame.body) {
5377                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
5378                    Err(err) => {
5379                        let message = format!("malformed route.bind ERROR body: {err}");
5380                        secondary_error = Some(control_error_frame(
5381                            &frame,
5382                            "invalid_control_body",
5383                            message.clone(),
5384                        )?);
5385                        RouteBindRelayOutcome::ModuleGone(message)
5386                    }
5387                }
5388            }
5389            ty => {
5390                return Ok(vec![control_error_frame(
5391                    &frame,
5392                    "unsupported_control_frame",
5393                    format!("unsupported module channel-0 frame {ty:?}"),
5394                )?])
5395            }
5396        };
5397
5398        let settled =
5399            self.forwarding
5400                .complete_pending_relay(connection_id, frame.header.corr, outcome);
5401        let completion = match settled {
5402            Ok(completion) => completion,
5403            Err(err) => {
5404                self.refuse_to_end_module_connection_for_a_client(
5405                    connection_id,
5406                    frame.header.corr,
5407                    err,
5408                )?;
5409                return Ok(secondary_error.into_iter().collect());
5410            }
5411        };
5412        if let Some(target) = completion.abandoned.as_ref() {
5413            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5414        }
5415        if !completion.settled {
5416            debug!(
5417                connection_id = connection_id.get(),
5418                corr = frame.header.corr,
5419                frame_type = ?frame.header.ty,
5420                "dropping late or unknown route.bind relay response"
5421            );
5422        }
5423        Ok(secondary_error.into_iter().collect())
5424    }
5425
5426    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5427        debug!(connection_id = connection_id.get(), "handling GOODBYE");
5428        // GOODBYE ends the connection's logical session even when its socket
5429        // stays open. Use disconnect teardown so verdicts, client notices and
5430        // scope authority are released at the same lifecycle boundary.
5431        self.cleanup_connection(connection_id)
5432            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5433        Ok(Vec::new())
5434    }
5435}
5436
5437impl Default for ControlHandler {
5438    fn default() -> Self {
5439        Self::new(Arc::new(Registry::default()))
5440    }
5441}
5442
5443impl crate::supervise::SwapPromotionObserver for ControlHandler {
5444    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5445        self.apply_registration_capabilities(registration);
5446    }
5447}
5448
5449fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5450    CapabilityRequirementStatus {
5451        consumer: status.consumer,
5452        capability: status.capability,
5453        need: match status.need {
5454            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5455            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5456        },
5457        verdict: status.verdict.as_str().to_string(),
5458        episode_seq: status.episode_seq,
5459        config_satisfiable: status.config_satisfiable,
5460        runtime_available: status.runtime_available,
5461        detail: status.detail,
5462    }
5463}
5464
5465fn append_capability_problem_detail(
5466    detail: Option<String>,
5467    capability_detail: Option<String>,
5468) -> Option<String> {
5469    match (detail, capability_detail) {
5470        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5471        (Some(detail), None) => Some(detail),
5472        (None, Some(capability_detail)) => Some(capability_detail),
5473        (None, None) => None,
5474    }
5475}
5476
5477fn subc_ops() -> Vec<String> {
5478    SUBC_CONTROL_OPS
5479        .iter()
5480        .map(|op| (*op).to_string())
5481        .collect()
5482}
5483
5484fn module_subc_ops() -> Vec<String> {
5485    SUBC_CONTROL_OPS
5486        .iter()
5487        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5488        .map(|op| (*op).to_string())
5489        .collect()
5490}
5491
5492#[cfg(test)]
5493fn module_baseline_control_ops() -> Vec<String> {
5494    MODULE_BASELINE_CONTROL_OPS
5495        .iter()
5496        .map(|op| (*op).to_string())
5497        .collect()
5498}
5499
5500fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5501    let mut seen = HashSet::new();
5502    let mut effective = Vec::new();
5503    for op in MODULE_BASELINE_CONTROL_OPS {
5504        if seen.insert((*op).to_string()) {
5505            effective.push((*op).to_string());
5506        }
5507    }
5508    for op in declared.unwrap_or_default() {
5509        if seen.insert(op.clone()) {
5510            effective.push(op);
5511        }
5512    }
5513    effective
5514}
5515
5516fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5517    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5518}
5519
5520fn target_module_id(target: &RouteTarget) -> &str {
5521    match target {
5522        RouteTarget::ToolProvider { module_id }
5523        | RouteTarget::ManagementSurface { module_id }
5524        | RouteTarget::InternalService { module_id, .. } => module_id,
5525    }
5526}
5527
5528fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5529    roles.iter().any(|role| match (target, role) {
5530        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5531        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5532        (
5533            RouteTarget::InternalService { service_id, .. },
5534            ProviderRole::InternalService {
5535                service_id: provided,
5536                ..
5537            },
5538        ) => service_id == provided,
5539        _ => false,
5540    })
5541}
5542
5543fn is_routable_role(role: &ProviderRole) -> bool {
5544    matches!(
5545        role,
5546        ProviderRole::ToolProvider { .. }
5547            | ProviderRole::ManagementSurface { .. }
5548            | ProviderRole::InternalService { .. }
5549    )
5550}
5551
5552#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5553enum ControlRequestBodyError {
5554    UnknownOp,
5555    InvalidBody,
5556}
5557
5558#[derive(Debug, Deserialize)]
5559struct ControlOpProbe {
5560    op: String,
5561}
5562
5563/// Channel-0 push ops this subc version understands. A push whose `op` is not in
5564/// this set is treated as a forward-compat unknown and ignored rather than errored.
5565const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5566
5567fn is_known_module_push_op(body: &[u8]) -> bool {
5568    serde_json::from_slice::<ControlOpProbe>(body)
5569        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5570        .unwrap_or(false)
5571}
5572
5573fn is_known_module_request_op(body: &[u8]) -> bool {
5574    serde_json::from_slice::<ControlOpProbe>(body)
5575        .map(|probe| is_module_to_subc_op(&probe.op))
5576        .unwrap_or(false)
5577}
5578
5579fn is_module_to_subc_op(op: &str) -> bool {
5580    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5581}
5582
5583fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5584    debug!(
5585        op = %op,
5586        connection_id = connection_id.get(),
5587        corr,
5588        "control dispatch"
5589    );
5590}
5591
5592fn log_slow_control_dispatch(
5593    dispatch_started_at: Option<StdInstant>,
5594    op: &'static str,
5595    connection_id: ConnectionId,
5596    corr: u64,
5597) {
5598    let Some(dispatch_started_at) = dispatch_started_at else {
5599        return;
5600    };
5601    let elapsed = dispatch_started_at.elapsed();
5602    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5603        warn!(
5604            op = %op,
5605            connection_id = connection_id.get(),
5606            corr,
5607            elapsed_ms = elapsed.as_millis() as u64,
5608            "slow control dispatch"
5609        );
5610    }
5611}
5612
5613fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5614    match request {
5615        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5616        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5617        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5618        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5619        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5620        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5621        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5622        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5623        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5624        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5625        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5626        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5627        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5628        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5629        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5630        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5631        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5632        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5633        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5634    }
5635}
5636
5637fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5638    match request {
5639        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5640        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5641        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5642        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5643    }
5644}
5645
5646fn parse_client_control_request(
5647    body: &[u8],
5648) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5649    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5650        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5651            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5652                ControlRequestBodyError::InvalidBody
5653            }
5654            Ok(_) => ControlRequestBodyError::UnknownOp,
5655            Err(_) => ControlRequestBodyError::InvalidBody,
5656        };
5657        (err, classification)
5658    })
5659}
5660
5661fn parse_module_control_request_from_module(
5662    body: &[u8],
5663) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5664    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5665        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5666            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5667            Ok(_) => ControlRequestBodyError::UnknownOp,
5668            Err(_) => ControlRequestBodyError::InvalidBody,
5669        };
5670        (err, classification)
5671    })
5672}
5673
5674#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5675enum ProviderRoleKind {
5676    ToolProvider,
5677    PipelineStage,
5678    ManagementSurface,
5679    InternalService,
5680}
5681
5682fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5683    match role {
5684        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5685        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5686        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5687        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5688    }
5689}
5690
5691fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5692    roles.iter().map(provider_role_kind).collect()
5693}
5694
5695/// Most refused scope records named individually in the log per sync; the
5696/// `refused` count on the accepted line is always complete.
5697const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5698
5699/// Per-outcome counts of one accepted `scope.sync`, for its log line.
5700#[derive(Debug, Default, PartialEq, Eq)]
5701struct ScopeOutcomeCounts {
5702    created: usize,
5703    replaced: usize,
5704    updated: usize,
5705    unchanged: usize,
5706    refused: usize,
5707}
5708
5709impl ScopeOutcomeCounts {
5710    fn of(results: &[ScopeRecordResult]) -> Self {
5711        let mut counts = Self::default();
5712        for result in results {
5713            let slot = match result.outcome {
5714                ScopeRecordOutcome::Created => &mut counts.created,
5715                ScopeRecordOutcome::Replaced => &mut counts.replaced,
5716                ScopeRecordOutcome::Updated => &mut counts.updated,
5717                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5718                ScopeRecordOutcome::Refused => &mut counts.refused,
5719            };
5720            *slot += 1;
5721        }
5722        counts
5723    }
5724}
5725
5726#[cfg(test)]
5727mod scope_outcome_count_tests {
5728    use super::*;
5729
5730    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5731        ScopeRecordResult {
5732            scope_ref: "r".to_string(),
5733            scope_epoch: 1,
5734            outcome,
5735            code: None,
5736            message: None,
5737            version: None,
5738            parent_state: None,
5739        }
5740    }
5741
5742    /// Each outcome lands in its own count, so a refused record can never be
5743    /// hidden inside the total the log already printed.
5744    #[test]
5745    fn every_outcome_is_counted_in_its_own_field() {
5746        let results = [
5747            result(ScopeRecordOutcome::Created),
5748            result(ScopeRecordOutcome::Created),
5749            result(ScopeRecordOutcome::Replaced),
5750            result(ScopeRecordOutcome::Updated),
5751            result(ScopeRecordOutcome::Unchanged),
5752            result(ScopeRecordOutcome::Refused),
5753            result(ScopeRecordOutcome::Refused),
5754            result(ScopeRecordOutcome::Refused),
5755        ];
5756        assert_eq!(
5757            ScopeOutcomeCounts::of(&results),
5758            ScopeOutcomeCounts {
5759                created: 2,
5760                replaced: 1,
5761                updated: 1,
5762                unchanged: 1,
5763                refused: 3,
5764            }
5765        );
5766    }
5767}
5768
5769/// Return whether a catalog change can create a newly violating live route.
5770/// Removing an attested claim is intentionally excluded: it makes fewer routes
5771/// forbidden and therefore must leave the existing route census untouched.
5772fn capability_census_trigger(
5773    old: Option<&CapabilityDeclarations>,
5774    new: Option<&CapabilityDeclarations>,
5775) -> bool {
5776    let old_provides = old
5777        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5778        .unwrap_or_default();
5779    let old_denies = old
5780        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5781        .unwrap_or_default();
5782    let new = new.cloned().unwrap_or(CapabilityDeclarations {
5783        provides: Vec::new(),
5784        requires: Vec::new(),
5785        must_never_reach: Vec::new(),
5786    });
5787
5788    new.provides
5789        .iter()
5790        .any(|capability| !old_provides.contains(capability))
5791        || new
5792            .must_never_reach
5793            .iter()
5794            .any(|capability| !old_denies.contains(capability))
5795}
5796
5797/// Find the first capability an attested opener denies that an attested target
5798/// claims. Both manifests are live registry records, never cached or client data.
5799fn denied_capability<'a>(
5800    opening_manifest: &'a ModuleManifest,
5801    target_manifest: &ModuleManifest,
5802) -> Option<&'a str> {
5803    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5804    let target_capabilities = target_manifest.capabilities.as_ref()?;
5805    opening_capabilities
5806        .must_never_reach
5807        .iter()
5808        .find(|denied| {
5809            target_capabilities
5810                .provides
5811                .iter()
5812                .any(|provided| provided == *denied)
5813        })
5814        .map(String::as_str)
5815}
5816
5817fn catalog_update_frozen_field_message(
5818    registered: &ModuleManifest,
5819    provides: &[ProviderRole],
5820) -> Option<String> {
5821    let old_has_provides = !registered.provides.is_empty();
5822    let new_has_provides = !provides.is_empty();
5823    if old_has_provides != new_has_provides {
5824        return Some(format!(
5825            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5826            registered.module_id
5827        ));
5828    }
5829
5830    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
5831        return Some(format!(
5832            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5833            registered.module_id
5834        ));
5835    }
5836
5837    let registered_concurrency = manifest_concurrency(registered);
5838    let mut candidate = registered.clone();
5839    candidate.provides = provides.to_vec();
5840    let candidate_concurrency = manifest_concurrency(&candidate);
5841    if candidate_concurrency != registered_concurrency {
5842        return Some(format!(
5843            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5844            registered.module_id, registered_concurrency, candidate_concurrency
5845        ));
5846    }
5847
5848    // control_ops live beside the manifest in the HELLO body, not inside
5849    // ModuleManifest, so a provides-only catalog.update cannot change them.
5850    None
5851}
5852
5853fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5854    manifest.provides.iter().any(is_routable_role)
5855}
5856
5857/// Returns the routable-provider concurrency subc should enforce for this manifest.
5858///
5859/// ToolProvider and ManagementSurface store their delivery concurrency directly.
5860/// InternalService has no role-specific concurrency field, so it retains the
5861/// existing ModuleManaged default for backward compatibility.
5862fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5863    manifest
5864        .provides
5865        .iter()
5866        .find_map(|provider| match provider {
5867            ProviderRole::ToolProvider { concurrency, .. }
5868            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5869            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5870        })
5871        .unwrap_or(Concurrency::ModuleManaged)
5872}
5873
5874/// True when the manifest carries a ManagementSurface role whose concurrency
5875/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
5876/// bytes because the typed manifest deliberately erases that distinction: the
5877/// default exists for wire compatibility, and this probe exists so the default
5878/// stays observable. Any parse irregularity returns false -- the caller only
5879/// logs, and a malformed body already failed registration upstream.
5880fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5881    let has_management_surface = manifest
5882        .provides
5883        .iter()
5884        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5885    if !has_management_surface {
5886        return false;
5887    }
5888    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5889        return false;
5890    };
5891    let Some(provides) = raw
5892        .get("manifest")
5893        .and_then(|manifest| manifest.get("provides"))
5894        .and_then(serde_json::Value::as_array)
5895    else {
5896        return false;
5897    };
5898    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
5899    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
5900    // against the management_surface_manifest_without_concurrency golden, not
5901    // recalled (the externally-tagged guess was this function's first bug).
5902    provides.iter().any(|role| {
5903        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5904            && role.get("concurrency").is_none()
5905    })
5906}
5907
5908fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5909    if peer_version != PROTOCOL_VERSION {
5910        return Err(format!(
5911            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5912        ));
5913    }
5914    Ok(PROTOCOL_VERSION)
5915}
5916
5917fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5918    Frame::build_with_version(
5919        response_version(frame),
5920        FrameType::Pong,
5921        frame.header.flags,
5922        0,
5923        0,
5924        frame.header.corr,
5925        Vec::new(),
5926    )
5927    .map_err(RouterError::FrameBuild)
5928}
5929
5930fn control_error_frame(
5931    frame: &Frame,
5932    code: &'static str,
5933    message: impl Into<String>,
5934) -> Result<Frame, RouterError> {
5935    control_error_body_frame(
5936        frame,
5937        ErrorBody {
5938            code: code.to_string(),
5939            message: message.into(),
5940            detail: None,
5941        },
5942    )
5943}
5944
5945fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5946    let body = serde_json::to_vec(&error).map_err(|err| {
5947        RouterError::backend(
5948            0,
5949            frame.header.corr,
5950            format!("failed to encode control ERROR: {err}"),
5951        )
5952    })?;
5953
5954    Frame::build_with_version(
5955        response_version(frame),
5956        FrameType::Error,
5957        control_flags(),
5958        0,
5959        0,
5960        frame.header.corr,
5961        body,
5962    )
5963    .map_err(RouterError::FrameBuild)
5964}
5965
5966fn control_response_body_frame<T: Serialize>(
5967    frame: &Frame,
5968    reply: &T,
5969    label: &'static str,
5970) -> Result<Frame, RouterError> {
5971    let body = serde_json::to_vec(reply).map_err(|err| {
5972        RouterError::backend(
5973            0,
5974            frame.header.corr,
5975            format!("failed to encode {label}: {err}"),
5976        )
5977    })?;
5978
5979    Frame::build_with_version(
5980        response_version(frame),
5981        FrameType::Response,
5982        control_flags(),
5983        0,
5984        0,
5985        frame.header.corr,
5986        body,
5987    )
5988    .map_err(RouterError::FrameBuild)
5989}
5990
5991/// Map a forwarding failure to the wire code a client sees.
5992///
5993/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
5994/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
5995/// chosen here decides whether a caller retries or gives up.
5996///
5997/// That makes attribution the load-bearing property, not merely having a code. A
5998/// permanent fault published as a retryable one produces a fleet-wide retry storm
5999/// against something that can never recover; a transient fault published as
6000/// permanent gives up on work that would have succeeded. Both look correct in a
6001/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
6002/// the mapping per variant rather than merely asserting that some code exists.
6003///
6004/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
6005/// two codes on the same side of the boundary passes it. Measured rather than
6006/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
6007/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
6008/// a test named for something else that happens to assert the string.
6009///
6010/// That accidental coverage is deliberately left alone rather than promoted to a
6011/// named test, because it guards a property this function does not promise.
6012/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
6013/// specific code within a class, so identity is free to change and only the
6014/// partition is a contract. Splitting it out would assert a guarantee nothing
6015/// depends on — and a suite that promises more than the code does is the harder
6016/// thing to correct later, because the next reader cannot tell which assertions
6017/// are load-bearing.
6018///
6019/// Pin identity here the moment a consumer branches on a specific code.
6020fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6021    match err {
6022        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6023        ForwardingError::NoModuleConnection => "target_unavailable",
6024        ForwardingError::ModuleReloading { .. } => "module_reloading",
6025        ForwardingError::ClientRouteChannelExhausted { .. }
6026        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6027        ForwardingError::StaleModuleEndpoint
6028        | ForwardingError::UnknownReservation { .. }
6029        | ForwardingError::ConnectionClosing { .. }
6030        | ForwardingError::ClientEgressClosed { .. }
6031        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6032        // Only a swap candidate's registration can produce this, and it means
6033        // exactly what a second active HELLO for a live id means.
6034        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6035        ForwardingError::RelayCorrelationExhausted
6036        | ForwardingError::RouteOpenBuild(_)
6037        | ForwardingError::Poisoned => "forwarding_error",
6038    }
6039}
6040
6041fn response_version(frame: &Frame) -> u8 {
6042    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6043        frame.header.ver
6044    } else {
6045        PROTOCOL_VERSION
6046    }
6047}
6048
6049fn control_flags() -> Flags {
6050    Flags::new(false, Priority::Passive, false)
6051}
6052
6053/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
6054/// channel. The target is the module (a client never saw the route), so this
6055/// takes the module path: delivered late rather than dropped when the module's
6056/// queue is momentarily full, and never closing its connection.
6057fn send_goodbye_target_best_effort(
6058    counters: &DaemonCounters,
6059    target: &GoodbyeTarget,
6060    context: &'static str,
6061) {
6062    let Ok(frame) = Frame::build_with_version(
6063        target.negotiated_ver,
6064        FrameType::Goodbye,
6065        control_flags(),
6066        target.channel,
6067        target.epoch,
6068        0,
6069        Vec::new(),
6070    ) else {
6071        return;
6072    };
6073    crate::forwarding::send_module_route_goodbye(
6074        counters,
6075        &target.sink,
6076        frame,
6077        target.module_id.as_deref(),
6078        context,
6079    );
6080}
6081
6082pub(crate) fn send_route_control_pushes(
6083    forwarding: &ForwardingTable,
6084    routes: Vec<EndpointRoute>,
6085    push: ClientControlPush,
6086) {
6087    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6088    for route in routes {
6089        let target = route.goodbye_target;
6090        if let Some((existing, channels)) = targets
6091            .iter_mut()
6092            .find(|(existing, _)| existing.connection_id == target.connection_id)
6093        {
6094            debug_assert_eq!(
6095                existing.negotiated_ver, target.negotiated_ver,
6096                "one connection cannot negotiate multiple frame versions"
6097            );
6098            if !channels.contains(&target.channel) {
6099                channels.push(target.channel);
6100            }
6101            continue;
6102        }
6103        let channel = target.channel;
6104        targets.push((target, vec![channel]));
6105    }
6106    for (target, mut channels) in targets {
6107        channels.sort_unstable();
6108        let mut push = push.clone();
6109        match &mut push {
6110            ClientControlPush::RouteClosing {
6111                channels: covered, ..
6112            }
6113            | ClientControlPush::RouteClosed {
6114                channels: covered, ..
6115            } => *covered = channels,
6116        }
6117        let body = match serde_json::to_vec(&push) {
6118            Ok(body) => body,
6119            Err(err) => {
6120                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6121                continue;
6122            }
6123        };
6124        let frame = match Frame::build_with_version(
6125            target.negotiated_ver,
6126            FrameType::Push,
6127            control_flags(),
6128            0,
6129            0,
6130            0,
6131            body.clone(),
6132        ) {
6133            Ok(frame) => frame,
6134            Err(err) => {
6135                warn!(
6136                    route_channel = target.channel,
6137                    error = %err,
6138                    "failed to build route lifecycle control PUSH frame"
6139                );
6140                continue;
6141            }
6142        };
6143        if let Err(err) = target.sink.try_send(frame) {
6144            if target.close_on_delivery_failure() {
6145                warn!(
6146                    target_connection_id = target.connection_id.get(),
6147                    route_channel = target.channel,
6148                    error = %err,
6149                    "route lifecycle control PUSH was not delivered to client; closing target connection"
6150                );
6151                let _ = forwarding.escalate_client_delivery_failure(
6152                    target.connection_id,
6153                    target.channel,
6154                    target.epoch,
6155                    CloseReason::new(
6156                        "route_lifecycle_push_delivery_failed",
6157                        format!(
6158                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6159                            target.channel
6160                        ),
6161                    ),
6162                    crate::forwarding::UndeliveredFrame {
6163                        module_id: target.module_id.as_deref(),
6164                        sink: &target.sink,
6165                    },
6166                );
6167            }
6168        }
6169    }
6170}
6171
6172#[cfg(test)]
6173mod tests {
6174    #[cfg(unix)]
6175    #[tokio::test]
6176    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6177        let dir = subc_test_support::TestTempDir::new("rescan-live-health");
6178        let path = dir.join("subc.jsonc");
6179        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6180            "program":"/bin/sleep","args":["60"],"protocol":"none",
6181            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6182        }}}).to_string()).unwrap();
6183        let mut configured = crate::daemon_config::load(&path)
6184            .unwrap()
6185            .unwrap()
6186            .modules
6187            .pop()
6188            .unwrap();
6189        let registry = std::sync::Arc::new(crate::Registry::default());
6190        let handle = crate::SupervisorHandle::new();
6191        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6192            .with_handle(handle.clone());
6193        let module = supervisor
6194            .supervise_configured_with_health(
6195                configured.module_spec(),
6196                true,
6197                configured.health.clone(),
6198                None,
6199                configured.restart,
6200            )
6201            .unwrap();
6202        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6203        let before = module.status().unwrap().pid;
6204        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6205        configured.health.cadence = std::time::Duration::from_secs(3600);
6206        let health_only = handler
6207            .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6208            .await
6209            .unwrap();
6210        assert!(
6211            health_only.changed_pending_reload.is_empty(),
6212            "health policy is already applied live"
6213        );
6214        assert_eq!(module.status().unwrap().pid, before);
6215        assert_eq!(
6216            module.configuration().unwrap().1.http,
6217            configured.health.http
6218        );
6219        configured.args = vec!["61".into()];
6220        let launch = handler
6221            .reconcile_supervised_modules(&supervisor, vec![configured], false)
6222            .await
6223            .unwrap();
6224        assert_eq!(launch.changed_pending_reload, ["stock"]);
6225        assert_eq!(
6226            module.status().unwrap().pid,
6227            before,
6228            "a launch edit is stored until reload"
6229        );
6230        module.drain().await.unwrap();
6231    }
6232    use std::{
6233        collections::BTreeMap,
6234        fmt,
6235        path::PathBuf,
6236        sync::{Arc, Mutex},
6237        time::Duration,
6238    };
6239    use subc_test_support::TestTempDir;
6240
6241    use serde_json::{json, Value};
6242    use subc_protocol::{
6243        manifest::{
6244            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6245            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6246        },
6247        session::HealthStatus,
6248        FrameType,
6249    };
6250
6251    use super::*;
6252    use crate::{
6253        forwarding::{DataRoute, DataRouteState},
6254        registry::ChannelState,
6255        router::FrameSink,
6256        stderr_tail::DEFAULT_MAX_LINE_BYTES,
6257        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6258        RouteCtx, Router,
6259    };
6260    use tokio::{
6261        sync::mpsc,
6262        time::{sleep, Instant},
6263    };
6264    use tracing::{
6265        field::{Field, Visit},
6266        Event, Subscriber,
6267    };
6268    use tracing_subscriber::{layer::Context, prelude::*, Layer};
6269
6270    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
6271    ///
6272    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
6273    /// is only populated for `tests/*.rs` integration test binaries -- this file
6274    /// compiles as part of the library target, which gets neither. This test's
6275    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
6276    /// and the sibling binary lives two directories up at
6277    /// `<target-dir>/<profile>/fake-aft-stub`.
6278    ///
6279    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
6280    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
6281    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
6282    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
6283    /// `NotFound`, which reads as a broken test rather than an unbuilt
6284    /// dependency -- so state the cause and the remedy instead. Deliberately a
6285    /// panic and not a silent skip: a test that quietly passes when it could not
6286    /// run is worse than one that fails, because it reports health it never
6287    /// verified.
6288    fn fake_aft_stub_path() -> PathBuf {
6289        let mut path = std::env::current_exe().expect("current_exe available in tests");
6290        path.pop(); // .../deps/
6291        path.pop(); // .../<profile>/
6292        path.push(if cfg!(windows) {
6293            "fake-aft-stub.exe"
6294        } else {
6295            "fake-aft-stub"
6296        });
6297        assert!(
6298            path.exists(),
6299            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6300             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6301            path.display()
6302        );
6303        path
6304    }
6305
6306    /// Whether clients retry `code` in place: the predicate itself, never a copy
6307    /// of its set. A copied list breaks silently when a code is added to or
6308    /// removed from the real one, and a stale copy here would let exactly the
6309    /// failure this test exists to catch pass.
6310    fn client_retries(code: &str) -> bool {
6311        subc_protocol::error_codes::is_retryable_route_open(code)
6312    }
6313
6314    /// A code is not a label — clients branch on it, so publishing the wrong KIND
6315    /// of failure is worse than publishing none. A permanent fault dressed as
6316    /// retryable makes every client in the fleet retry forever against something
6317    /// that cannot recover; a transient fault dressed as permanent abandons work
6318    /// that would have succeeded.
6319    ///
6320    /// Asserting "a code exists" cannot catch either, because the string is free
6321    /// to say anything. This enumerates every variant and pins which side of the
6322    /// retry boundary it lands on, so a new variant must be classified here
6323    /// deliberately rather than inheriting whichever arm it was appended to.
6324    #[test]
6325    fn retryability_of_forwarding_codes_matches_the_failure() {
6326        // Transient by nature: the target is booting, reloading, or its endpoint
6327        // was swapped mid-flight. Retrying is how these resolve.
6328        let transient = [
6329            ForwardingError::NoModuleConnection,
6330            ForwardingError::ModuleReloading {
6331                module_id: "m".into(),
6332            },
6333            ForwardingError::StaleModuleEndpoint,
6334            ForwardingError::UnknownReservation {
6335                client_channel: 1,
6336                module_channel: 1,
6337            },
6338            ForwardingError::ConnectionClosing {
6339                connection_id: ConnectionId::new(1),
6340            },
6341            ForwardingError::ClientEgressClosed {
6342                connection_id: ConnectionId::new(1),
6343            },
6344            ForwardingError::ModuleEgressUnavailable {
6345                connection_id: ConnectionId::new(1),
6346            },
6347        ];
6348        for err in transient {
6349            let code = forwarding_error_code(&err);
6350            assert!(
6351                client_retries(code),
6352                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6353            );
6354        }
6355
6356        // Not fixed by retrying. Channel and correlation exhaustion need the
6357        // caller to close routes, and a poisoned lock is a daemon that cannot
6358        // recover at all — the worst thing to advertise as retryable, since every
6359        // client would storm a daemon that will never answer.
6360        let permanent = [
6361            ForwardingError::ConnectionRoleConflict {
6362                connection_id: ConnectionId::new(1),
6363            },
6364            ForwardingError::ClientRouteChannelExhausted {
6365                connection_id: ConnectionId::new(1),
6366            },
6367            ForwardingError::ModuleRouteChannelExhausted {
6368                endpoint: ModuleEndpointId {
6369                    connection_id: ConnectionId::new(1),
6370                    generation: 1,
6371                },
6372            },
6373            ForwardingError::RelayCorrelationExhausted,
6374            ForwardingError::RouteOpenBuild("x".into()),
6375            ForwardingError::Poisoned,
6376        ];
6377        for err in permanent {
6378            let code = forwarding_error_code(&err);
6379            assert!(
6380                !client_retries(code),
6381                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6382            );
6383        }
6384    }
6385
6386    /// The principal is the daemon's answer to "who is calling", and modules
6387    /// branch on it: aft gates bash on it, cerebellum gates browser control,
6388    /// plexus gates connector invocation. So a stamp is an authorization input in
6389    /// another process, not a label — and both possible answers SUCCEED, which is
6390    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
6391    /// first-party capability to something that never proved it; a supervised one
6392    /// stamped `Direct` silently strips a module of capability it is entitled to.
6393    ///
6394    /// Neither shows up in a test that only checks the bind succeeded. Before this
6395    /// test the only coverage was accidental —
6396    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
6397    /// stamped principal on its way past, so narrowing that wire-shape test to its
6398    /// stated subject would have deleted the last assertion on this value. It
6399    /// still asserts the stamp, which is now redundancy rather than the only
6400    /// guard: both fail under the same mutation, and this one names the reason.
6401    /// SCOPE: this handler's supervisor has spawned nothing, so
6402    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
6403    /// is unreachable here. Both assertions below are refusals, and a mutant that
6404    /// refuses everything would satisfy them.
6405    ///
6406    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
6407    /// spawns a supervised consumer, reads its live nonce, and asserts the module
6408    /// observed `principal.kind == "reserved"` carrying that module_id — verified
6409    /// at source rather than assumed, since a citation is a claim about another
6410    /// file and ages like one. Recorded because a harness that structurally
6411    /// cannot reach an arm reports "none" for that arm identically to one that
6412    /// covers it and found nothing.
6413    #[tokio::test]
6414    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6415        let handler = ControlHandler::default();
6416        let frame =
6417            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6418
6419        // Absent consumer_identity is the ordinary case: a human at a terminal, or
6420        // any process holding the connection file. Nothing was proved, so nothing
6421        // may be granted beyond the unattested floor.
6422        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6423        assert_eq!(
6424            stamped,
6425            Principal::Direct,
6426            "a caller that proved nothing must not be stamped as a supervised module"
6427        );
6428
6429        // A claimed module_id with a nonce no supervised child was given is a
6430        // forgery attempt, not a weaker caller: it must be REFUSED rather than
6431        // quietly demoted to Direct, or an impersonation attempt looks identical
6432        // to an ordinary unattested connection.
6433        let forged = handler
6434            .route_open_principal(
6435                &frame,
6436                Some(ConsumerIdentity {
6437                    module_id: "aft".to_string(),
6438                    launch_nonce: "not-a-real-nonce".to_string(),
6439                }),
6440            )
6441            .unwrap();
6442        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6443        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6444    }
6445
6446    /// The test above hands `route_open_principal` an identity it built itself,
6447    /// which proves the stamping rule and nothing about where the identity comes
6448    /// from. The real producer is a wire body, and the two are joined by a serde
6449    /// field name that nothing else asserts.
6450    ///
6451    /// That join fails quietly in one specific way: an unrecognised key is simply
6452    /// absent after parsing, so a renamed or misspelled `consumer_identity`
6453    /// yields `None` and every supervised module silently drops to `Direct`.
6454    /// Capability-wise that is the safe direction, but it surfaces far from its
6455    /// cause — as a module mysteriously refused bash — and it would pass every
6456    /// test that builds its own input.
6457    ///
6458    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
6459    /// would break every client the moment the daemon gains a field, trading a
6460    /// quiet demotion for a hard refusal on additive change. Asserting the join
6461    /// instead means a rename breaks a test here rather than the fleet.
6462    #[test]
6463    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6464        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6465        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6466        let ClientControlRequest::RouteOpen {
6467            consumer_identity, ..
6468        } = parsed
6469        else {
6470            panic!("route.open body must parse as RouteOpen");
6471        };
6472        assert_eq!(
6473            consumer_identity,
6474            Some(ConsumerIdentity {
6475                module_id: "aft".to_string(),
6476                launch_nonce: "n".to_string(),
6477            }),
6478            "the wire field name must reach the value route_open_principal reads"
6479        );
6480    }
6481
6482    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6483        ModuleManifest::builder(module_id, "0.1.0")
6484            .protocol_ver(protocol_ver)
6485            .provides(vec![ProviderRole::ToolProvider {
6486                tools: vec![Tool {
6487                    name: "read".to_string(),
6488                    description: None,
6489                    execution_mode: ExecutionMode::Pure,
6490                    schema: json!({"type": "object"}),
6491                }],
6492                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6493                concurrency: Concurrency::ModuleManaged,
6494                emits_push: true,
6495                sub_supervises: true,
6496            }])
6497            .build()
6498    }
6499
6500    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6501        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6502    }
6503
6504    fn hello_frame_with_control_ops(
6505        module_id: &str,
6506        protocol_ver: u8,
6507        corr: u64,
6508        control_ops: Option<Vec<String>>,
6509    ) -> Frame {
6510        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6511    }
6512
6513    fn hello_frame_with_nonce(
6514        module_id: &str,
6515        protocol_ver: u8,
6516        corr: u64,
6517        launch_nonce: Option<&str>,
6518    ) -> Frame {
6519        hello_frame_full(
6520            module_id,
6521            protocol_ver,
6522            corr,
6523            None,
6524            launch_nonce.map(ToOwned::to_owned),
6525        )
6526    }
6527
6528    fn hello_frame_full(
6529        module_id: &str,
6530        protocol_ver: u8,
6531        corr: u64,
6532        control_ops: Option<Vec<String>>,
6533        launch_nonce: Option<String>,
6534    ) -> Frame {
6535        let body = serde_json::to_vec(&ModuleHelloBody {
6536            manifest: manifest(module_id, protocol_ver),
6537            protocol_ver,
6538            control_ops,
6539            launch_nonce,
6540        })
6541        .unwrap();
6542        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6543    }
6544
6545    fn non_routable_hello_frame_with_control_ops(
6546        module_id: &str,
6547        corr: u64,
6548        control_ops: Option<Vec<String>>,
6549    ) -> Frame {
6550        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6551        manifest.provides.clear();
6552        let body = serde_json::to_vec(&ModuleHelloBody {
6553            manifest,
6554            protocol_ver: PROTOCOL_VERSION,
6555            control_ops,
6556            launch_nonce: None,
6557        })
6558        .unwrap();
6559        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6560    }
6561
6562    fn capability_grammar_hello_frame(
6563        capabilities: Value,
6564        runtime_computed: Option<Value>,
6565        corr: u64,
6566    ) -> Frame {
6567        let mut body = serde_json::to_value(ModuleHelloBody {
6568            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6569            protocol_ver: PROTOCOL_VERSION,
6570            control_ops: None,
6571            launch_nonce: None,
6572        })
6573        .expect("HELLO body serializes");
6574        body["manifest"]["capabilities"] = capabilities;
6575        if let Some(runtime_computed) = runtime_computed {
6576            body["runtime_computed"] = runtime_computed;
6577        }
6578        Frame::build(
6579            FrameType::Hello,
6580            control_flags(),
6581            0,
6582            0,
6583            corr,
6584            serde_json::to_vec(&body).expect("HELLO body reserializes"),
6585        )
6586        .expect("HELLO frame builds")
6587    }
6588
6589    fn channel_request(channel: u16, corr: u64) -> Frame {
6590        Frame::build(
6591            FrameType::Request,
6592            Flags::new(true, Priority::Interactive, false),
6593            channel,
6594            0,
6595            corr,
6596            b"opaque".to_vec(),
6597        )
6598        .unwrap()
6599    }
6600
6601    fn route_ctx(
6602        connection_id: ConnectionId,
6603    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6604        let (tx, rx) = mpsc::channel(8);
6605        (
6606            RouteCtx {
6607                connection_id,
6608                egress: FrameSink::new(tx),
6609            },
6610            rx,
6611        )
6612    }
6613
6614    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6615        serde_json::from_slice(&frame.body).unwrap()
6616    }
6617
6618    /// Register a module over a connection that has a sink and return the
6619    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
6620    /// module's own sink rather than returning it as a reply, so the ack is
6621    /// taken off `rx` here and whatever the test reads next is what followed it.
6622    async fn hello_via_sink(
6623        handler: &ControlHandler,
6624        ctx: &RouteCtx,
6625        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6626        hello: Frame,
6627    ) -> Frame {
6628        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6629        assert!(
6630            replies.is_empty(),
6631            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6632        );
6633        let ack = rx
6634            .try_recv()
6635            .expect("HELLO_ACK is queued on the module sink")
6636            .frame;
6637        assert_eq!(ack.header.ty, FrameType::HelloAck);
6638        ack
6639    }
6640
6641    fn parse_error(frame: &Frame) -> Value {
6642        serde_json::from_slice(&frame.body).unwrap()
6643    }
6644
6645    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6646        serde_json::from_slice(&frame.body).unwrap()
6647    }
6648
6649    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6650        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6651            route_channel,
6652            route_epoch: 0,
6653            kind,
6654        })
6655        .unwrap();
6656        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6657    }
6658
6659    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6660        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6661            module_id: module_id.to_string(),
6662        })
6663        .unwrap();
6664        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6665    }
6666
6667    fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6668        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6669    }
6670
6671    fn route_open_frame_with_consumer_capabilities(
6672        corr: u64,
6673        module_id: &str,
6674        project_root: TestTempDir,
6675        consumer_capabilities: Option<Vec<String>>,
6676    ) -> Frame {
6677        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6678            target: RouteTarget::ToolProvider {
6679                module_id: module_id.to_string(),
6680            },
6681            identity: BindIdentity::new(
6682                project_root.path().to_path_buf(),
6683                "unit".to_string(),
6684                "session".to_string(),
6685            ),
6686            consumer_identity: None,
6687            consumer_capabilities,
6688            role_versions: None,
6689            admission_facts: None,
6690            scope: None,
6691        })
6692        .unwrap();
6693        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6694    }
6695
6696    fn route_open_frame_with_role_versions(
6697        corr: u64,
6698        module_id: &str,
6699        project_root: TestTempDir,
6700        role_versions: Option<BTreeMap<String, String>>,
6701    ) -> Frame {
6702        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6703            target: RouteTarget::ToolProvider {
6704                module_id: module_id.to_string(),
6705            },
6706            identity: BindIdentity::new(
6707                project_root.path().to_path_buf(),
6708                "unit".to_string(),
6709                format!("session-{corr}"),
6710            ),
6711            consumer_identity: None,
6712            consumer_capabilities: None,
6713            role_versions,
6714            admission_facts: None,
6715            scope: None,
6716        })
6717        .unwrap();
6718        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6719    }
6720
6721    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6722        entries
6723            .iter()
6724            .map(|(role, version)| (role.to_string(), version.to_string()))
6725            .collect()
6726    }
6727
6728    fn route_open_frame_with_admission_facts(
6729        corr: u64,
6730        module_id: &str,
6731        project_root: TestTempDir,
6732        consumer_identity: Option<subc_control::ConsumerIdentity>,
6733        facts: Option<Value>,
6734    ) -> Frame {
6735        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6736            target: RouteTarget::ToolProvider {
6737                module_id: module_id.to_string(),
6738            },
6739            identity: BindIdentity::new(
6740                project_root.path().to_path_buf(),
6741                "unit".to_string(),
6742                format!("session-{corr}"),
6743            ),
6744            consumer_identity,
6745            consumer_capabilities: None,
6746            role_versions: None,
6747            admission_facts: facts,
6748            scope: None,
6749        })
6750        .unwrap();
6751        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6752    }
6753
6754    #[derive(Clone, Default)]
6755    struct EventCapture {
6756        events: Arc<Mutex<Vec<CapturedEvent>>>,
6757    }
6758
6759    #[derive(Clone, Debug)]
6760    struct CapturedEvent {
6761        target: String,
6762        level: tracing::Level,
6763        fields: BTreeMap<String, String>,
6764    }
6765
6766    impl EventCapture {
6767        fn events(&self) -> Vec<CapturedEvent> {
6768            self.events.lock().unwrap().clone()
6769        }
6770    }
6771
6772    impl<S> Layer<S> for EventCapture
6773    where
6774        S: Subscriber,
6775    {
6776        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6777            let mut visitor = EventFieldVisitor::default();
6778            event.record(&mut visitor);
6779            self.events.lock().unwrap().push(CapturedEvent {
6780                target: event.metadata().target().to_string(),
6781                level: *event.metadata().level(),
6782                fields: visitor.fields,
6783            });
6784        }
6785    }
6786
6787    #[derive(Default)]
6788    struct EventFieldVisitor {
6789        fields: BTreeMap<String, String>,
6790    }
6791
6792    impl Visit for EventFieldVisitor {
6793        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6794            self.fields
6795                .insert(field.name().to_string(), format!("{value:?}"));
6796        }
6797    }
6798
6799    fn health_response(corr: u64, status: HealthStatus) -> Frame {
6800        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6801            status,
6802            detail: Some("warming".to_string()),
6803            metrics: Some(json!({"queue_depth": 3})),
6804        })
6805        .unwrap();
6806        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6807    }
6808
6809    fn route_bind_ack(corr: u64) -> Frame {
6810        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6811        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6812    }
6813
6814    fn unique_project_root(label: &str) -> TestTempDir {
6815        TestTempDir::new(label)
6816    }
6817
6818    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6819        match parse_route_poll(frame) {
6820            ClientControlResponse::RoutePoll {
6821                status: None,
6822                live: Some(live),
6823                ..
6824            } => assert_eq!(live, expected_live),
6825            other => panic!("unexpected route.poll response: {other:?}"),
6826        }
6827    }
6828
6829    fn bind_liveness_route(
6830        registry: &Registry,
6831        forwarding: &ForwardingTable,
6832        module_id: &str,
6833    ) -> (RouteCtx, u16, u32) {
6834        let module_connection = ConnectionId::new(101);
6835        let client_connection = ConnectionId::new(202);
6836        let registration = registry
6837            .register_with_control_ops(
6838                manifest(module_id, PROTOCOL_VERSION),
6839                PROTOCOL_VERSION,
6840                module_connection,
6841                module_baseline_control_ops(),
6842            )
6843            .unwrap();
6844        let (module_tx, _module_rx) = mpsc::channel(8);
6845        let endpoint = forwarding
6846            .register_module_connection(
6847                module_connection,
6848                module_id.to_string(),
6849                PROTOCOL_VERSION,
6850                manifest_concurrency(&registration.manifest),
6851                FrameSink::new(module_tx),
6852            )
6853            .unwrap();
6854        let (client_ctx, _client_rx) = route_ctx(client_connection);
6855        let pending = forwarding
6856            .begin_route_bind_relay_for_test(
6857                client_connection,
6858                client_ctx.egress.clone(),
6859                1,
6860                module_id,
6861            )
6862            .unwrap();
6863        assert_eq!(pending.endpoint, endpoint);
6864        let route_channel = pending.client_channel;
6865        let route_epoch = pending.client_epoch;
6866        forwarding
6867            .complete_pending_relay(
6868                module_connection,
6869                pending.corr,
6870                RouteBindRelayOutcome::Accepted,
6871            )
6872            .unwrap();
6873        (client_ctx, route_channel, route_epoch)
6874    }
6875
6876    struct FakeProcessLiveness {
6877        live: Option<bool>,
6878    }
6879
6880    impl ModuleProcessLiveness for FakeProcessLiveness {
6881        fn process_live(&self, _module_id: &str) -> Option<bool> {
6882            self.live
6883        }
6884    }
6885
6886    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6887    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6888    {
6889        let registry = Arc::new(Registry::default());
6890        let supervisor_handle = SupervisorHandle::new();
6891        let supervisor = Supervisor::new(
6892            Arc::clone(&registry),
6893            RestartPolicy::new(1, Duration::from_millis(10)),
6894        )
6895        .with_handle(supervisor_handle.clone());
6896        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6897        let module = supervisor
6898            .spawn(ModuleSpec {
6899                module_id: "stderr-tail-wire".to_string(),
6900                program: fake_aft_stub_path(),
6901                args: Vec::new(),
6902                env: vec![
6903                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6904                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6905                ],
6906                reserved: false,
6907                reserved_prefixes: Vec::new(),
6908                protocol: ModuleProtocol::Subc,
6909                overlap: Default::default(),
6910            })
6911            .unwrap();
6912
6913        let deadline = Instant::now() + Duration::from_secs(5);
6914        loop {
6915            let tail = module.stderr_tail(None, None);
6916            if tail
6917                .entries
6918                .iter()
6919                .any(|entry| matches!(entry, TailEntry::ProcessStart))
6920                && tail.entries.iter().any(|entry| {
6921                    matches!(
6922                        entry,
6923                        TailEntry::Line {
6924                            truncated: true,
6925                            ..
6926                        }
6927                    )
6928                })
6929            {
6930                break;
6931            }
6932            assert!(
6933                Instant::now() < deadline,
6934                "module did not produce a truncated line and restart boundary: {tail:?}"
6935            );
6936            sleep(Duration::from_millis(10)).await;
6937        }
6938
6939        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6940        let request = ClientControlRequest::SupervisorStderrTail {
6941            module_id: "stderr-tail-wire".to_string(),
6942            max_lines: None,
6943            max_bytes: None,
6944        };
6945        let frame = Frame::build(
6946            FrameType::Request,
6947            control_flags(),
6948            0,
6949            0,
6950            1,
6951            serde_json::to_vec(&request).unwrap(),
6952        )
6953        .unwrap();
6954        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6955        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6956        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6957            serde_json::from_slice(&responses[0].body).unwrap()
6958        else {
6959            panic!("expected supervisor.stderr_tail response");
6960        };
6961
6962        assert!(
6963            tail.entries
6964                .iter()
6965                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6966            "the control response lost the restart boundary"
6967        );
6968        let Some(StderrTailEntry::Line {
6969            text,
6970            truncated,
6971            at_ms,
6972        }) = tail.entries.iter().find(|entry| {
6973            matches!(
6974                entry,
6975                StderrTailEntry::Line {
6976                    truncated: true,
6977                    ..
6978                }
6979            )
6980        })
6981        else {
6982            panic!("the control response lost the truncated line");
6983        };
6984        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6985        assert!(*truncated);
6986        assert!(
6987            at_ms.is_some(),
6988            "the control response lost the line's capture time"
6989        );
6990    }
6991
6992    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
6993    /// read done on the worker thread would stall every other task until it
6994    /// finished; the read must run off the worker so this test's own task keeps
6995    /// running while the read is paused.
6996    #[tokio::test(flavor = "current_thread")]
6997    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6998        let dir = TestTempDir::new("terminals-off-worker");
6999        let journal_path = dir.join("terminals.jsonl");
7000        let registry = Arc::new(Registry::default());
7001        let supervisor_handle = SupervisorHandle::new();
7002        let supervisor =
7003            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
7004                .with_handle(supervisor_handle.clone())
7005                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7006        let module = supervisor
7007            .spawn(ModuleSpec {
7008                module_id: "terminal-off-worker".to_string(),
7009                program: fake_aft_stub_path(),
7010                args: Vec::new(),
7011                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7012                reserved: false,
7013                reserved_prefixes: Vec::new(),
7014                protocol: ModuleProtocol::Subc,
7015                overlap: Default::default(),
7016            })
7017            .unwrap();
7018        let deadline = Instant::now() + Duration::from_secs(5);
7019        while module.terminal_history().entries.len() != 2 {
7020            assert!(Instant::now() < deadline, "module did not record two exits");
7021            sleep(Duration::from_millis(10)).await;
7022        }
7023
7024        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7025        let handler =
7026            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
7027        let frame = Frame::build(
7028            FrameType::Request,
7029            control_flags(),
7030            0,
7031            0,
7032            1,
7033            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7034                module_id: "terminal-off-worker".to_string(),
7035            })
7036            .unwrap(),
7037        )
7038        .unwrap();
7039        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7040        let spawned_at = std::time::Instant::now();
7041        let read = tokio::spawn({
7042            let handler = Arc::clone(&handler);
7043            async move { handler.handle_control_frame(&ctx, frame).await }
7044        });
7045        // Waiting for the pause from a blocking thread keeps this task pending,
7046        // so the runtime's single worker is free to run the read task.
7047        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7048            .await
7049            .unwrap()
7050            .expect("the history read reached its pause");
7051        let elapsed = spawned_at.elapsed();
7052        assert!(
7053            elapsed < Duration::from_secs(2) && !read.is_finished(),
7054            "this task could not run while the history read was paused \
7055             (resumed after {elapsed:?}, read finished: {})",
7056            read.is_finished()
7057        );
7058
7059        drop(release);
7060        let responses = read.await.unwrap().unwrap();
7061        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7062        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7063            panic!("expected supervisor.terminals response");
7064        };
7065        assert_eq!(terminals.entries.len(), 2);
7066        assert_eq!(terminals.journal_skipped_lines, 0);
7067        assert_eq!(terminals.journal_read_errors, 0);
7068    }
7069
7070    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7071    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7072        let registry = Arc::new(Registry::default());
7073        let supervisor_handle = SupervisorHandle::new();
7074        let supervisor =
7075            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
7076                .with_handle(supervisor_handle.clone());
7077        let module = supervisor
7078            .spawn(ModuleSpec {
7079                module_id: "terminal-golden".to_string(),
7080                program: fake_aft_stub_path(),
7081                args: Vec::new(),
7082                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7083                reserved: false,
7084                reserved_prefixes: Vec::new(),
7085                protocol: ModuleProtocol::Subc,
7086                overlap: Default::default(),
7087            })
7088            .unwrap();
7089
7090        let deadline = Instant::now() + Duration::from_secs(5);
7091        while module.terminal_history().entries.len() != 2 {
7092            assert!(
7093                Instant::now() < deadline,
7094                "module did not retain two terminal exits: {:?}",
7095                module.terminal_history()
7096            );
7097            sleep(Duration::from_millis(10)).await;
7098        }
7099
7100        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
7101        let request = ClientControlRequest::SupervisorTerminals {
7102            module_id: "terminal-golden".to_string(),
7103        };
7104        let frame = Frame::build(
7105            FrameType::Request,
7106            control_flags(),
7107            0,
7108            0,
7109            1,
7110            serde_json::to_vec(&request).unwrap(),
7111        )
7112        .unwrap();
7113        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7114        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7115        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7116        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7117            panic!("expected supervisor.terminals response");
7118        };
7119        assert_eq!(terminals.entries.len(), 2);
7120        assert_eq!(terminals.dropped, 0);
7121
7122        let mut rendered = serde_json::to_value(response).unwrap();
7123        // Wall-clock fields are the observation contract, but not stable fixture
7124        // bytes; normalize only them after the real handler has shaped the response.
7125        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7126        for (index, entry) in rendered["entries"]
7127            .as_array_mut()
7128            .expect("terminal response entries array")
7129            .iter_mut()
7130            .enumerate()
7131        {
7132            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7133        }
7134
7135        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7136            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7137        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7138        if std::env::var_os("UPDATE_GOLDEN").is_some() {
7139            std::fs::write(&golden_path, &serialized).unwrap();
7140        }
7141        let expected: Value =
7142            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7143        assert_eq!(rendered, expected);
7144    }
7145
7146    #[test]
7147    fn hello_registers_manifest_and_returns_ack() {
7148        let registry = Arc::new(Registry::default());
7149        let handler = ControlHandler::new(Arc::clone(&registry));
7150        let conn = ConnectionId::new(1);
7151
7152        let responses = handler
7153            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7154            .unwrap();
7155
7156        assert_eq!(responses.len(), 1);
7157        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7158        assert_eq!(responses[0].header.channel, 0);
7159        assert_eq!(responses[0].header.corr, 7);
7160        let ack = parse_ack(&responses[0]);
7161        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7162        assert!(ack
7163            .subc_capabilities
7164            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7165        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7166        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7167        assert!(ack
7168            .subc_ops
7169            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7170        assert!(ack
7171            .subc_ops
7172            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7173
7174        let registration = registry.get_module("aft").unwrap().unwrap();
7175        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7176        assert_eq!(registration.state, ChannelState::Active);
7177        assert_eq!(registration.connection_id, conn);
7178        assert_eq!(registration.control_ops, module_baseline_control_ops());
7179    }
7180
7181    #[test]
7182    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7183        let invalid_identifiers = [
7184            ("case_change", "credentials-Provider/v1"),
7185            ("leading_zero", "credentials-provider/v01"),
7186            ("trailing_hyphen", "credentials-provider-/v1"),
7187            ("consecutive_hyphens", "credentials--provider/v1"),
7188            ("uppercase", "Credentials-provider/v1"),
7189            ("missing_v", "credentials-provider/1"),
7190            ("whitespace", "credentials provider/v1"),
7191            ("zero_version", "credentials-provider/v0"),
7192            ("out_of_range_version", "credentials-provider/v4294967296"),
7193            (
7194                "overlength_name",
7195                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7196            ),
7197        ];
7198        let mut cases = invalid_identifiers
7199            .into_iter()
7200            .map(|(name, identifier)| {
7201                (
7202                    format!("identifier_{name}"),
7203                    "capabilities.provides[0]".to_string(),
7204                    identifier.to_string(),
7205                    json!({ "provides": [identifier] }),
7206                    None,
7207                )
7208            })
7209            .collect::<Vec<_>>();
7210        cases.extend([
7211            (
7212                "unknown_need".to_string(),
7213                "capabilities.requires[0].need".to_string(),
7214                "deferred".to_string(),
7215                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7216                None,
7217            ),
7218            (
7219                "duplicate_provides".to_string(),
7220                "capabilities.provides[1]".to_string(),
7221                "credentials-provider/v1".to_string(),
7222                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7223                None,
7224            ),
7225            (
7226                "duplicate_must_never_reach".to_string(),
7227                "capabilities.must_never_reach[1]".to_string(),
7228                "credentials-provider/v1".to_string(),
7229                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7230                None,
7231            ),
7232            (
7233                "duplicate_requires_same_need".to_string(),
7234                "capabilities.requires[1]".to_string(),
7235                "credentials-provider/v1".to_string(),
7236                json!({ "requires": [
7237                    { "capability": "credentials-provider/v1", "need": "required" },
7238                    { "capability": "credentials-provider/v1", "need": "required" }
7239                ] }),
7240                None,
7241            ),
7242            (
7243                "duplicate_requires_conflicting_need".to_string(),
7244                "capabilities.requires[1]".to_string(),
7245                "credentials-provider/v1".to_string(),
7246                json!({ "requires": [
7247                    { "capability": "credentials-provider/v1", "need": "required" },
7248                    { "capability": "credentials-provider/v1", "need": "optional" }
7249                ] }),
7250                None,
7251            ),
7252            (
7253                "capabilities_root_pointer".to_string(),
7254                "runtime_computed[0]".to_string(),
7255                "/capabilities".to_string(),
7256                json!({}),
7257                Some(json!(["/capabilities"])),
7258            ),
7259            (
7260                "capabilities_descendant_pointer".to_string(),
7261                "runtime_computed[0]".to_string(),
7262                "/capabilities/provides".to_string(),
7263                json!({}),
7264                Some(json!(["/capabilities/provides"])),
7265            ),
7266            (
7267                "malformed_pointer_without_leading_slash".to_string(),
7268                "runtime_computed[0]".to_string(),
7269                "capabilities".to_string(),
7270                json!({}),
7271                Some(json!(["capabilities"])),
7272            ),
7273            (
7274                "malformed_pointer_escape".to_string(),
7275                "runtime_computed[0]".to_string(),
7276                "/roles/~2/tools".to_string(),
7277                json!({}),
7278                Some(json!(["/roles/~2/tools"])),
7279            ),
7280            (
7281                "unknown_capabilities_field".to_string(),
7282                "capabilities.future".to_string(),
7283                "<array>".to_string(),
7284                json!({ "future": [] }),
7285                None,
7286            ),
7287        ]);
7288
7289        for (index, (name, field, value, capabilities, runtime_computed)) in
7290            cases.into_iter().enumerate()
7291        {
7292            let registry = Arc::new(Registry::default());
7293            let handler = ControlHandler::new(Arc::clone(&registry));
7294            let response = handler
7295                .handle_control(
7296                    ConnectionId::new((index + 1) as u64),
7297                    capability_grammar_hello_frame(
7298                        capabilities,
7299                        runtime_computed,
7300                        index as u64 + 1,
7301                    ),
7302                )
7303                .expect("invalid HELLO returns a refusal");
7304
7305            assert_eq!(response.len(), 1, "{name} must emit one refusal");
7306            let error = parse_error(&response[0]);
7307            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7308            let message = error["message"]
7309                .as_str()
7310                .expect("error message is a string");
7311            assert!(
7312                message.contains(&field),
7313                "{name}: field missing from {message}"
7314            );
7315            assert!(
7316                message.contains(&value),
7317                "{name}: value missing from {message}"
7318            );
7319            assert_eq!(
7320                registry
7321                    .active_registration_count()
7322                    .expect("registry reads"),
7323                0,
7324                "{name}: refused HELLO must not create a catalog entry"
7325            );
7326        }
7327    }
7328
7329    #[test]
7330    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7331        let registry = Arc::new(Registry::default());
7332        let handler = ControlHandler::new(Arc::clone(&registry));
7333        let capabilities = json!({
7334            "provides": ["credentials-provider/v1"],
7335            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7336            "must_never_reach": ["federation-transport/v1"]
7337        });
7338        let response = handler
7339            .handle_control(
7340                ConnectionId::new(99),
7341                capability_grammar_hello_frame(
7342                    capabilities.clone(),
7343                    Some(json!(["/roles/0/tools"])),
7344                    99,
7345                ),
7346            )
7347            .expect("valid HELLO registers");
7348        assert_eq!(response[0].header.ty, FrameType::HelloAck);
7349
7350        let request = Frame::build(
7351            FrameType::Request,
7352            control_flags(),
7353            0,
7354            0,
7355            100,
7356            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7357                .expect("catalog request serializes"),
7358        )
7359        .expect("catalog request frame builds");
7360        let response = handler
7361            .handle_catalog_list(request, None)
7362            .expect("catalog list succeeds");
7363        let ClientControlResponse::CatalogList { modules, .. } =
7364            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7365        else {
7366            panic!("catalog request must return catalog.list");
7367        };
7368        assert_eq!(modules.len(), 1);
7369        assert_eq!(
7370            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7371            capabilities
7372        );
7373    }
7374
7375    #[test]
7376    fn catalog_list_mirrors_management_operation_description() {
7377        let registry = Arc::new(Registry::default());
7378        let handler = ControlHandler::new(Arc::clone(&registry));
7379        let description = "List managed records and return their identifiers and metadata.";
7380        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7381        manifest.provides = vec![ProviderRole::ManagementSurface {
7382            operations: vec![ManagementOperation {
7383                name: "records.list".to_string(),
7384                kind: ManagementOperationKind::Query,
7385                description: Some(description.to_string()),
7386            }],
7387            config_schema: json!({"type": "object"}),
7388            observability: vec![ObservabilitySurface {
7389                name: "records.stats".to_string(),
7390                kind: ObservabilityKind::Snapshot,
7391            }],
7392            identity_scope: vec![IdentityScope::Project],
7393            concurrency: Concurrency::ModuleManaged,
7394        }];
7395        registry
7396            .register_with_control_ops(
7397                manifest,
7398                PROTOCOL_VERSION,
7399                ConnectionId::new(99),
7400                Vec::new(),
7401            )
7402            .expect("described management manifest registers");
7403
7404        let request = Frame::build(
7405            FrameType::Request,
7406            control_flags(),
7407            0,
7408            0,
7409            100,
7410            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7411                .expect("catalog request serializes"),
7412        )
7413        .expect("catalog request frame builds");
7414        let response = handler
7415            .handle_catalog_list(request, None)
7416            .expect("catalog list succeeds");
7417        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7418        assert_eq!(
7419            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7420            "catalog.list must preserve the declared operation description verbatim"
7421        );
7422    }
7423
7424    #[test]
7425    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7426        let registry = Arc::new(Registry::default());
7427        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7428            [("vault".to_string(), true), ("squatter".to_string(), true)],
7429            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7430        );
7431        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7432        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7433            provides: vec!["credentials-provider/v1".to_string()],
7434            requires: Vec::new(),
7435            must_never_reach: Vec::new(),
7436        });
7437        let frame = Frame::build(
7438            FrameType::Hello,
7439            control_flags(),
7440            0,
7441            0,
7442            77,
7443            serde_json::to_vec(&ModuleHelloBody {
7444                manifest: squatter,
7445                protocol_ver: PROTOCOL_VERSION,
7446                control_ops: None,
7447                launch_nonce: None,
7448            })
7449            .expect("HELLO serializes"),
7450        )
7451        .expect("HELLO frame builds");
7452        let response = handler
7453            .handle_control(ConnectionId::new(77), frame)
7454            .expect("reserved claim receives a typed refusal");
7455        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7456        assert_eq!(
7457            registry
7458                .active_registration_count()
7459                .expect("registry reads"),
7460            0,
7461            "a reserved capability refusal must not leave a catalog entry"
7462        );
7463    }
7464
7465    #[test]
7466    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7467        for settlement in ["timeout", "inconclusive", "drop"] {
7468            let breakers = RouteBindBreakers::default();
7469            let RouteBindAdmission::Admitted {
7470                guard: mut old,
7471                probe: false,
7472            } = breakers.admit("prov")
7473            else {
7474                panic!("ordinary relay admitted")
7475            };
7476            let RouteBindAdmission::Admitted {
7477                guard: mut opener, ..
7478            } = breakers.admit("prov")
7479            else {
7480                panic!("second relay admitted")
7481            };
7482            assert!(
7483                !opener
7484                    .record_timeout(1, Duration::ZERO)
7485                    .unwrap()
7486                    .reopened_after_probe
7487            );
7488            let RouteBindAdmission::Admitted {
7489                guard: mut probe,
7490                probe: true,
7491            } = breakers.admit("prov")
7492            else {
7493                panic!("one cooldown probe admitted")
7494            };
7495            match settlement {
7496                "timeout" => assert!(
7497                    !old.record_timeout(1, Duration::ZERO)
7498                        .unwrap()
7499                        .reopened_after_probe
7500                ),
7501                "inconclusive" => old.record_inconclusive(),
7502                "drop" => drop(old),
7503                _ => unreachable!(),
7504            }
7505            assert!(
7506                matches!(
7507                    breakers.admit("prov"),
7508                    RouteBindAdmission::Refused {
7509                        probe_in_flight: true,
7510                        ..
7511                    }
7512                ),
7513                "{settlement} of a pre-open relay cannot release the real probe"
7514            );
7515            assert!(
7516                probe
7517                    .record_timeout(1, Duration::ZERO)
7518                    .unwrap()
7519                    .reopened_after_probe
7520            );
7521            assert!(matches!(
7522                breakers.admit("prov"),
7523                RouteBindAdmission::Admitted { probe: true, .. }
7524            ));
7525        }
7526        let breakers = RouteBindBreakers::default();
7527        let admit = || match breakers.admit("prov") {
7528            RouteBindAdmission::Admitted { guard, .. } => guard,
7529            _ => panic!("relay admitted"),
7530        };
7531        admit().record_timeout(1, Duration::ZERO);
7532        let mut old_probe = admit();
7533        breakers.reset_for_new_module_connection("prov");
7534        admit().record_timeout(1, Duration::ZERO);
7535        let _new_probe = admit();
7536        old_probe.record_inconclusive();
7537        assert!(matches!(
7538            breakers.admit("prov"),
7539            RouteBindAdmission::Refused {
7540                probe_in_flight: true,
7541                ..
7542            }
7543        ));
7544    }
7545
7546    #[tokio::test]
7547    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7548        for candidate in [false, true] {
7549            let registry = Arc::new(Registry::default());
7550            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7551                [("vault".to_string(), true), ("squatter".to_string(), true)],
7552                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7553            );
7554            let conn = ConnectionId::new(77);
7555            let (ctx, mut rx) = route_ctx(conn);
7556            let initial = capability_manifest("squatter", &[], &[]);
7557            if candidate {
7558                registry
7559                    .register_candidate_with_control_ops(
7560                        initial.clone(),
7561                        PROTOCOL_VERSION,
7562                        conn,
7563                        module_baseline_control_ops(),
7564                    )
7565                    .unwrap();
7566                handler
7567                    .forwarding
7568                    .register_candidate_module_connection(
7569                        conn,
7570                        "squatter".to_string(),
7571                        PROTOCOL_VERSION,
7572                        manifest_concurrency(&initial),
7573                        ctx.egress.clone(),
7574                    )
7575                    .unwrap();
7576            } else {
7577                hello_via_sink(
7578                    &handler,
7579                    &ctx,
7580                    &mut rx,
7581                    hello_frame_with_manifest(initial.clone(), 1),
7582                )
7583                .await;
7584            }
7585            let response = handler
7586                .handle_control_frame(
7587                    &ctx,
7588                    catalog_update_with_capabilities_frame(
7589                        2,
7590                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
7591                            .capabilities
7592                            .unwrap(),
7593                    ),
7594                )
7595                .await
7596                .unwrap();
7597            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7598            assert_eq!(
7599                registry
7600                    .get_module_by_connection(conn)
7601                    .unwrap()
7602                    .unwrap()
7603                    .manifest,
7604                initial
7605            );
7606        }
7607    }
7608
7609    #[test]
7610    fn server_describe_surfaces_required_capability_verdict_fields() {
7611        let registry = Arc::new(Registry::default());
7612        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7613            [
7614                ("consumer".to_string(), true),
7615                ("provider".to_string(), false),
7616            ],
7617            BTreeMap::new(),
7618        );
7619        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7620        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7621            provides: Vec::new(),
7622            requires: vec![subc_protocol::manifest::CapabilityRequirement {
7623                capability: "credentials-provider/v1".to_string(),
7624                need: subc_protocol::manifest::CapabilityNeed::Required,
7625            }],
7626            must_never_reach: Vec::new(),
7627        });
7628        let hello = Frame::build(
7629            FrameType::Hello,
7630            control_flags(),
7631            0,
7632            0,
7633            78,
7634            serde_json::to_vec(&ModuleHelloBody {
7635                manifest: consumer,
7636                protocol_ver: PROTOCOL_VERSION,
7637                control_ops: None,
7638                launch_nonce: None,
7639            })
7640            .expect("HELLO serializes"),
7641        )
7642        .expect("HELLO frame builds");
7643        handler
7644            .handle_control(ConnectionId::new(78), hello)
7645            .expect("consumer registers");
7646        let describe = Frame::build(
7647            FrameType::Request,
7648            control_flags(),
7649            0,
7650            0,
7651            79,
7652            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7653                .expect("request serializes"),
7654        )
7655        .expect("describe frame builds");
7656        let response = handler
7657            .handle_server_describe(describe)
7658            .expect("server.describe succeeds");
7659        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7660        let requirement = &rendered["capability_requirements"][0];
7661        assert_eq!(requirement["consumer"], "consumer");
7662        assert_eq!(requirement["verdict"], "never_provided");
7663        assert_eq!(requirement["episode_seq"], 1);
7664        assert_eq!(requirement["config_satisfiable"], false);
7665        assert_eq!(requirement["runtime_available"], false);
7666        assert!(requirement["detail"]
7667            .as_str()
7668            .expect("detail string")
7669            .contains("credentials-provider/v1"));
7670    }
7671
7672    #[test]
7673    fn catalog_list_omits_capabilities_for_legacy_manifest() {
7674        let registry = Arc::new(Registry::default());
7675        let handler = ControlHandler::new(Arc::clone(&registry));
7676        let hello = handler
7677            .handle_control(
7678                ConnectionId::new(101),
7679                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7680            )
7681            .expect("legacy HELLO registers");
7682        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7683
7684        let request = Frame::build(
7685            FrameType::Request,
7686            control_flags(),
7687            0,
7688            0,
7689            102,
7690            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7691                .expect("catalog request serializes"),
7692        )
7693        .expect("catalog request frame builds");
7694        let response = handler
7695            .handle_catalog_list(request, None)
7696            .expect("catalog list succeeds");
7697        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7698        assert!(
7699            body["modules"][0].get("capabilities").is_none(),
7700            "legacy manifest must retain an absent capabilities field on catalog.list"
7701        );
7702    }
7703
7704    #[test]
7705    fn hello_ack_omits_storage_when_no_storage_config() {
7706        let registry = Arc::new(Registry::default());
7707        let handler = ControlHandler::new(Arc::clone(&registry));
7708        let responses = handler
7709            .handle_control(
7710                ConnectionId::new(1),
7711                hello_frame("aft", PROTOCOL_VERSION, 7),
7712            )
7713            .unwrap();
7714        let ack = parse_ack(&responses[0]);
7715        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7716        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7717    }
7718
7719    #[tokio::test]
7720    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7721        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7722        let registry = Arc::new(Registry::default());
7723        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
7724        let responses = handler
7725            .handle_control(
7726                ConnectionId::new(1),
7727                hello_frame("aft", PROTOCOL_VERSION, 7),
7728            )
7729            .unwrap();
7730        let ack = parse_ack(&responses[0]);
7731        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7732
7733        let described = handler
7734            .handle_control_frame(
7735                &route_ctx(ConnectionId::new(2)).0,
7736                Frame::build(
7737                    FrameType::Request,
7738                    control_flags(),
7739                    0,
7740                    0,
7741                    9,
7742                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7743                )
7744                .unwrap(),
7745            )
7746            .await
7747            .unwrap();
7748        let ClientControlResponse::ServerDescribe { machine_id, .. } =
7749            serde_json::from_slice(&described[0].body).unwrap()
7750        else {
7751            panic!("server.describe answered with another shape");
7752        };
7753        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7754    }
7755
7756    #[test]
7757    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7758        // With a central sqlite storage policy, each registering module gets its
7759        // own resolved descriptor in HELLO_ACK, keyed by its module id.
7760        let registry = Arc::new(Registry::default());
7761        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
7762            crate::daemon_config::StorageConfig::Sqlite {
7763                data_home: std::path::PathBuf::from("/data"),
7764            },
7765        ));
7766
7767        let responses = handler
7768            .handle_control(
7769                ConnectionId::new(1),
7770                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7771            )
7772            .unwrap();
7773        let ack = parse_ack(&responses[0]);
7774        assert_eq!(
7775            ack.storage,
7776            Some(serde_json::json!({
7777                "module_id": "alfonso-routing",
7778                "storage_namespace": "default",
7779                "isolation": { "kind": "module" },
7780                "backend": {
7781                    "backend": "sqlite",
7782                    "path": "/data/cortexkit/alfonso-routing/store.db"
7783                }
7784            })),
7785            "the delivered descriptor is the module's own sqlite store path"
7786        );
7787    }
7788
7789    #[test]
7790    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7791        let registry = Arc::new(Registry::default());
7792        let handler = ControlHandler::new(Arc::clone(&registry));
7793        let conn = ConnectionId::new(1);
7794        let responses = handler
7795            .handle_control(
7796                conn,
7797                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7798            )
7799            .unwrap();
7800        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7801        let registration = registry.get_module("aft").unwrap().unwrap();
7802        assert_eq!(registration.control_ops, module_baseline_control_ops());
7803
7804        let frame =
7805            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7806        assert!(handler
7807            .guard_module_control_op(&frame, "aft", "route.bind")
7808            .unwrap()
7809            .is_none());
7810        let error = handler
7811            .guard_module_control_op(&frame, "aft", "test.synthetic")
7812            .unwrap()
7813            .expect("synthetic ungranted op should be rejected");
7814        assert_eq!(error.header.ty, FrameType::Error);
7815        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7816    }
7817
7818    #[test]
7819    fn hello_control_ops_some_adds_optional_grants() {
7820        let registry = Arc::new(Registry::default());
7821        let handler = ControlHandler::new(Arc::clone(&registry));
7822        handler
7823            .handle_control(
7824                ConnectionId::new(1),
7825                hello_frame_with_control_ops(
7826                    "aft",
7827                    PROTOCOL_VERSION,
7828                    7,
7829                    Some(vec![
7830                        "future.synthetic".to_string(),
7831                        "route.bind".to_string(),
7832                    ]),
7833                ),
7834            )
7835            .unwrap();
7836        let registration = registry.get_module("aft").unwrap().unwrap();
7837        assert_eq!(
7838            registration.control_ops,
7839            vec![
7840                "route.bind".to_string(),
7841                "route.status".to_string(),
7842                "future.synthetic".to_string(),
7843            ]
7844        );
7845        let frame =
7846            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7847        assert!(handler
7848            .guard_module_control_op(&frame, "aft", "future.synthetic")
7849            .unwrap()
7850            .is_none());
7851    }
7852
7853    #[tokio::test]
7854    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7855        let registry = Arc::new(Registry::default());
7856        let forwarding = Arc::new(ForwardingTable::default());
7857        let handler =
7858            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7859        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7860        hello_via_sink(
7861            &handler,
7862            &module_ctx,
7863            &mut module_rx,
7864            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7865        )
7866        .await;
7867
7868        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7869        let responses = handler
7870            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7871            .await
7872            .unwrap();
7873        assert_eq!(responses.len(), 1);
7874        assert_eq!(responses[0].header.ty, FrameType::Error);
7875        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7876        assert!(module_rx.try_recv().is_err());
7877    }
7878
7879    #[tokio::test]
7880    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7881        let registry = Arc::new(Registry::default());
7882        let forwarding = Arc::new(ForwardingTable::default());
7883        let handler =
7884            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7885        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7886        hello_via_sink(
7887            &handler,
7888            &module_ctx,
7889            &mut module_rx,
7890            hello_frame_with_control_ops(
7891                "aft",
7892                PROTOCOL_VERSION,
7893                7,
7894                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7895            ),
7896        )
7897        .await;
7898
7899        let project_root = unique_project_root("demux");
7900        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7901        let route_handler = handler.clone();
7902        let route_task = tokio::spawn(async move {
7903            route_handler
7904                .handle_control_frame(
7905                    &route_client_ctx,
7906                    route_open_frame(100, "aft", project_root),
7907                )
7908                .await
7909                .unwrap()
7910        });
7911        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7912            .await
7913            .unwrap()
7914            .unwrap();
7915        assert!(matches!(
7916            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7917            ModuleControlRequest::RouteBind { .. }
7918        ));
7919
7920        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7921        let health_handler = handler.clone();
7922        let health_task = tokio::spawn(async move {
7923            health_handler
7924                .handle_control_frame(
7925                    &health_client_ctx,
7926                    supervisor_health_probe_frame(101, "aft"),
7927                )
7928                .await
7929                .unwrap()
7930        });
7931        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7932            .await
7933            .unwrap()
7934            .unwrap();
7935        assert_eq!(
7936            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7937            ModuleControlRequest::HealthCheck {}
7938        );
7939
7940        handler
7941            .handle_control_frame(
7942                &module_ctx,
7943                health_response(health_frame.header.corr, HealthStatus::Degraded),
7944            )
7945            .await
7946            .unwrap();
7947        let health_response = health_task.await.unwrap();
7948        assert_eq!(health_response.len(), 1);
7949        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7950            ClientControlResponse::SupervisorHealthProbe {
7951                module_id,
7952                status,
7953                detail,
7954                metrics,
7955            } => {
7956                assert_eq!(module_id, "aft");
7957                assert_eq!(status, HealthStatus::Degraded);
7958                assert_eq!(detail.as_deref(), Some("warming"));
7959                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7960            }
7961            other => panic!("unexpected health response: {other:?}"),
7962        }
7963
7964        handler
7965            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7966            .await
7967            .unwrap();
7968        let route_response = route_task.await.unwrap();
7969        assert!(route_response.is_empty());
7970        let published = route_client_rx.recv().await.unwrap();
7971        assert!(matches!(
7972            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7973            ClientControlResponse::RouteOpen { .. }
7974        ));
7975    }
7976
7977    /// Start one `route.open` on `client_connection` and return its still-running
7978    /// handler task together with the `route.bind` the module received for it.
7979    /// The handler blocks until the module answers, so it has to run as a task
7980    /// while the test drives the module side.
7981    async fn relay_route_open(
7982        handler: &ControlHandler,
7983        client_connection: ConnectionId,
7984        client_egress: &FrameSink,
7985        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7986        corr: u64,
7987        module_id: &str,
7988        project_root_label: &str,
7989    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7990        let ctx = RouteCtx {
7991            connection_id: client_connection,
7992            egress: client_egress.clone(),
7993        };
7994        let handler = handler.clone();
7995        let project_root = unique_project_root(project_root_label);
7996        let module_id = module_id.to_string();
7997        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7998        let task = tokio::spawn(async move {
7999            let _guard = tracing::dispatcher::set_default(&dispatch);
8000            handler
8001                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8002                .await
8003                .unwrap()
8004        });
8005        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8006            .await
8007            .expect("module receives the relayed route.bind")
8008            .expect("module egress is open");
8009        (task, bind.frame)
8010    }
8011
8012    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8013        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8014            ModuleControlRequest::RouteBind {
8015                route_channel,
8016                epoch,
8017                ..
8018            } => (route_channel, epoch),
8019            other => panic!("expected a route.bind request, got {other:?}"),
8020        }
8021    }
8022
8023    fn published_route(frame: &Frame) -> (u16, u32) {
8024        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8025            ClientControlResponse::RouteOpen {
8026                route_channel,
8027                route_epoch,
8028            } => (route_channel, route_epoch),
8029            other => panic!("expected a route.open response, got {other:?}"),
8030        }
8031    }
8032
8033    /// Reproduction of a production outage. A client had `route.open`s in
8034    /// flight to a module and was already marked closing -- its egress had refused a
8035    /// module frame, so the daemon asked its connection to end -- while its sink
8036    /// was still open. When the module acked those binds, the daemon refused to
8037    /// commit a route for a closing client, and that refusal was returned from
8038    /// the MODULE connection's frame handler, where a router error that has no
8039    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
8040    /// the supervisor correctly did not respawn a clean exit, and every seat lost
8041    /// its tools for hours -- one client's teardown took down a connection
8042    /// carrying ~170 other routes.
8043    ///
8044    /// The window is opened here by calling the production path that opens it
8045    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
8046    /// state that matters is "in `closing_connections`, sink still open, relay
8047    /// still pending", and it lasts only from the close request until the
8048    /// connection loop reacts to it; a socket-level test can flood a client into
8049    /// that escalation but cannot pin the module's ack inside the window. Closing
8050    /// the socket instead takes the other path entirely -- connection teardown
8051    /// removes the pending relay under the same lock, so the ack finds nothing.
8052    #[tokio::test]
8053    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8054        let registry = Arc::new(Registry::default());
8055        let forwarding = Arc::new(ForwardingTable::default());
8056        let handler =
8057            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8058
8059        let module_connection = ConnectionId::new(30);
8060        let (module_ctx, mut module_rx) = route_ctx(module_connection);
8061        hello_via_sink(
8062            &handler,
8063            &module_ctx,
8064            &mut module_rx,
8065            hello_frame("aft", PROTOCOL_VERSION, 7),
8066        )
8067        .await;
8068
8069        let dying_client = ConnectionId::new(31);
8070        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8071
8072        // A published route on the dying client. The escalation below only marks
8073        // a connection closing for a route it has already published.
8074        let (first_task, first_bind) = relay_route_open(
8075            &handler,
8076            dying_client,
8077            &dying_ctx.egress,
8078            &mut module_rx,
8079            100,
8080            "aft",
8081            "closing-first",
8082        )
8083        .await;
8084        handler
8085            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8086            .await
8087            .unwrap();
8088        assert!(first_task.await.unwrap().is_empty());
8089        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8090
8091        // A second route.open from the same client, relayed and awaiting its ack.
8092        let (second_task, second_bind) = relay_route_open(
8093            &handler,
8094            dying_client,
8095            &dying_ctx.egress,
8096            &mut module_rx,
8097            101,
8098            "aft",
8099            "closing-second",
8100        )
8101        .await;
8102        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8103
8104        // The window: the client is closing, its sink is still open, and its
8105        // second bind is still pending.
8106        assert!(forwarding
8107            .escalate_client_delivery_failure(
8108                dying_client,
8109                first_channel,
8110                first_epoch,
8111                CloseReason::new(
8112                    "module_to_client_delivery_failed",
8113                    "client egress refused a module frame",
8114                ),
8115                crate::forwarding::UndeliveredFrame {
8116                    module_id: None,
8117                    sink: &dying_ctx.egress,
8118                },
8119            )
8120            .unwrap());
8121        assert!(!dying_ctx.egress.is_closed());
8122
8123        // The frame that used to end the module connection.
8124        let ack = handler
8125            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8126            .await;
8127        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8128        if module_loop_error.is_some() {
8129            // What the server's connection loop does with a router error that has
8130            // no ERROR-frame translation: end the connection, which releases the
8131            // module's registration and every route on it.
8132            handler.cleanup_connection(module_connection).unwrap();
8133        }
8134        // Read the module's next frame before opening the co-tenant's route, so
8135        // the GOODBYE assertion below is about THIS ack and not about later
8136        // traffic. `None` means the module was told nothing.
8137        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8138            .await
8139            .ok()
8140            .flatten();
8141
8142        // 1. The module connection is still registered.
8143        assert!(
8144            registry
8145                .get_module_by_connection(module_connection)
8146                .unwrap()
8147                .is_some(),
8148            "one client's closing connection ended the shared module connection: \
8149             {module_loop_error:?}"
8150        );
8151        // ...and still serving: another client can open and use a route on it.
8152        let cotenant = ConnectionId::new(32);
8153        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8154        let (cotenant_task, cotenant_bind) = relay_route_open(
8155            &handler,
8156            cotenant,
8157            &cotenant_ctx.egress,
8158            &mut module_rx,
8159            102,
8160            "aft",
8161            "closing-cotenant",
8162        )
8163        .await;
8164        handler
8165            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8166            .await
8167            .unwrap();
8168        assert!(cotenant_task.await.unwrap().is_empty());
8169        let (cotenant_channel, cotenant_epoch) =
8170            published_route(&cotenant_rx.recv().await.unwrap());
8171        assert!(matches!(
8172            forwarding
8173                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8174                .unwrap(),
8175            DataRoute::Client(DataRouteState::Bound(_))
8176        ));
8177
8178        // 2. The module was told to drop the binding it created for the route
8179        //    that will never be published.
8180        let goodbye = post_ack_module_frame
8181            .expect("module receives a GOODBYE for the abandoned route channel");
8182        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8183        assert_eq!(goodbye.header.channel, abandoned_channel);
8184        assert_eq!(goodbye.header.epoch, abandoned_epoch);
8185
8186        // 3. The dying client received nothing: no route was ever published to
8187        //    it. Its route.open is answered as unavailable, which the connection
8188        //    loop would write to a socket that is already going away.
8189        assert!(dying_rx.try_recv().is_err());
8190        let second_response = second_task.await.unwrap();
8191        assert_eq!(second_response.len(), 1);
8192        assert_eq!(
8193            parse_error(&second_response[0])["code"],
8194            "target_unavailable"
8195        );
8196    }
8197
8198    /// The fence at the module-loop boundary, stated as its own contract: which
8199    /// forwarding failures are allowed to end the module connection that is being
8200    /// served. A `ConnectionClosing` naming some client is about that client, and
8201    /// a module connection is shared; the same error naming the module's own
8202    /// connection is about this connection and must stay fatal, as must failures
8203    /// that are about the forwarding table itself.
8204    #[test]
8205    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8206        let handler = ControlHandler::default();
8207        let module_connection = ConnectionId::new(30);
8208        let client_connection = ConnectionId::new(31);
8209
8210        handler
8211            .refuse_to_end_module_connection_for_a_client(
8212                module_connection,
8213                77,
8214                ForwardingError::ConnectionClosing {
8215                    connection_id: client_connection,
8216                },
8217            )
8218            .expect("a closing client must never end the module connection");
8219
8220        assert!(matches!(
8221            handler.refuse_to_end_module_connection_for_a_client(
8222                module_connection,
8223                78,
8224                ForwardingError::ConnectionClosing {
8225                    connection_id: module_connection,
8226                },
8227            ),
8228            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8229                connection_id
8230            })) if connection_id == module_connection
8231        ));
8232        assert!(matches!(
8233            handler.refuse_to_end_module_connection_for_a_client(
8234                module_connection,
8235                79,
8236                ForwardingError::Poisoned,
8237            ),
8238            Err(RouterError::Forwarding(ForwardingError::Poisoned))
8239        ));
8240        assert!(matches!(
8241            handler.refuse_to_end_module_connection_for_a_client(
8242                module_connection,
8243                80,
8244                ForwardingError::StaleModuleEndpoint,
8245            ),
8246            Err(RouterError::Forwarding(
8247                ForwardingError::StaleModuleEndpoint
8248            ))
8249        ));
8250    }
8251
8252    /// The spawn-attestation guard is what stops a connected module from claiming
8253    /// another module's identity and being stamped `Reserved` for it. Every other
8254    /// test that supplies a consumer_identity supplies a CORRECT one, because a
8255    /// correct one is what the rest of the flow needs -- so the guard's rejection
8256    /// branch was never the subject of an assertion, only its acceptance branch.
8257    ///
8258    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
8259    /// whole subc-core library suite green; only the forwarding integration tests
8260    /// notice, and they notice for unrelated reasons. This test exists so the
8261    /// refusal itself is asserted where the guard lives: it fails if the identity
8262    /// check stops refusing, which is the direction that matters, since a guard
8263    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
8264    #[tokio::test]
8265    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8266        let registry = Arc::new(Registry::default());
8267        let forwarding = Arc::new(ForwardingTable::default());
8268        let supervisor = SupervisorHandle::new();
8269        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8270        let handler =
8271            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8272                .with_supervisor(supervisor);
8273
8274        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8275        hello_via_sink(
8276            &handler,
8277            &target_ctx,
8278            &mut target_rx,
8279            hello_frame("target", PROTOCOL_VERSION, 1),
8280        )
8281        .await;
8282
8283        // A real supervised module id presenting the wrong nonce. This is the
8284        // impersonation case: the attacker knows a privileged module_id, which is
8285        // public, and guesses at the nonce, which is not.
8286        let wrong_nonce = handler
8287            .handle_control_frame(
8288                &route_ctx(ConnectionId::new(91)).0,
8289                route_open_frame_with_admission_facts(
8290                    20,
8291                    "target",
8292                    unique_project_root("admission-facts"),
8293                    Some(subc_control::ConsumerIdentity {
8294                        module_id: "fed".to_string(),
8295                        launch_nonce: "not-the-real-nonce".to_string(),
8296                    }),
8297                    None,
8298                ),
8299            )
8300            .await
8301            .unwrap();
8302        assert_eq!(
8303            parse_error(&wrong_nonce[0])["code"],
8304            "bad_consumer_identity",
8305            "a mismatched launch nonce must be refused, not stamped Reserved"
8306        );
8307
8308        // A module id the supervisor never spawned at all, so no nonce exists to
8309        // compare against. An implementation that treats "no record" as "nothing
8310        // to check" fails open here while passing the case above.
8311        let never_spawned = handler
8312            .handle_control_frame(
8313                &route_ctx(ConnectionId::new(92)).0,
8314                route_open_frame_with_admission_facts(
8315                    21,
8316                    "target",
8317                    unique_project_root("admission-facts"),
8318                    Some(subc_control::ConsumerIdentity {
8319                        module_id: "never-spawned".to_string(),
8320                        launch_nonce: "any-nonce".to_string(),
8321                    }),
8322                    None,
8323                ),
8324            )
8325            .await
8326            .unwrap();
8327        assert_eq!(
8328            parse_error(&never_spawned[0])["code"],
8329            "bad_consumer_identity",
8330            "an unspawned module_id must be refused rather than accepted for lack of a record"
8331        );
8332    }
8333
8334    /// The refusal test above proves the guard says NO. Nothing proved it can say
8335    /// YES, and the difference is not academic: replacing the whole authorization
8336    /// with `false` -- admitting no consumer identity at all, revoking Reserved
8337    /// standing for every supervised module in the fleet -- leaves 110 of the 111
8338    /// library tests GREEN. The one that notices does so by HANGING, because it
8339    /// waits for a bind that can no longer happen.
8340    ///
8341    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
8342    /// or flaky test, invites a RETRY rather than an investigation, and the retry
8343    /// hangs too and gets blamed on the runner. So a total revocation of the
8344    /// daemon's trust grant would have shipped behind a symptom nobody attributes
8345    /// to code.
8346    ///
8347    /// The bias is structural rather than accidental. A REFUSAL looks like a
8348    /// failure someone writes a test for; a GRANT looks like the happy path. Every
8349    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
8350    /// suite for that reason, and this one is the purest case in the daemon.
8351    ///
8352    /// This test asserts the EFFECT rather than the absence of an error: the module
8353    /// receives a RouteBind and it carries `Reserved` naming the attested module.
8354    /// A guard that admitted nobody would produce no bind at all; one that admitted
8355    /// everybody would stamp the wrong principal, which the refusal test catches.
8356    #[tokio::test]
8357    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8358        let registry = Arc::new(Registry::default());
8359        let forwarding = Arc::new(ForwardingTable::default());
8360        let supervisor = SupervisorHandle::new();
8361        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8362        let handler =
8363            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8364                .with_supervisor(supervisor);
8365
8366        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8367        hello_via_sink(
8368            &handler,
8369            &target_ctx,
8370            &mut target_rx,
8371            hello_frame("target", PROTOCOL_VERSION, 1),
8372        )
8373        .await;
8374
8375        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8376        let route_handler = handler.clone();
8377        let route_task = tokio::spawn(async move {
8378            route_handler
8379                .handle_control_frame(
8380                    &client_ctx,
8381                    route_open_frame_with_admission_facts(
8382                        30,
8383                        "target",
8384                        unique_project_root("admission-facts"),
8385                        Some(subc_control::ConsumerIdentity {
8386                            module_id: "fed".to_string(),
8387                            launch_nonce: "fed-nonce".to_string(),
8388                        }),
8389                        None,
8390                    ),
8391                )
8392                .await
8393                .unwrap()
8394        });
8395
8396        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
8397        // the very mutation it exists to catch -- a guard that admits nobody -- no
8398        // bind is ever sent, so it HUNG rather than failing. That reproduces the
8399        // exact defect being fixed: a total revocation detected only as a stalled
8400        // suite, which reads as flakiness and invites a retry. An acceptance test
8401        // that waits for an effect must bound the wait, or a red becomes a hang.
8402        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8403            .await
8404            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8405            .expect("module control channel closed before route.bind");
8406        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8407        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8408            panic!("expected route.bind")
8409        };
8410        assert_eq!(
8411            principal,
8412            Some(Principal::Reserved {
8413                module_id: "fed".to_string()
8414            }),
8415            "a correctly attested consumer must be stamped Reserved for its own id"
8416        );
8417
8418        handler
8419            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8420            .await
8421            .unwrap();
8422        assert!(route_task.await.unwrap().is_empty());
8423        assert!(
8424            matches!(
8425                serde_json::from_slice::<ClientControlResponse>(
8426                    &client_rx.recv().await.unwrap().body
8427                )
8428                .unwrap(),
8429                ClientControlResponse::RouteOpen { .. }
8430            ),
8431            "the route must actually open, not merely avoid an error"
8432        );
8433    }
8434
8435    #[tokio::test(start_paused = true)]
8436    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8437        let registry = Arc::new(Registry::default());
8438        let forwarding = Arc::new(ForwardingTable::default());
8439        let supervisor = SupervisorHandle::new();
8440        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8441        let handler =
8442            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8443                .with_supervisor(supervisor);
8444
8445        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8446        hello_via_sink(
8447            &handler,
8448            &target_ctx,
8449            &mut target_rx,
8450            hello_frame("target", PROTOCOL_VERSION, 1),
8451        )
8452        .await;
8453
8454        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8455        let direct_handler = handler.clone();
8456        let direct_open = tokio::spawn(async move {
8457            direct_handler
8458                .handle_control_frame(
8459                    &direct_ctx,
8460                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
8461                )
8462                .await
8463                .unwrap()
8464        });
8465        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8466            .await
8467            .expect("no direct route.bind within 5s")
8468            .expect("target control channel closed before direct route.bind");
8469        handler
8470            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8471            .await
8472            .unwrap();
8473        assert!(direct_open.await.unwrap().is_empty());
8474        let _ = direct_rx.recv().await.unwrap();
8475
8476        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8477        let reserved_handler = handler.clone();
8478        let reserved_open = tokio::spawn(async move {
8479            reserved_handler
8480                .handle_control_frame(
8481                    &reserved_ctx,
8482                    route_open_frame_with_admission_facts(
8483                        3,
8484                        "target",
8485                        unique_project_root("admission-facts"),
8486                        Some(ConsumerIdentity {
8487                            module_id: "fed".to_string(),
8488                            launch_nonce: "fed-nonce".to_string(),
8489                        }),
8490                        None,
8491                    ),
8492                )
8493                .await
8494                .unwrap()
8495        });
8496        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8497            .await
8498            .expect("no reserved route.bind within 5s")
8499            .expect("target control channel closed before reserved route.bind");
8500        handler
8501            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8502            .await
8503            .unwrap();
8504        assert!(reserved_open.await.unwrap().is_empty());
8505        let _ = reserved_rx.recv().await.unwrap();
8506
8507        forwarding
8508            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8509            .unwrap();
8510        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8511        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8512            module_id: Some("target".to_string()),
8513        })
8514        .unwrap();
8515        let census_frame =
8516            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8517        let response = handler
8518            .handle_control_frame(&census_ctx, census_frame)
8519            .await
8520            .unwrap()
8521            .pop()
8522            .unwrap();
8523        let actual: Value = serde_json::from_slice(&response.body).unwrap();
8524        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8525        assert!(matches!(
8526            decoded,
8527            ClientControlResponse::SupervisorRoutes { .. }
8528        ));
8529        let routes = actual["modules"][0]["routes"].as_array().unwrap();
8530        assert_eq!(routes.len(), 2);
8531        assert!(routes.iter().all(|route| route["draining"] == true));
8532        // The census carries WHY: the reason the drain was begun with, in the
8533        // route.closing vocabulary, on every draining route this drain marked.
8534        assert!(
8535            routes.iter().all(|route| route["drain_reason"] == "reload"),
8536            "draining routes must name the drain's reason: {routes:?}"
8537        );
8538        assert!(routes.iter().any(|route| {
8539            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8540        }));
8541        assert!(routes.iter().any(|route| {
8542            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8543        }));
8544
8545        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8546            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8547        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8548            std::fs::write(
8549                &golden_path,
8550                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8551            )
8552            .unwrap();
8553        }
8554        let expected: Value =
8555            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8556        assert_eq!(actual, expected);
8557    }
8558
8559    async fn query_live_roots(
8560        handler: &ControlHandler,
8561        module_ctx: &RouteCtx,
8562    ) -> ModuleControlResponseToModule {
8563        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8564        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8565        let response = handler
8566            .handle_control_frame(module_ctx, frame)
8567            .await
8568            .unwrap()
8569            .pop()
8570            .unwrap();
8571        serde_json::from_slice(&response.body).unwrap()
8572    }
8573
8574    #[tokio::test(start_paused = true)]
8575    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8576        let registry = Arc::new(Registry::default());
8577        let forwarding = Arc::new(ForwardingTable::default());
8578        let handler = ControlHandler::with_forwarding(registry, forwarding);
8579        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8580        hello_via_sink(
8581            &handler,
8582            &target_ctx,
8583            &mut target_rx,
8584            hello_frame("target", PROTOCOL_VERSION, 1),
8585        )
8586        .await;
8587        let root = unique_project_root("live-roots-known");
8588        let path = ProjectRootId::from_path_allowing_missing(root.path())
8589            .unwrap()
8590            .as_path()
8591            .to_path_buf();
8592        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8593        let open_handler = handler.clone();
8594        let opened = tokio::spawn(async move {
8595            open_handler
8596                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8597                .await
8598                .unwrap()
8599        });
8600        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8601            .await
8602            .unwrap()
8603            .unwrap();
8604        handler
8605            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8606            .await
8607            .unwrap();
8608        assert!(opened.await.unwrap().is_empty());
8609        let _ = client_rx.recv().await.unwrap();
8610
8611        let root = unique_project_root("live-roots-pending");
8612        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8613            .unwrap()
8614            .as_path()
8615            .to_path_buf();
8616        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8617        let open_handler = handler.clone();
8618        let pending = tokio::spawn(async move {
8619            open_handler
8620                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8621                .await
8622                .unwrap()
8623        });
8624        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8625            .await
8626            .unwrap()
8627            .unwrap();
8628        let actual = query_live_roots(&handler, &target_ctx).await;
8629        let ModuleControlResponseToModule::LiveRoots {
8630            roots,
8631            unknown_root_bindings,
8632            total_bindings,
8633        } = actual
8634        else {
8635            panic!("expected live roots")
8636        };
8637        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8638        assert_eq!(unknown_root_bindings, 0);
8639        assert_eq!(
8640            roots.len(),
8641            2,
8642            "root-known arm must retain each canonical root"
8643        );
8644        assert_eq!(
8645            total_bindings,
8646            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8647        );
8648        let counts = roots
8649            .iter()
8650            .map(|root| (root.project_root.clone(), root.bound, root.pending))
8651            .collect::<Vec<_>>();
8652        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8653        expected.sort_by(|a, b| a.0.cmp(&b.0));
8654        assert_eq!(
8655            counts, expected,
8656            "roots must sort by path and count pending separately"
8657        );
8658        handler
8659            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8660            .await
8661            .unwrap();
8662        assert!(pending.await.unwrap().is_empty());
8663    }
8664
8665    #[tokio::test(start_paused = true)]
8666    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8667        let forwarding = Arc::new(ForwardingTable::default());
8668        let handler =
8669            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8670        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8671        hello_via_sink(
8672            &handler,
8673            &target_ctx,
8674            &mut target_rx,
8675            hello_frame("target", PROTOCOL_VERSION, 1),
8676        )
8677        .await;
8678        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8679        let pending = forwarding
8680            .begin_route_bind_relay_for_test(
8681                client_ctx.connection_id,
8682                client_ctx.egress.clone(),
8683                2,
8684                "target",
8685            )
8686            .unwrap();
8687        forwarding
8688            .complete_pending_relay(
8689                target_ctx.connection_id,
8690                pending.corr,
8691                RouteBindRelayOutcome::Accepted,
8692            )
8693            .unwrap();
8694        let actual = query_live_roots(&handler, &target_ctx).await;
8695        let ModuleControlResponseToModule::LiveRoots {
8696            roots,
8697            unknown_root_bindings,
8698            total_bindings,
8699        } = actual
8700        else {
8701            panic!("expected live roots")
8702        };
8703        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8704        assert_eq!(
8705            unknown_root_bindings, 1,
8706            "unknown-root arm must not read as no bindings"
8707        );
8708        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8709        assert_eq!(
8710            total_bindings,
8711            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8712        );
8713    }
8714
8715    /// A module reads HELLO_ACK as its first frame and exits on anything else,
8716    /// so the ack has to be on its outbound queue before the module is
8717    /// routable. The connection loop writes a handler's replies only after the
8718    /// handler returns; this test stops in exactly that gap, runs a real
8719    /// route.open from another connection, and only then writes whatever the
8720    /// HELLO handler returned, the way the loop would. If the ack were still a
8721    /// reply, the route.bind request would reach the module first.
8722    #[tokio::test(start_paused = true)]
8723    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8724        let forwarding = Arc::new(ForwardingTable::default());
8725        let handler =
8726            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8727        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8728        let replies = handler
8729            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8730            .await
8731            .unwrap();
8732        let queued_by_hello = module_rx.len();
8733
8734        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8735        let open_handler = handler.clone();
8736        let open = tokio::spawn(async move {
8737            open_handler
8738                .handle_control_frame(
8739                    &client_ctx,
8740                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8741                )
8742                .await
8743                .unwrap()
8744        });
8745        // Let the route.open run until its route.bind is on the module's queue.
8746        let mut spins = 0;
8747        while module_rx.len() == queued_by_hello {
8748            spins += 1;
8749            assert!(spins < 10_000, "route.open never queued a route.bind");
8750            tokio::task::yield_now().await;
8751        }
8752
8753        // Now the connection loop's half: write the HELLO handler's replies.
8754        for reply in replies {
8755            module_ctx.egress.send(reply).await.unwrap();
8756        }
8757
8758        let first = module_rx.recv().await.unwrap().frame;
8759        assert_eq!(
8760            first.header.ty,
8761            FrameType::HelloAck,
8762            "the first frame a registering module reads must be its HELLO_ACK"
8763        );
8764        assert_eq!(first.header.corr, 7);
8765        let second = module_rx.recv().await.unwrap().frame;
8766        assert_eq!(second.header.ty, FrameType::Request);
8767        assert!(
8768            matches!(
8769                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8770                ModuleControlRequest::RouteBind { .. }
8771            ),
8772            "the route.bind follows the ack"
8773        );
8774        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8775
8776        handler
8777            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8778            .await
8779            .unwrap();
8780        assert!(open.await.unwrap().is_empty());
8781        let _ = client_rx.recv().await.unwrap();
8782    }
8783
8784    #[tokio::test(start_paused = true)]
8785    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8786        let handler = ControlHandler::with_forwarding(
8787            Arc::new(Registry::default()),
8788            Arc::new(ForwardingTable::default()),
8789        );
8790        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8791        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8792        hello_via_sink(
8793            &handler,
8794            &first_ctx,
8795            &mut first_rx,
8796            hello_frame("first", PROTOCOL_VERSION, 1),
8797        )
8798        .await;
8799        hello_via_sink(
8800            &handler,
8801            &second_ctx,
8802            &mut second_rx,
8803            hello_frame("second", PROTOCOL_VERSION, 2),
8804        )
8805        .await;
8806        let root = unique_project_root("second-only");
8807        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8808        let cloned = handler.clone();
8809        let open = tokio::spawn(async move {
8810            cloned
8811                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8812                .await
8813                .unwrap()
8814        });
8815        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8816            .await
8817            .unwrap()
8818            .unwrap();
8819        let first = query_live_roots(&handler, &first_ctx).await;
8820        let second = query_live_roots(&handler, &second_ctx).await;
8821        assert!(
8822            matches!(
8823                first,
8824                ModuleControlResponseToModule::LiveRoots {
8825                    total_bindings: 0,
8826                    ..
8827                }
8828            ),
8829            "cross-module scope must not expose another module's roots"
8830        );
8831        assert!(
8832            matches!(
8833                second,
8834                ModuleControlResponseToModule::LiveRoots {
8835                    total_bindings: 1,
8836                    ..
8837                }
8838            ),
8839            "second module must see its pending route"
8840        );
8841        handler
8842            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8843            .await
8844            .unwrap();
8845        assert!(open.await.unwrap().is_empty());
8846    }
8847
8848    #[tokio::test(start_paused = true)]
8849    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8850        let handler = ControlHandler::with_forwarding(
8851            Arc::new(Registry::default()),
8852            Arc::new(ForwardingTable::default()),
8853        );
8854        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8855        hello_via_sink(
8856            &handler,
8857            &target_ctx,
8858            &mut target_rx,
8859            hello_frame("target", PROTOCOL_VERSION, 1),
8860        )
8861        .await;
8862        let actual = query_live_roots(&handler, &target_ctx).await;
8863        let ModuleControlResponseToModule::LiveRoots {
8864            roots,
8865            unknown_root_bindings,
8866            total_bindings,
8867        } = actual
8868        else {
8869            panic!("expected live roots")
8870        };
8871        assert!(roots.is_empty());
8872        assert_eq!(unknown_root_bindings, 0);
8873        assert_eq!(total_bindings, 0);
8874        assert_eq!(
8875            total_bindings,
8876            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8877        );
8878    }
8879
8880    /// Read the vendored fed corpus rather than hand-building a package.
8881    ///
8882    /// A hand-built object encodes what the test author believed the carrier
8883    /// emits. These vectors are what it actually emits, and one of them exists
8884    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
8885    /// ignores additive unknown fields at the traversal emit terminus."
8886    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8887        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8888            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8889        let text = std::fs::read_to_string(&path)
8890            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8891        let vectors: Vec<(String, Value)> = text
8892            .lines()
8893            .filter(|line| !line.trim().is_empty())
8894            .map(|line| {
8895                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8896                let id = entry["corpus_id"]
8897                    .as_str()
8898                    .expect("every vector carries a corpus_id")
8899                    .to_string();
8900                (id, entry["package"].clone())
8901            })
8902            .collect();
8903        // Pin the count: a corpus that silently shrinks would take its coverage
8904        // with it, and a suite reading N-1 vectors reports the same clean pass
8905        // as one reading N.
8906        assert_eq!(
8907            vectors.len(),
8908            3,
8909            "vendored fed corpus changed size; re-sync from subc-federation"
8910        );
8911
8912        // Pin what makes the corpus DISCRIMINATING, not just present.
8913        //
8914        // The relay test below takes its expected value from the corpus, so the
8915        // corpus supplies the test's power to detect a lossy relay rather than
8916        // its correctness. A relay that dropped unrecognised fields would still
8917        // be caught -- but only by a package carrying fields it does not know.
8918        // Shrink every package to the handful of keys any implementation would
8919        // recognise and the test keeps passing over an input that can no longer
8920        // fail, which is the same clean green as a corpus that shrank away.
8921        //
8922        // So assert the precondition rather than duplicating the packages here:
8923        // at least one vector must carry a field beyond the small common set.
8924        // That is one claim to maintain instead of nine, and it fails loudly if
8925        // a re-sync ever flattens the corpus.
8926        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8927        let richest = vectors
8928            .iter()
8929            .filter_map(|(_, package)| package.as_object())
8930            .map(|object| {
8931                object
8932                    .keys()
8933                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8934                    .count()
8935            })
8936            .max()
8937            .unwrap_or(0);
8938        assert!(
8939            richest >= 2,
8940            "vendored corpus no longer carries a package with unmodelled fields, \
8941             so the relay test can no longer distinguish a verbatim relay from a lossy one"
8942        );
8943
8944        vectors
8945    }
8946
8947    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
8948    ///
8949    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
8950    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
8951    /// three-key object, so a relay that quietly dropped fields it did not
8952    /// recognise would satisfy it. These vectors carry nine keys including ones
8953    /// this crate has no type for, so a typed relay fails here and only here.
8954    #[tokio::test]
8955    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8956        for (corpus_id, package) in fed_admission_facts_vectors() {
8957            let registry = Arc::new(Registry::default());
8958            let forwarding = Arc::new(ForwardingTable::default());
8959            let supervisor = SupervisorHandle::new();
8960            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8961            let handler =
8962                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8963                    .with_supervisor(supervisor)
8964                    .with_admission_facts_config(
8965                        Some("fed".to_string()),
8966                        Some(vec!["target".to_string()]),
8967                    );
8968
8969            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8970            hello_via_sink(
8971                &handler,
8972                &target_ctx,
8973                &mut target_rx,
8974                hello_frame("target", PROTOCOL_VERSION, 1),
8975            )
8976            .await;
8977
8978            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8979            let route_handler = handler.clone();
8980            let expected = package.clone();
8981            let route_task = tokio::spawn(async move {
8982                route_handler
8983                    .handle_control_frame(
8984                        &client_ctx,
8985                        route_open_frame_with_admission_facts(
8986                            20,
8987                            "target",
8988                            unique_project_root("admission-facts"),
8989                            Some(subc_control::ConsumerIdentity {
8990                                module_id: "fed".to_string(),
8991                                launch_nonce: "fed-nonce".to_string(),
8992                            }),
8993                            Some(package),
8994                        ),
8995                    )
8996                    .await
8997                    .unwrap()
8998            });
8999
9000            let bind_frame = target_rx.recv().await.unwrap();
9001            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9002            let ModuleControlRequest::RouteBind {
9003                admission_facts, ..
9004            } = bind
9005            else {
9006                panic!("{corpus_id}: expected route.bind")
9007            };
9008            assert_eq!(
9009                admission_facts,
9010                Some(expected),
9011                "{corpus_id}: relay must not add, drop or reshape any field"
9012            );
9013
9014            handler
9015                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9016                .await
9017                .unwrap();
9018            route_task.await.unwrap();
9019        }
9020    }
9021
9022    #[tokio::test]
9023    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9024        let registry = Arc::new(Registry::default());
9025        let forwarding = Arc::new(ForwardingTable::default());
9026        let supervisor = SupervisorHandle::new();
9027        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9028        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9029        let handler =
9030            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9031                .with_supervisor(supervisor)
9032                .with_admission_facts_config(
9033                    Some("fed".to_string()),
9034                    Some(vec!["target".to_string()]),
9035                );
9036
9037        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9038        hello_via_sink(
9039            &handler,
9040            &target_ctx,
9041            &mut target_rx,
9042            hello_frame("target", PROTOCOL_VERSION, 1),
9043        )
9044        .await;
9045        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9046        hello_via_sink(
9047            &handler,
9048            &other_ctx,
9049            &mut other_rx,
9050            hello_frame("other", PROTOCOL_VERSION, 2),
9051        )
9052        .await;
9053
9054        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9055        let expected_facts = facts.clone();
9056        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9057        let route_handler = handler.clone();
9058        let route_task = tokio::spawn(async move {
9059            route_handler
9060                .handle_control_frame(
9061                    &client_ctx,
9062                    route_open_frame_with_admission_facts(
9063                        10,
9064                        "target",
9065                        unique_project_root("admission-facts"),
9066                        Some(subc_control::ConsumerIdentity {
9067                            module_id: "fed".to_string(),
9068                            launch_nonce: "fed-nonce".to_string(),
9069                        }),
9070                        Some(facts.clone()),
9071                    ),
9072                )
9073                .await
9074                .unwrap()
9075        });
9076        let bind_frame = target_rx.recv().await.unwrap();
9077        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9078        let ModuleControlRequest::RouteBind {
9079            admission_facts, ..
9080        } = bind
9081        else {
9082            panic!("expected route.bind")
9083        };
9084        assert_eq!(admission_facts, Some(expected_facts));
9085        handler
9086            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9087            .await
9088            .unwrap();
9089        assert!(route_task.await.unwrap().is_empty());
9090        assert!(matches!(
9091            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9092                .unwrap(),
9093            ClientControlResponse::RouteOpen { .. }
9094        ));
9095
9096        let direct = handler
9097            .handle_control_frame(
9098                &route_ctx(ConnectionId::new(73)).0,
9099                route_open_frame_with_admission_facts(
9100                    11,
9101                    "target",
9102                    unique_project_root("admission-facts"),
9103                    None,
9104                    Some(json!({"x": 1})),
9105                ),
9106            )
9107            .await
9108            .unwrap();
9109        assert_eq!(
9110            parse_error(&direct[0])["code"],
9111            "admission_facts_not_permitted"
9112        );
9113
9114        let different_reserved = handler
9115            .handle_control_frame(
9116                &route_ctx(ConnectionId::new(77)).0,
9117                route_open_frame_with_admission_facts(
9118                    15,
9119                    "target",
9120                    unique_project_root("admission-facts"),
9121                    Some(subc_control::ConsumerIdentity {
9122                        module_id: "other".to_string(),
9123                        launch_nonce: "other-nonce".to_string(),
9124                    }),
9125                    Some(json!({"x": 1})),
9126                ),
9127            )
9128            .await
9129            .unwrap();
9130        assert_eq!(
9131            parse_error(&different_reserved[0])["code"],
9132            "admission_facts_not_permitted"
9133        );
9134
9135        let other_target = handler
9136            .handle_control_frame(
9137                &route_ctx(ConnectionId::new(74)).0,
9138                route_open_frame_with_admission_facts(
9139                    12,
9140                    "other",
9141                    unique_project_root("admission-facts"),
9142                    Some(subc_control::ConsumerIdentity {
9143                        module_id: "fed".to_string(),
9144                        launch_nonce: "fed-nonce".to_string(),
9145                    }),
9146                    Some(json!({"x": 1})),
9147                ),
9148            )
9149            .await
9150            .unwrap();
9151        assert_eq!(
9152            parse_error(&other_target[0])["code"],
9153            "admission_facts_target_not_allowed"
9154        );
9155
9156        let nonexistent = handler
9157            .handle_control_frame(
9158                &route_ctx(ConnectionId::new(75)).0,
9159                route_open_frame_with_admission_facts(
9160                    13,
9161                    "missing",
9162                    unique_project_root("admission-facts"),
9163                    None,
9164                    Some(json!({"x": 1})),
9165                ),
9166            )
9167            .await
9168            .unwrap();
9169        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9170
9171        let described = handler
9172            .handle_control_frame(
9173                &route_ctx(ConnectionId::new(76)).0,
9174                Frame::build(
9175                    FrameType::Request,
9176                    control_flags(),
9177                    0,
9178                    0,
9179                    14,
9180                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9181                )
9182                .unwrap(),
9183            )
9184            .await
9185            .unwrap();
9186        let ClientControlResponse::ServerDescribe { capabilities, .. } =
9187            serde_json::from_slice(&described[0].body).unwrap()
9188        else {
9189            panic!("expected server.describe response")
9190        };
9191        assert!(capabilities
9192            .iter()
9193            .any(|cap| cap == "admission_facts_relay_v1"));
9194    }
9195
9196    #[tokio::test]
9197    async fn admission_facts_without_configured_carrier_are_rejected() {
9198        let registry = Arc::new(Registry::default());
9199        let forwarding = Arc::new(ForwardingTable::default());
9200        let handler = ControlHandler::with_forwarding(registry, forwarding);
9201        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9202        hello_via_sink(
9203            &handler,
9204            &target_ctx,
9205            &mut target_rx,
9206            hello_frame("target", PROTOCOL_VERSION, 1),
9207        )
9208        .await;
9209
9210        let responses = handler
9211            .handle_control_frame(
9212                &route_ctx(ConnectionId::new(79)).0,
9213                route_open_frame_with_admission_facts(
9214                    16,
9215                    "target",
9216                    unique_project_root("admission-facts"),
9217                    None,
9218                    Some(json!({"x": 1})),
9219                ),
9220            )
9221            .await
9222            .unwrap();
9223        assert_eq!(
9224            parse_error(&responses[0])["code"],
9225            "admission_facts_not_permitted"
9226        );
9227    }
9228
9229    #[tokio::test]
9230    async fn route_open_relays_consumer_capabilities_verbatim() {
9231        let registry = Arc::new(Registry::default());
9232        let forwarding = Arc::new(ForwardingTable::default());
9233        let handler =
9234            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9235        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9236        hello_via_sink(
9237            &handler,
9238            &module_ctx,
9239            &mut module_rx,
9240            hello_frame("aft", PROTOCOL_VERSION, 7),
9241        )
9242        .await;
9243
9244        let expected = vec!["elicitation".to_string(), "roots".to_string()];
9245        let expected_for_request = expected.clone();
9246        let project_root = unique_project_root("consumer-capabilities-present");
9247        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9248        let route_handler = handler.clone();
9249        let route_task = tokio::spawn(async move {
9250            route_handler
9251                .handle_control_frame(
9252                    &client_ctx,
9253                    route_open_frame_with_consumer_capabilities(
9254                        401,
9255                        "aft",
9256                        project_root,
9257                        Some(expected_for_request),
9258                    ),
9259                )
9260                .await
9261                .unwrap()
9262        });
9263        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9264            .await
9265            .unwrap()
9266            .unwrap();
9267        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9268        let ModuleControlRequest::RouteBind {
9269            consumer_capabilities,
9270            ..
9271        } = bind
9272        else {
9273            panic!("expected route.bind request, got {bind:?}");
9274        };
9275        assert_eq!(consumer_capabilities, Some(expected.clone()));
9276
9277        handler
9278            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9279            .await
9280            .unwrap();
9281        let route_response = route_task.await.unwrap();
9282        assert!(route_response.is_empty());
9283        let published = client_rx.recv().await.unwrap();
9284        assert!(matches!(
9285            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9286            ClientControlResponse::RouteOpen { .. }
9287        ));
9288    }
9289
9290    #[tokio::test]
9291    async fn route_open_without_consumer_capabilities_relays_none() {
9292        let registry = Arc::new(Registry::default());
9293        let forwarding = Arc::new(ForwardingTable::default());
9294        let handler =
9295            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9296        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9297        hello_via_sink(
9298            &handler,
9299            &module_ctx,
9300            &mut module_rx,
9301            hello_frame("aft", PROTOCOL_VERSION, 7),
9302        )
9303        .await;
9304
9305        let project_root = unique_project_root("consumer-capabilities-absent");
9306        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9307        let route_handler = handler.clone();
9308        let route_task = tokio::spawn(async move {
9309            route_handler
9310                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9311                .await
9312                .unwrap()
9313        });
9314        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9315            .await
9316            .unwrap()
9317            .unwrap();
9318        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9319        let ModuleControlRequest::RouteBind {
9320            consumer_capabilities,
9321            ..
9322        } = bind
9323        else {
9324            panic!("expected route.bind request, got {bind:?}");
9325        };
9326        assert_eq!(consumer_capabilities, None);
9327
9328        handler
9329            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9330            .await
9331            .unwrap();
9332        let route_response = route_task.await.unwrap();
9333        assert!(route_response.is_empty());
9334        let published = client_rx.recv().await.unwrap();
9335        assert!(matches!(
9336            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9337            ClientControlResponse::RouteOpen { .. }
9338        ));
9339    }
9340
9341    /// Opens a route to a freshly registered `aft` with `sent` as the
9342    /// route.open's role_versions, acks the bind, and returns the role_versions
9343    /// the module's bind carried.
9344    async fn bind_role_versions_for(
9345        sent: Option<BTreeMap<String, String>>,
9346        connection: u64,
9347    ) -> Option<BTreeMap<String, String>> {
9348        let registry = Arc::new(Registry::default());
9349        let forwarding = Arc::new(ForwardingTable::default());
9350        let handler =
9351            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9352        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9353        hello_via_sink(
9354            &handler,
9355            &module_ctx,
9356            &mut module_rx,
9357            hello_frame("aft", PROTOCOL_VERSION, 7),
9358        )
9359        .await;
9360        let project_root = unique_project_root("role-versions");
9361        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9362        let route_handler = handler.clone();
9363        let route_task = tokio::spawn(async move {
9364            route_handler
9365                .handle_control_frame(
9366                    &client_ctx,
9367                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
9368                )
9369                .await
9370                .unwrap()
9371        });
9372        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9373            .await
9374            .expect("a well-formed route.open reaches the module as a bind")
9375            .unwrap();
9376        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9377        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9378            panic!("expected route.bind request, got {bind:?}");
9379        };
9380        handler
9381            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9382            .await
9383            .unwrap();
9384        assert!(route_task.await.unwrap().is_empty());
9385        let published = client_rx.recv().await.unwrap();
9386        assert!(matches!(
9387            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9388            ClientControlResponse::RouteOpen { .. }
9389        ));
9390        role_versions
9391    }
9392
9393    #[tokio::test]
9394    async fn route_open_relays_role_versions_verbatim() {
9395        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9396        assert_eq!(
9397            bind_role_versions_for(Some(sent.clone()), 141).await,
9398            Some(sent)
9399        );
9400    }
9401
9402    /// An empty map declares nothing, so the provider sees no field rather
9403    /// than an empty object it would have to treat as a second "none".
9404    #[tokio::test]
9405    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9406        assert_eq!(bind_role_versions_for(None, 143).await, None);
9407        assert_eq!(
9408            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9409            None
9410        );
9411    }
9412
9413    #[tokio::test]
9414    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9415        let registry = Arc::new(Registry::default());
9416        let forwarding = Arc::new(ForwardingTable::default());
9417        let handler =
9418            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9419        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9420        hello_via_sink(
9421            &handler,
9422            &module_ctx,
9423            &mut module_rx,
9424            hello_frame("aft", PROTOCOL_VERSION, 7),
9425        )
9426        .await;
9427
9428        let nine: BTreeMap<String, String> = (0..9)
9429            .map(|index| (format!("role-{index}"), "v1".to_string()))
9430            .collect();
9431        for (label, malformed) in [
9432            (
9433                "invalid role name",
9434                role_versions(&[("Tool_Provider", "v1")]),
9435            ),
9436            ("invalid version", role_versions(&[("tool-provider", "v0")])),
9437            ("nine entries", nine),
9438        ] {
9439            // A refused open answers at once; one that reached the module
9440            // would wait for its bind ack and trip this timeout.
9441            let responses = tokio::time::timeout(
9442                Duration::from_secs(1),
9443                handler.handle_control_frame(
9444                    &route_ctx(ConnectionId::new(148)).0,
9445                    route_open_frame_with_role_versions(
9446                        404,
9447                        "aft",
9448                        unique_project_root("role-versions-malformed"),
9449                        Some(malformed),
9450                    ),
9451                ),
9452            )
9453            .await
9454            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9455            .unwrap();
9456            assert_eq!(responses.len(), 1, "{label}");
9457            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9458            let error = parse_error(&responses[0]);
9459            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9460            assert_eq!(
9461                error["detail"]["field"], "role_versions",
9462                "{label}: {error}"
9463            );
9464            assert!(
9465                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9466                "{label}: a malformed declaration is terminal"
9467            );
9468            assert!(
9469                module_rx.try_recv().is_err(),
9470                "{label}: the module must never see a bind"
9471            );
9472        }
9473    }
9474
9475    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
9476    /// consumer can tell this daemon forwards the field from one that would
9477    /// drop it.
9478    #[tokio::test]
9479    async fn route_role_versions_capability_is_advertised() {
9480        let handler = ControlHandler::new(Arc::new(Registry::default()));
9481        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9482        let ack = hello_via_sink(
9483            &handler,
9484            &ctx,
9485            &mut rx,
9486            hello_frame("m", PROTOCOL_VERSION, 1),
9487        )
9488        .await;
9489        let ack = parse_ack(&ack);
9490        assert!(
9491            ack.subc_capabilities
9492                .iter()
9493                .any(|c| c == "route-role-versions/v1"),
9494            "{:?}",
9495            ack.subc_capabilities
9496        );
9497
9498        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9499        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9500        let reply = handler
9501            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9502            .await
9503            .unwrap()
9504            .pop()
9505            .unwrap();
9506        let ClientControlResponse::ServerDescribe { capabilities, .. } =
9507            serde_json::from_slice(&reply.body).unwrap()
9508        else {
9509            panic!("not a server.describe reply");
9510        };
9511        assert!(
9512            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9513            "{capabilities:?}"
9514        );
9515    }
9516
9517    #[tokio::test]
9518    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9519        let registry = Arc::new(Registry::default());
9520        let forwarding = Arc::new(ForwardingTable::default());
9521        let handler =
9522            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9523                .with_health_probe_timeout(Duration::from_secs(5));
9524        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9525        hello_via_sink(
9526            &handler,
9527            &module_ctx,
9528            &mut module_rx,
9529            non_routable_hello_frame_with_control_ops(
9530                "mcp",
9531                300,
9532                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9533            ),
9534        )
9535        .await;
9536        assert!(registry
9537            .get_module("mcp")
9538            .unwrap()
9539            .unwrap()
9540            .manifest
9541            .provides
9542            .is_empty());
9543
9544        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9545        let route_response = handler
9546            .handle_control_frame(
9547                &route_client_ctx,
9548                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9549            )
9550            .await
9551            .unwrap();
9552        assert_eq!(route_response[0].header.ty, FrameType::Error);
9553        assert_eq!(
9554            parse_error(&route_response[0])["code"],
9555            "target_unavailable"
9556        );
9557        assert!(parse_error(&route_response[0])["message"]
9558            .as_str()
9559            .unwrap()
9560            .contains("does not provide the requested target"));
9561        assert!(module_rx.try_recv().is_err());
9562
9563        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9564        let health_handler = handler.clone();
9565        let health_task = tokio::spawn(async move {
9566            health_handler
9567                .handle_control_frame(
9568                    &health_client_ctx,
9569                    supervisor_health_probe_frame(302, "mcp"),
9570                )
9571                .await
9572                .unwrap()
9573        });
9574        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9575            .await
9576            .unwrap()
9577            .unwrap();
9578        assert_eq!(
9579            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9580            ModuleControlRequest::HealthCheck {}
9581        );
9582        handler
9583            .handle_control_frame(
9584                &module_ctx,
9585                health_response(health_frame.header.corr, HealthStatus::Ok),
9586            )
9587            .await
9588            .unwrap();
9589        let health_response = health_task.await.unwrap();
9590        assert_eq!(health_response[0].header.ty, FrameType::Response);
9591        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9592            ClientControlResponse::SupervisorHealthProbe {
9593                module_id, status, ..
9594            } => {
9595                assert_eq!(module_id, "mcp");
9596                assert_eq!(status, HealthStatus::Ok);
9597            }
9598            other => panic!("unexpected health response: {other:?}"),
9599        }
9600
9601        // Exercise the forwarding cleanup path directly while leaving the registry
9602        // advertisement in place. If cleanup leaves a stale control sink behind,
9603        // the next probe will enqueue onto it and wait for the long probe timeout
9604        // instead of returning an immediate no-connection error.
9605        forwarding
9606            .cleanup_connection(module_ctx.connection_id)
9607            .unwrap();
9608        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9609        let cleanup_response = tokio::time::timeout(
9610            Duration::from_millis(200),
9611            handler.handle_control_frame(
9612                &cleanup_probe_ctx,
9613                supervisor_health_probe_frame(303, "mcp"),
9614            ),
9615        )
9616        .await
9617        .expect("probe should fail immediately when the control lane is gone")
9618        .unwrap();
9619        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9620        assert_eq!(
9621            parse_error(&cleanup_response[0])["code"],
9622            "target_unavailable"
9623        );
9624        assert!(parse_error(&cleanup_response[0])["message"]
9625            .as_str()
9626            .unwrap()
9627            .contains("no module connection"));
9628
9629        handler
9630            .cleanup_connection(module_ctx.connection_id)
9631            .unwrap();
9632    }
9633
9634    #[tokio::test]
9635    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9636        let registry = Arc::new(Registry::default());
9637        let supervisor_handle = SupervisorHandle::new();
9638        let supervisor =
9639            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9640                .with_handle(supervisor_handle.clone())
9641                .with_connection_file_path(
9642                    std::env::temp_dir()
9643                        .join(format!("subc-route-open-warming-{}", std::process::id())),
9644                );
9645        let module = supervisor
9646            .supervise_configured(
9647                ModuleSpec {
9648                    module_id: "warming".to_string(),
9649                    program: fake_aft_stub_path(),
9650                    args: Vec::new(),
9651                    env: Vec::new(),
9652                    reserved: false,
9653                    reserved_prefixes: Vec::new(),
9654                    protocol: ModuleProtocol::Subc,
9655                    overlap: Default::default(),
9656                },
9657                true,
9658            )
9659            .unwrap();
9660        assert_eq!(module.state().unwrap(), ModuleState::Running);
9661
9662        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9663        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9664        let response = handler
9665            .handle_control_frame(
9666                &ctx,
9667                route_open_frame(304, "warming", unique_project_root("warming")),
9668            )
9669            .await
9670            .unwrap();
9671        module.stop().await.unwrap();
9672
9673        assert_eq!(response[0].header.ty, FrameType::Error);
9674        let error = parse_error(&response[0]);
9675        assert_eq!(error["code"], "module_warming");
9676        assert!(error["message"]
9677            .as_str()
9678            .unwrap()
9679            .contains("state=running, enabled=true, live=false"));
9680    }
9681
9682    #[test]
9683    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9684        let handler = ControlHandler::new(Arc::new(Registry::default()));
9685        let capture = EventCapture::default();
9686        let _subscriber =
9687            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9688        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9689        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9690        let pending = (0..limit).collect::<Vec<_>>();
9691        let response = handler
9692            .route_open_capacity_refusal(
9693                &ctx,
9694                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9695                "busy",
9696                pending.len(),
9697                limit,
9698            )
9699            .unwrap();
9700        assert_eq!(parse_error(&response)["code"], "target_unavailable");
9701        let event = capture
9702            .events()
9703            .into_iter()
9704            .find(|event| {
9705                event.target == "control"
9706                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9707            })
9708            .expect("connection admission refusal event");
9709        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9710        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9711    }
9712
9713    #[test]
9714    fn route_open_target_cap_logs_admission_reason_and_capacity() {
9715        let handler = ControlHandler::new(Arc::new(Registry::default()));
9716        let capture = EventCapture::default();
9717        let _subscriber =
9718            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9719        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9720        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9721        let guards = (0..limit)
9722            .map(|_| {
9723                handler
9724                    .route_bind_concurrency
9725                    .try_admit("busy", limit)
9726                    .unwrap()
9727            })
9728            .collect::<Vec<_>>();
9729        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9730            Err(in_flight) => in_flight,
9731            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9732        };
9733        let response = handler
9734            .route_open_target_capacity_refusal(
9735                &ctx,
9736                &route_open_frame(397, "busy", unique_project_root("target-cap")),
9737                "busy",
9738                in_flight,
9739            )
9740            .unwrap();
9741        assert_eq!(parse_error(&response)["code"], "target_unavailable");
9742        let event = capture
9743            .events()
9744            .into_iter()
9745            .find(|event| {
9746                event.target == "control"
9747                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9748            })
9749            .expect("target admission refusal event");
9750        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9751        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9752        drop(guards);
9753    }
9754
9755    /// One wire code has several senders, so the refusal line names the check
9756    /// that refused. This drives the shared refusal path for ordinary refusals
9757    /// with an unregistered
9758    /// target and requires the branch label on the event.
9759    #[tokio::test]
9760    async fn route_open_refusal_names_the_check_that_refused() {
9761        let handler = ControlHandler::new(Arc::new(Registry::default()));
9762        let capture = EventCapture::default();
9763        let _subscriber =
9764            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9765        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9766        let response = handler
9767            .handle_control_frame(
9768                &ctx,
9769                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9770            )
9771            .await
9772            .unwrap();
9773
9774        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9775        let event = capture
9776            .events()
9777            .into_iter()
9778            .find(|event| {
9779                event.target == "control"
9780                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9781            })
9782            .expect("route.open refusal event");
9783        assert_eq!(
9784            event.fields.get("reason"),
9785            Some(&"\"not_registered\"".to_string())
9786        );
9787    }
9788
9789    #[tokio::test]
9790    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9791        let registry = Arc::new(Registry::default());
9792        let supervisor_handle = SupervisorHandle::new();
9793        let supervisor =
9794            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9795                .with_handle(supervisor_handle.clone())
9796                .with_connection_file_path(std::env::temp_dir().join(format!(
9797                    "subc-route-open-refusal-info-{}",
9798                    std::process::id()
9799                )));
9800        let module = supervisor
9801            .supervise_configured(
9802                ModuleSpec {
9803                    module_id: "warming".to_string(),
9804                    program: fake_aft_stub_path(),
9805                    args: Vec::new(),
9806                    env: Vec::new(),
9807                    reserved: false,
9808                    reserved_prefixes: Vec::new(),
9809                    protocol: ModuleProtocol::Subc,
9810                    overlap: Default::default(),
9811                },
9812                true,
9813            )
9814            .unwrap();
9815        assert_eq!(module.state().unwrap(), ModuleState::Running);
9816
9817        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9818        assert!(handler
9819            .counters()
9820            .snapshot()
9821            .get("route_open_refused_by_code")
9822            .is_none());
9823        let capture = EventCapture::default();
9824        let _subscriber =
9825            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9826        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9827        let response = handler
9828            .handle_control_frame(
9829                &ctx,
9830                route_open_frame(394, "warming", unique_project_root("refusal-info")),
9831            )
9832            .await
9833            .unwrap();
9834        module.stop().await.unwrap();
9835
9836        assert_eq!(parse_error(&response[0])["code"], "module_warming");
9837        let event = capture
9838            .events()
9839            .into_iter()
9840            .find(|event| {
9841                event.target == "control"
9842                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9843            })
9844            .expect("route.open refusal event");
9845        assert_eq!(
9846            event.fields.get("module_id"),
9847            Some(&"\"warming\"".to_string())
9848        );
9849        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9850        assert_eq!(
9851            event.fields.get("reason"),
9852            Some(&"\"supervised_not_registered\"".to_string())
9853        );
9854        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9855        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9856        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9857        assert_eq!(
9858            handler.counters().snapshot()["route_open_refused_by_code"],
9859            json!({ "module_warming": 1 })
9860        );
9861    }
9862
9863    const OUTAGE_START: &str = "route.open refusing module: not serving";
9864    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9865
9866    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9867        capture
9868            .events()
9869            .into_iter()
9870            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9871            .collect()
9872    }
9873
9874    fn supervise_stub(
9875        registry: &Arc<Registry>,
9876        module_id: &str,
9877        enabled: bool,
9878    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9879        let supervisor_handle = SupervisorHandle::new();
9880        let supervisor =
9881            Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9882                .with_handle(supervisor_handle.clone())
9883                .with_connection_file_path(std::env::temp_dir().join(format!(
9884                    "subc-route-outage-{module_id}-{}",
9885                    std::process::id()
9886                )));
9887        let module = supervisor
9888            .supervise_configured(
9889                ModuleSpec {
9890                    module_id: module_id.to_string(),
9891                    program: fake_aft_stub_path(),
9892                    args: Vec::new(),
9893                    env: Vec::new(),
9894                    reserved: false,
9895                    reserved_prefixes: Vec::new(),
9896                    protocol: ModuleProtocol::Subc,
9897                    overlap: Default::default(),
9898                },
9899                enabled,
9900            )
9901            .unwrap();
9902        (supervisor_handle, module)
9903    }
9904
9905    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9906        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9907            module_id: module_id.to_string(),
9908            drain_timeout_ms: Some(50),
9909        })
9910        .unwrap();
9911        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9912    }
9913
9914    /// Two handlers built over one forwarding table must share one outage
9915    /// tracker; separate trackers would each log their own opening line for
9916    /// the same outage.
9917    #[test]
9918    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9919        let registry = Arc::new(Registry::default());
9920        let forwarding = Arc::new(ForwardingTable::default());
9921        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9922        let second = ControlHandler::with_forwarding(registry, forwarding);
9923        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9924    }
9925
9926    /// A client can name any module id it likes. Refusing an unknown one,
9927    /// however often, must not create outage state or outage lines, or the
9928    /// tracker would be a memory sink any client could fill.
9929    #[tokio::test(flavor = "current_thread")]
9930    async fn route_open_unknown_module_refusals_add_no_outage_state() {
9931        let handler = ControlHandler::new(Arc::new(Registry::default()));
9932        let capture = EventCapture::default();
9933        let _subscriber =
9934            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9935        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9936        for corr in 0..8 {
9937            let response = handler
9938                .handle_control_frame(
9939                    &ctx,
9940                    route_open_frame(
9941                        380 + corr,
9942                        &format!("nobody-{corr}"),
9943                        unique_project_root("outage-unknown"),
9944                    ),
9945                )
9946                .await
9947                .unwrap();
9948            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9949        }
9950
9951        assert_eq!(handler.route_outages.tracked_module_count(), 0);
9952        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9953        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9954    }
9955
9956    /// Drives the refusal path end to end: a supervised module that served
9957    /// before and stopped being registered with no instruction to stop is a
9958    /// WARN, and the same module refused after an operator `supervisor.restart`
9959    /// is an INFO.
9960    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9961    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
9962        let registry = Arc::new(Registry::default());
9963        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
9964        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9965        let capture = EventCapture::default();
9966        let _subscriber =
9967            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9968        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
9969        // The stub never registers, so pretend it served once: otherwise every
9970        // refusal would fall in its startup window.
9971        handler.route_outages.record_accepted("outage-restart");
9972
9973        let response = handler
9974            .handle_control_frame(
9975                &ctx,
9976                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
9977            )
9978            .await
9979            .unwrap();
9980        assert_eq!(response[0].header.ty, FrameType::Error);
9981        let starts = outage_lines(&capture, OUTAGE_START);
9982        assert_eq!(starts.len(), 1, "{starts:?}");
9983        assert_eq!(starts[0].level, tracing::Level::WARN);
9984        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
9985        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
9986        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
9987        handler.route_outages.record_accepted("outage-restart");
9988        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9989
9990        let restart = handler
9991            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9992            .await
9993            .unwrap();
9994        assert_eq!(
9995            restart[0].header.ty,
9996            FrameType::Response,
9997            "{:?}",
9998            parse_error(&restart[0])
9999        );
10000        handler
10001            .handle_control_frame(
10002                &ctx,
10003                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10004            )
10005            .await
10006            .unwrap();
10007        module.stop().await.unwrap();
10008
10009        let starts = outage_lines(&capture, OUTAGE_START);
10010        assert_eq!(starts.len(), 2, "{starts:?}");
10011        assert_eq!(starts[1].level, tracing::Level::INFO);
10012        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10013    }
10014
10015    /// A restart refused before it touched the module (here: the module is
10016    /// disabled) must clear its operator mark, so the next real outage is
10017    /// still reported as a warning.
10018    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10019    async fn failed_operator_restart_leaves_no_operator_mark() {
10020        let registry = Arc::new(Registry::default());
10021        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
10022        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10023        let capture = EventCapture::default();
10024        let _subscriber =
10025            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10026        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10027        handler.route_outages.record_accepted("outage-disabled");
10028
10029        let restart = handler
10030            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10031            .await
10032            .unwrap();
10033        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10034        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10035
10036        handler
10037            .handle_control_frame(
10038                &ctx,
10039                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10040            )
10041            .await
10042            .unwrap();
10043        let starts = outage_lines(&capture, OUTAGE_START);
10044        assert_eq!(starts.len(), 1, "{starts:?}");
10045        assert_eq!(starts[0].level, tracing::Level::WARN);
10046    }
10047
10048    #[tokio::test(flavor = "current_thread")]
10049    async fn route_open_unknown_module_escapes_target_module_id() {
10050        let handler = ControlHandler::new(Arc::new(Registry::default()));
10051        let capture = EventCapture::default();
10052        let _subscriber =
10053            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10054        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10055        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10056        let response = handler
10057            .handle_control_frame(
10058                &ctx,
10059                route_open_frame(
10060                    395,
10061                    hostile_module_id,
10062                    unique_project_root("hostile-target-module-id"),
10063                ),
10064            )
10065            .await
10066            .unwrap();
10067
10068        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10069        let event = capture
10070            .events()
10071            .into_iter()
10072            .find(|event| {
10073                event.target == "control"
10074                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10075            })
10076            .expect("route.open unknown-module refusal event");
10077        let logged = event.fields.get("module_id").expect("module_id field");
10078        assert!(!logged.bytes().any(|byte| byte < 0x20));
10079        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10080    }
10081
10082    #[tokio::test(flavor = "current_thread")]
10083    async fn route_open_module_rejection_uses_daemon_counter_key() {
10084        let registry = Arc::new(Registry::default());
10085        let forwarding = Arc::new(ForwardingTable::default());
10086        let handler =
10087            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10088        let module_connection = ConnectionId::new(95);
10089        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10090        hello_via_sink(
10091            &handler,
10092            &module_ctx,
10093            &mut module_rx,
10094            hello_frame("aft", PROTOCOL_VERSION, 395),
10095        )
10096        .await;
10097
10098        let client_connection = ConnectionId::new(96);
10099        let (client_ctx, _client_rx) = route_ctx(client_connection);
10100        let capture = EventCapture::default();
10101        let _subscriber =
10102            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10103        let (route_task, bind) = relay_route_open(
10104            &handler,
10105            client_connection,
10106            &client_ctx.egress,
10107            &mut module_rx,
10108            396,
10109            "aft",
10110            "hostile-module-code",
10111        )
10112        .await;
10113        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10114        let rejection = Frame::build(
10115            FrameType::Error,
10116            control_flags(),
10117            0,
10118            0,
10119            bind.header.corr,
10120            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10121        )
10122        .unwrap();
10123        handler
10124            .handle_control_frame(&module_ctx, rejection)
10125            .await
10126            .unwrap();
10127
10128        let response = route_task.await.unwrap();
10129        assert_eq!(parse_error(&response[0])["code"], hostile_code);
10130        let counters = handler.counters().snapshot();
10131        assert_eq!(
10132            counters["route_open_refused_by_code"],
10133            json!({ "module_rejected": 1 })
10134        );
10135        assert!(counters["route_open_refused_by_code"]
10136            .get(hostile_code)
10137            .is_none());
10138
10139        let event = capture
10140            .events()
10141            .into_iter()
10142            .find(|event| {
10143                event.target == "control"
10144                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10145            })
10146            .expect("route.open module-rejection refusal event");
10147        let logged = event.fields.get("module_code").expect("module_code field");
10148        assert!(!logged.bytes().any(|byte| byte < 0x20));
10149        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10150    }
10151
10152    #[tokio::test]
10153    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10154        let registry = Arc::new(Registry::default());
10155        let supervisor_handle = SupervisorHandle::new();
10156        let missing_program = std::env::temp_dir().join(format!(
10157            "subc-route-open-missing-program-{}",
10158            std::process::id()
10159        ));
10160        let supervisor =
10161            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
10162                .with_handle(supervisor_handle.clone());
10163        let module = supervisor
10164            .supervise_configured(
10165                ModuleSpec {
10166                    module_id: "failed".to_string(),
10167                    program: missing_program,
10168                    args: Vec::new(),
10169                    env: Vec::new(),
10170                    reserved: false,
10171                    reserved_prefixes: Vec::new(),
10172                    protocol: ModuleProtocol::Subc,
10173                    overlap: Default::default(),
10174                },
10175                true,
10176            )
10177            .unwrap();
10178        assert_eq!(module.state().unwrap(), ModuleState::Failed);
10179
10180        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10181        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10182        let response = handler
10183            .handle_control_frame(
10184                &ctx,
10185                route_open_frame(305, "failed", unique_project_root("failed")),
10186            )
10187            .await
10188            .unwrap();
10189
10190        assert_eq!(response[0].header.ty, FrameType::Error);
10191        let error = parse_error(&response[0]);
10192        assert_eq!(error["code"], "target_unavailable");
10193        assert!(error["message"]
10194            .as_str()
10195            .unwrap()
10196            .contains("state=failed, enabled=true, live=false"));
10197    }
10198
10199    #[tokio::test]
10200    async fn route_open_role_mismatch_remains_target_unavailable() {
10201        let registry = Arc::new(Registry::default());
10202        let handler = ControlHandler::new(Arc::clone(&registry));
10203        handler
10204            .handle_control(
10205                ConnectionId::new(41),
10206                non_routable_hello_frame_with_control_ops("health-only", 306, None),
10207            )
10208            .unwrap();
10209
10210        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10211        let response = handler
10212            .handle_control_frame(
10213                &ctx,
10214                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10215            )
10216            .await
10217            .unwrap();
10218
10219        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10220        assert!(parse_error(&response[0])["message"]
10221            .as_str()
10222            .unwrap()
10223            .contains("does not provide the requested target"));
10224    }
10225
10226    #[tokio::test]
10227    async fn route_open_inactive_registration_remains_target_unavailable() {
10228        let registry = Arc::new(Registry::default());
10229        let handler = ControlHandler::new(Arc::clone(&registry));
10230        handler
10231            .handle_control(
10232                ConnectionId::new(43),
10233                hello_frame("inactive", PROTOCOL_VERSION, 308),
10234            )
10235            .unwrap();
10236        assert!(registry
10237            .set_module_state_for_test("inactive", ChannelState::Closed)
10238            .unwrap());
10239
10240        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10241        let response = handler
10242            .handle_control_frame(
10243                &ctx,
10244                route_open_frame(309, "inactive", unique_project_root("inactive")),
10245            )
10246            .await
10247            .unwrap();
10248
10249        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10250        assert!(parse_error(&response[0])["message"]
10251            .as_str()
10252            .unwrap()
10253            .contains("is not active"));
10254    }
10255
10256    #[tokio::test]
10257    async fn late_health_reply_is_recorded_through_the_module_response_path() {
10258        let registry = Arc::new(Registry::default());
10259        let forwarding = Arc::new(ForwardingTable::default());
10260        let supervisor_handle = SupervisorHandle::new();
10261        let supervisor = Supervisor::new(Arc::clone(&registry), crate::RestartPolicy::default())
10262            .with_forwarding(Arc::clone(&forwarding))
10263            .with_handle(supervisor_handle.clone());
10264        let module = supervisor
10265            .supervise_configured(
10266                crate::ModuleSpec {
10267                    module_id: "late-health-response".to_string(),
10268                    program: PathBuf::from("disabled-module"),
10269                    args: Vec::new(),
10270                    env: Vec::new(),
10271                    reserved: false,
10272                    reserved_prefixes: Vec::new(),
10273                    protocol: ModuleProtocol::Subc,
10274                    overlap: Default::default(),
10275                },
10276                false,
10277            )
10278            .unwrap();
10279        let handler =
10280            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10281                .with_supervisor(supervisor_handle);
10282        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
10283        handler
10284            .handle_control_frame(
10285                &module_ctx,
10286                hello_frame_with_control_ops(
10287                    "late-health-response",
10288                    PROTOCOL_VERSION,
10289                    7,
10290                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10291                ),
10292            )
10293            .await
10294            .unwrap();
10295        let probe_started_at = Instant::now() - Duration::from_millis(80);
10296        let pending = forwarding
10297            .begin_health_probe_rpc_for(
10298                "late-health-response",
10299                MODULE_CONTROL_OP_HEALTH_CHECK,
10300                probe_started_at,
10301                Instant::now() - Duration::from_millis(1),
10302            )
10303            .unwrap();
10304        assert!(forwarding
10305            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
10306            .unwrap());
10307
10308        let responses = handler
10309            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
10310            .await
10311            .unwrap();
10312
10313        assert!(responses.is_empty());
10314        let health = module.status().unwrap().health;
10315        assert_eq!(health.late_answer_count, 1);
10316        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10317    }
10318
10319    #[tokio::test]
10320    async fn health_probe_timeout_and_module_death_are_typed() {
10321        let registry = Arc::new(Registry::default());
10322        let forwarding = Arc::new(ForwardingTable::default());
10323        let handler =
10324            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10325                .with_health_probe_timeout(Duration::from_millis(50));
10326        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10327        hello_via_sink(
10328            &handler,
10329            &module_ctx,
10330            &mut module_rx,
10331            hello_frame_with_control_ops(
10332                "aft",
10333                PROTOCOL_VERSION,
10334                7,
10335                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10336            ),
10337        )
10338        .await;
10339
10340        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10341        let responses = handler
10342            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10343            .await
10344            .unwrap();
10345        assert_eq!(responses[0].header.ty, FrameType::Error);
10346        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10347        let _ = module_rx.try_recv();
10348
10349        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10350        let health_handler = handler.clone();
10351        let death_task = tokio::spawn(async move {
10352            health_handler
10353                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10354                .await
10355                .unwrap()
10356        });
10357        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10358            .await
10359            .unwrap()
10360            .unwrap();
10361        handler
10362            .cleanup_connection(module_ctx.connection_id)
10363            .unwrap();
10364        let responses = death_task.await.unwrap();
10365        assert_eq!(responses[0].header.ty, FrameType::Error);
10366        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10367    }
10368
10369    #[test]
10370    fn hello_requires_exact_protocol_version() {
10371        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10372            let registry = Arc::new(Registry::default());
10373            let handler = ControlHandler::new(Arc::clone(&registry));
10374            let responses = handler
10375                .handle_control(
10376                    ConnectionId::new(connection),
10377                    hello_frame("aft", offered, 9),
10378                )
10379                .unwrap();
10380
10381            assert_eq!(responses.len(), 1);
10382            assert_eq!(responses[0].header.ty, FrameType::Error);
10383            let error = parse_error(&responses[0]);
10384            assert_eq!(error["code"], "version_unsupported");
10385            assert!(registry.get_module("aft").unwrap().is_none());
10386            assert_eq!(registry.active_registration_count().unwrap(), 0);
10387        }
10388    }
10389
10390    #[test]
10391    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10392        let registry = Arc::new(Registry::default());
10393        let forwarding = Arc::new(ForwardingTable::default());
10394        let handler =
10395            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10396        let module_connection = ConnectionId::new(301);
10397        let registration = registry
10398            .register_with_control_ops(
10399                manifest("aft-push", PROTOCOL_VERSION),
10400                PROTOCOL_VERSION,
10401                module_connection,
10402                module_baseline_control_ops(),
10403            )
10404            .unwrap();
10405        let (module_tx, _module_rx) = mpsc::channel(8);
10406        let endpoint = forwarding
10407            .register_module_connection(
10408                module_connection,
10409                "aft-push".to_string(),
10410                PROTOCOL_VERSION,
10411                manifest_concurrency(&registration.manifest),
10412                FrameSink::new(module_tx),
10413            )
10414            .unwrap();
10415
10416        // A push op this version does not know is ignored (forward-compat), not errored.
10417        let unknown = Frame::build(
10418            FrameType::Push,
10419            control_flags(),
10420            0,
10421            0,
10422            5,
10423            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10424        )
10425        .unwrap();
10426        let out = handler.handle_status_update(endpoint, unknown).unwrap();
10427        assert!(
10428            out.is_empty(),
10429            "unknown push op must be ignored, got {out:?}"
10430        );
10431
10432        // A malformed body for a KNOWN op is a real error worth surfacing.
10433        let malformed = Frame::build(
10434            FrameType::Push,
10435            control_flags(),
10436            0,
10437            0,
10438            6,
10439            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10440        )
10441        .unwrap();
10442        let out = handler.handle_status_update(endpoint, malformed).unwrap();
10443        assert_eq!(out.len(), 1);
10444        assert_eq!(out[0].header.ty, FrameType::Error);
10445        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10446    }
10447
10448    #[test]
10449    fn hello_rejected_when_connection_already_owns_client_routes() {
10450        let registry = Arc::new(Registry::default());
10451        let forwarding = Arc::new(ForwardingTable::default());
10452        let handler =
10453            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10454        // Commits a client route on connection 202 (bound to a module on conn 101).
10455        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
10456        let client_connection = ConnectionId::new(202);
10457
10458        // That same connection now tries to register as a module: rejected, so one
10459        // connection never holds both client-route and module-endpoint state.
10460        let responses = handler
10461            .handle_control(
10462                client_connection,
10463                hello_frame("aft-second", PROTOCOL_VERSION, 9),
10464            )
10465            .unwrap();
10466        assert_eq!(responses[0].header.ty, FrameType::Error);
10467        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10468        assert!(registry.get_module("aft-second").unwrap().is_none());
10469    }
10470
10471    #[tokio::test]
10472    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
10473        let registry = Arc::new(Registry::default());
10474        let forwarding = Arc::new(ForwardingTable::default());
10475        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
10476        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
10477        hello_via_sink(
10478            &handler,
10479            &module_ctx,
10480            &mut module_rx,
10481            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
10482        )
10483        .await;
10484        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
10485        let pending = forwarding
10486            .begin_route_bind_relay_for_test(
10487                client_ctx.connection_id,
10488                client_ctx.egress.clone(),
10489                2,
10490                "alpha",
10491            )
10492            .unwrap();
10493        forwarding
10494            .complete_pending_relay(
10495                module_ctx.connection_id,
10496                pending.corr,
10497                RouteBindRelayOutcome::Accepted,
10498            )
10499            .unwrap();
10500        client_rx.try_recv().unwrap();
10501        for module_id in ["beta", "alpha"] {
10502            let replies = handler
10503                .handle_control_frame(
10504                    &module_ctx,
10505                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
10506                )
10507                .await
10508                .unwrap();
10509            assert_eq!(replies.len(), 1, "second HELLO must be refused");
10510            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
10511        }
10512        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
10513        assert!(registry.get_module("beta").unwrap().is_none());
10514        assert!(matches!(
10515            forwarding
10516                .lookup_data_route(
10517                    client_ctx.connection_id,
10518                    pending.client_channel,
10519                    pending.client_epoch,
10520                )
10521                .unwrap(),
10522            DataRoute::Client(DataRouteState::Bound(_))
10523        ));
10524        assert!(handler
10525            .hello_launch_nonces
10526            .lock()
10527            .unwrap()
10528            .presented(module_ctx.connection_id, Some("alpha-nonce")));
10529        assert!(module_rx.try_recv().is_err());
10530    }
10531
10532    #[test]
10533    fn reserved_module_hello_requires_matching_launch_nonce() {
10534        let registry = Arc::new(Registry::default());
10535        let supervisor = SupervisorHandle::new();
10536        // The supervisor recorded the nonce it injected when it spawned the reserved
10537        // module; the HELLO verifier checks against the same shared handle.
10538        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10539        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10540
10541        // A HELLO with NO nonce is rejected.
10542        let no_nonce = handler
10543            .handle_control(
10544                ConnectionId::new(1),
10545                hello_frame("vault", PROTOCOL_VERSION, 1),
10546            )
10547            .unwrap();
10548        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10549        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10550        assert!(registry.get_module("vault").unwrap().is_none());
10551
10552        // A HELLO with the WRONG nonce is rejected.
10553        let wrong = handler
10554            .handle_control(
10555                ConnectionId::new(2),
10556                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10557            )
10558            .unwrap();
10559        assert_eq!(wrong[0].header.ty, FrameType::Error);
10560        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10561        assert!(registry.get_module("vault").unwrap().is_none());
10562
10563        // A HELLO with the CORRECT nonce registers.
10564        let ok = handler
10565            .handle_control(
10566                ConnectionId::new(3),
10567                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10568            )
10569            .unwrap();
10570        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10571        assert!(registry.get_module("vault").unwrap().is_some());
10572    }
10573
10574    #[test]
10575    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10576        let registry = Arc::new(Registry::default());
10577        let supervisor = SupervisorHandle::new();
10578        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10579        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10580        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10581
10582        let squat = handler
10583            .handle_control(
10584                ConnectionId::new(1),
10585                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10586            )
10587            .unwrap();
10588        assert_eq!(squat[0].header.ty, FrameType::Error);
10589        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10590        assert!(parse_error(&squat[0])["message"]
10591            .as_str()
10592            .unwrap()
10593            .contains("fed:"));
10594
10595        let accepted_peer = handler
10596            .handle_control(
10597                ConnectionId::new(2),
10598                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10599            )
10600            .unwrap();
10601        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10602
10603        let accepted_short = handler
10604            .handle_control(
10605                ConnectionId::new(3),
10606                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10607            )
10608            .unwrap();
10609        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10610
10611        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10612            let response = handler
10613                .handle_control(
10614                    ConnectionId::new(conn),
10615                    hello_frame(module_id, PROTOCOL_VERSION, conn),
10616                )
10617                .unwrap();
10618            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10619        }
10620    }
10621
10622    #[test]
10623    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10624        let registry = Arc::new(Registry::default());
10625        let supervisor = SupervisorHandle::new();
10626        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10627        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10628        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10629        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10630
10631        let owner_nonce = handler
10632            .handle_control(
10633                ConnectionId::new(1),
10634                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10635            )
10636            .unwrap();
10637        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10638        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10639        assert!(registry.get_module("fed:special").unwrap().is_none());
10640
10641        let exact_nonce = handler
10642            .handle_control(
10643                ConnectionId::new(2),
10644                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10645            )
10646            .unwrap();
10647        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10648        assert!(registry.get_module("fed:special").unwrap().is_some());
10649    }
10650
10651    #[test]
10652    fn non_reserved_module_ignores_launch_nonce() {
10653        let registry = Arc::new(Registry::default());
10654        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
10655        // registration succeeds whether a spawned process echoes a nonce or not.
10656        let handler = ControlHandler::new(Arc::clone(&registry));
10657        let no_nonce = handler
10658            .handle_control(
10659                ConnectionId::new(1),
10660                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10661            )
10662            .unwrap();
10663        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10664        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10665
10666        let echoed_nonce = handler
10667            .handle_control(
10668                ConnectionId::new(2),
10669                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10670            )
10671            .unwrap();
10672        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10673        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10674    }
10675
10676    #[test]
10677    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10678        let handler = ControlHandler::default();
10679        let conn = ConnectionId::new(1);
10680        let malformed = Frame::build(
10681            FrameType::Hello,
10682            control_flags(),
10683            0,
10684            0,
10685            3,
10686            b"{not json".to_vec(),
10687        )
10688        .unwrap();
10689
10690        let error = handler.handle_control(conn, malformed).unwrap();
10691        assert_eq!(error[0].header.ty, FrameType::Error);
10692        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10693
10694        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10695        let pong = handler.handle_control(conn, ping).unwrap();
10696        assert_eq!(pong[0].header.ty, FrameType::Pong);
10697        assert_eq!(pong[0].header.corr, 4);
10698    }
10699
10700    #[test]
10701    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10702        let registry = Arc::new(Registry::default());
10703        let handler = ControlHandler::new(Arc::clone(&registry));
10704
10705        handler
10706            .handle_control(
10707                ConnectionId::new(1),
10708                hello_frame("aft", PROTOCOL_VERSION, 1),
10709            )
10710            .unwrap();
10711        let duplicate = handler
10712            .handle_control(
10713                ConnectionId::new(2),
10714                hello_frame("aft", PROTOCOL_VERSION, 2),
10715            )
10716            .unwrap();
10717
10718        assert_eq!(duplicate[0].header.ty, FrameType::Error);
10719        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10720        let registration = registry.get_module("aft").unwrap().unwrap();
10721        assert_eq!(registration.connection_id, ConnectionId::new(1));
10722    }
10723
10724    #[test]
10725    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10726        let registry = Arc::new(Registry::default());
10727        let forwarding = Arc::new(ForwardingTable::default());
10728        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10729        let handler =
10730            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10731                .with_process_liveness(process_liveness);
10732        let (ctx, route_channel, route_epoch) =
10733            bind_liveness_route(&registry, &forwarding, "aft-dead");
10734        let responses = handler
10735            .handle_route_poll(
10736                &ctx,
10737                route_poll_frame(41, PollKind::Liveness, route_channel),
10738                route_channel,
10739                route_epoch,
10740                PollKind::Liveness,
10741            )
10742            .unwrap();
10743
10744        assert_eq!(responses.len(), 1);
10745        assert_eq!(responses[0].header.ty, FrameType::Response);
10746        assert_route_poll_liveness(&responses[0], false);
10747    }
10748
10749    #[test]
10750    fn liveness_poll_without_process_source_uses_bound_route() {
10751        let registry = Arc::new(Registry::default());
10752        let forwarding = Arc::new(ForwardingTable::default());
10753        let handler =
10754            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10755        let (ctx, route_channel, route_epoch) =
10756            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
10757        let responses = handler
10758            .handle_route_poll(
10759                &ctx,
10760                route_poll_frame(42, PollKind::Liveness, route_channel),
10761                route_channel,
10762                route_epoch,
10763                PollKind::Liveness,
10764            )
10765            .unwrap();
10766
10767        assert_route_poll_liveness(&responses[0], true);
10768    }
10769
10770    #[test]
10771    fn liveness_poll_untracked_process_source_uses_bound_route() {
10772        let registry = Arc::new(Registry::default());
10773        let forwarding = Arc::new(ForwardingTable::default());
10774        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10775        let handler =
10776            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10777                .with_process_liveness(process_liveness);
10778        let (ctx, route_channel, route_epoch) =
10779            bind_liveness_route(&registry, &forwarding, "aft-untracked");
10780        let responses = handler
10781            .handle_route_poll(
10782                &ctx,
10783                route_poll_frame(43, PollKind::Liveness, route_channel),
10784                route_channel,
10785                route_epoch,
10786                PollKind::Liveness,
10787            )
10788            .unwrap();
10789
10790        assert_route_poll_liveness(&responses[0], true);
10791    }
10792
10793    #[tokio::test]
10794    async fn unknown_op_returns_unknown_control_op() {
10795        let handler = ControlHandler::default();
10796        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10797        let request = Frame::build(
10798            FrameType::Request,
10799            control_flags(),
10800            0,
10801            0,
10802            55,
10803            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10804        )
10805        .unwrap();
10806
10807        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10808
10809        assert_eq!(response.len(), 1);
10810        assert_eq!(response[0].header.ty, FrameType::Error);
10811        assert_eq!(response[0].header.corr, 55);
10812        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10813    }
10814
10815    #[tokio::test]
10816    async fn supervisor_provenance_rejects_unknown_exact_module() {
10817        let handler = ControlHandler::default();
10818        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10819        let request = Frame::build(
10820            FrameType::Request,
10821            control_flags(),
10822            0,
10823            0,
10824            57,
10825            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10826        )
10827        .unwrap();
10828
10829        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10830
10831        assert_eq!(response.len(), 1);
10832        assert_eq!(response[0].header.ty, FrameType::Error);
10833        assert_eq!(response[0].header.corr, 57);
10834        let error = parse_error(&response[0]);
10835        assert_eq!(error["code"], "unknown_module");
10836        assert_eq!(error["message"], "module_id 'missing' is not supervised");
10837    }
10838
10839    #[test]
10840    fn provenance_probe_override_keeps_handler_tests_deterministic() {
10841        let expected = subc_control::RunningImageAgreement::Unavailable {
10842            reason: subc_control::RunningImageUnavailableReason::HashFailed,
10843        };
10844        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10845        assert_eq!(handler.provenance_probe_override, Some(expected));
10846    }
10847
10848    #[test]
10849    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10850        let verdict = reload_verdict(
10851            std::path::Path::new("/bin/new"),
10852            Some(std::path::Path::new("/bin/old")),
10853            subc_control::RunningImageAgreement::Unavailable {
10854                reason: subc_control::RunningImageUnavailableReason::HashFailed,
10855            },
10856        );
10857        assert!(matches!(
10858            verdict.path,
10859            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10860                if configured == std::path::Path::new("/bin/new")
10861                    && spawned_from == std::path::Path::new("/bin/old")
10862        ));
10863    }
10864
10865    #[test]
10866    fn reload_verdict_detects_replaced_image_at_same_path() {
10867        let image = subc_control::RunningImageAgreement::Mismatch {
10868            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10869                digest: "old".into(),
10870            },
10871            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10872                digest: "new".into(),
10873            },
10874        };
10875        let verdict = reload_verdict(
10876            std::path::Path::new("/bin/same"),
10877            Some(std::path::Path::new("/bin/same")),
10878            image.clone(),
10879        );
10880        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10881        assert_eq!(verdict.image, image);
10882    }
10883
10884    #[test]
10885    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10886        let image = subc_control::RunningImageAgreement::Unavailable {
10887            reason: subc_control::RunningImageUnavailableReason::NotRunning,
10888        };
10889        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10890        assert_eq!(
10891            verdict.path,
10892            subc_control::ReloadPathAgreement::Unavailable {
10893                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10894            }
10895        );
10896        assert_eq!(verdict.image, image);
10897
10898        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10899            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10900        };
10901        let verdict = reload_verdict(
10902            std::path::Path::new("/bin/same"),
10903            Some(std::path::Path::new("/bin/same")),
10904            unconfirmed.clone(),
10905        );
10906        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10907        assert_eq!(verdict.image, unconfirmed);
10908    }
10909
10910    #[test]
10911    fn reload_verdict_preserves_each_image_unavailability_reason() {
10912        use subc_control::RunningImageUnavailableReason as Reason;
10913
10914        for reason in [
10915            Reason::NotRunning,
10916            Reason::UnsupportedPlatform,
10917            Reason::RunningExecutableUnreadable,
10918            Reason::SpawnedPathUnreadable,
10919            Reason::HashFailed,
10920            Reason::ProcessIdentityUnconfirmed,
10921            Reason::Unknown("future_probe_reason".to_string()),
10922        ] {
10923            let image = subc_control::RunningImageAgreement::Unavailable {
10924                reason: reason.clone(),
10925            };
10926            let verdict = reload_verdict(
10927                std::path::Path::new("/bin/same"),
10928                Some(std::path::Path::new("/bin/same")),
10929                image.clone(),
10930            );
10931            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10932            assert_eq!(verdict.image, image, "{reason:?}");
10933        }
10934    }
10935
10936    #[tokio::test]
10937    async fn malformed_control_bodies_return_invalid_control_body() {
10938        let handler = ControlHandler::default();
10939        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10940
10941        for (corr, body) in [
10942            (56, br#"{"route_channel":1}"#.as_slice()),
10943            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10944            (
10945                58,
10946                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10947            ),
10948        ] {
10949            let request = Frame::build(
10950                FrameType::Request,
10951                control_flags(),
10952                0,
10953                0,
10954                corr,
10955                body.to_vec(),
10956            )
10957            .unwrap();
10958            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10959
10960            assert_eq!(response.len(), 1);
10961            assert_eq!(response[0].header.ty, FrameType::Error);
10962            assert_eq!(response[0].header.corr, corr);
10963            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
10964        }
10965    }
10966
10967    #[tokio::test]
10968    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
10969        let registry = Arc::new(Registry::default());
10970        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
10971        let router = Router::with_control_handler(Arc::clone(&control));
10972        let connection = router.begin_connection();
10973        let (ctx, mut rx) = route_ctx(connection.id());
10974
10975        router
10976            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
10977            .await
10978            .unwrap();
10979        let response = rx.recv().await.unwrap();
10980        let ack = parse_ack(&response);
10981        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10982        let channel = 1;
10983
10984        let goodbye =
10985            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
10986        router.route_for_connection(&ctx, goodbye).await.unwrap();
10987        assert!(rx.try_recv().is_err());
10988        assert!(registry.get_module("aft").unwrap().is_none());
10989
10990        router
10991            .route_for_connection(&ctx, channel_request(channel, 13))
10992            .await
10993            .unwrap();
10994        let error_frame = rx.recv().await.unwrap();
10995        assert_eq!(error_frame.header.ty, FrameType::Error);
10996        assert_eq!(error_frame.header.channel, channel);
10997    }
10998
10999    #[tokio::test]
11000    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11001        let registry = Arc::new(Registry::default());
11002        let handler = ControlHandler::new(registry).with_capability_config(
11003            [("prov".to_string(), true), ("cons".to_string(), true)],
11004            BTreeMap::new(),
11005        );
11006        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11007        register_capability_manifest(
11008            &handler,
11009            &provider_ctx,
11010            &mut provider_rx,
11011            capability_manifest("prov", &["thing/v1"], &[]),
11012            1,
11013        )
11014        .await;
11015        let mut consumer = capability_manifest("cons", &[], &[]);
11016        consumer.capabilities.as_mut().unwrap().requires.push(
11017            subc_protocol::manifest::CapabilityRequirement {
11018                capability: "thing/v1".to_string(),
11019                need: subc_protocol::manifest::CapabilityNeed::Required,
11020            },
11021        );
11022        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11023        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11024        assert_eq!(
11025            handler.capability_evaluator.verdict("cons", "thing/v1"),
11026            Some(CapabilityVerdict::Provided)
11027        );
11028        let (mut client_rx, _) = open_route_for_capability_test(
11029            &handler,
11030            &provider_ctx,
11031            &mut provider_rx,
11032            703,
11033            3,
11034            "prov",
11035            None,
11036        )
11037        .await;
11038        let goodbye =
11039            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11040        handler
11041            .handle_control_frame(&provider_ctx, goodbye)
11042            .await
11043            .unwrap();
11044        assert_eq!(
11045            handler.capability_evaluator.verdict("cons", "thing/v1"),
11046            Some(CapabilityVerdict::NeverProvided)
11047        );
11048        let closed = client_rx
11049            .try_recv()
11050            .expect("GOODBYE pushes route.closed before route GOODBYE");
11051        assert!(
11052            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11053            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11054        );
11055        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11056        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11057    }
11058
11059    #[tokio::test]
11060    async fn dropping_router_connection_releases_registration() {
11061        let registry = Arc::new(Registry::default());
11062        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
11063        let router = Router::with_control_handler(control);
11064        let connection = router.begin_connection();
11065        let (ctx, mut rx) = route_ctx(connection.id());
11066
11067        router
11068            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11069            .await
11070            .unwrap();
11071        let response = rx.recv().await.unwrap();
11072        let ack = parse_ack(&response);
11073        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11074        assert!(registry.get_module("aft").unwrap().is_some());
11075
11076        drop(connection);
11077
11078        assert!(registry.get_module("aft").unwrap().is_none());
11079        assert_eq!(registry.active_registration_count().unwrap(), 0);
11080    }
11081
11082    fn capability_manifest(
11083        module_id: &str,
11084        provides: &[&str],
11085        must_never_reach: &[&str],
11086    ) -> ModuleManifest {
11087        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11088        manifest.capabilities = Some(CapabilityDeclarations {
11089            provides: provides
11090                .iter()
11091                .map(|capability| (*capability).to_string())
11092                .collect(),
11093            requires: Vec::new(),
11094            must_never_reach: must_never_reach
11095                .iter()
11096                .map(|capability| (*capability).to_string())
11097                .collect(),
11098        });
11099        manifest
11100    }
11101
11102    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11103        Frame::build(
11104            FrameType::Hello,
11105            control_flags(),
11106            0,
11107            0,
11108            corr,
11109            serde_json::to_vec(&ModuleHelloBody {
11110                protocol_ver: manifest.protocol_ver,
11111                manifest,
11112                control_ops: None,
11113                launch_nonce: None,
11114            })
11115            .expect("capability test HELLO serializes"),
11116        )
11117        .expect("capability test HELLO frame builds")
11118    }
11119
11120    fn catalog_update_with_capabilities_frame(
11121        corr: u64,
11122        capabilities: CapabilityDeclarations,
11123    ) -> Frame {
11124        Frame::build(
11125            FrameType::Request,
11126            control_flags(),
11127            0,
11128            0,
11129            corr,
11130            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11131                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11132                capabilities: Some(capabilities),
11133                ready: None,
11134            })
11135            .expect("capability catalog.update serializes"),
11136        )
11137        .expect("capability catalog.update frame builds")
11138    }
11139
11140    async fn register_capability_manifest(
11141        handler: &ControlHandler,
11142        ctx: &RouteCtx,
11143        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11144        manifest: ModuleManifest,
11145        corr: u64,
11146    ) {
11147        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11148    }
11149
11150    async fn open_route_for_capability_test(
11151        handler: &ControlHandler,
11152        target_ctx: &RouteCtx,
11153        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11154        client_connection_id: u64,
11155        corr: u64,
11156        target_module_id: &str,
11157        consumer_identity: Option<ConsumerIdentity>,
11158    ) -> (
11159        mpsc::Receiver<crate::router::OutboundFrame>,
11160        ModuleControlRequest,
11161    ) {
11162        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11163        let route_handler = handler.clone();
11164        let target_module_id = target_module_id.to_string();
11165        let route_task = tokio::spawn(async move {
11166            route_handler
11167                .handle_control_frame(
11168                    &client_ctx,
11169                    route_open_frame_with_admission_facts(
11170                        corr,
11171                        &target_module_id,
11172                        unique_project_root("admission-facts"),
11173                        consumer_identity,
11174                        None,
11175                    ),
11176                )
11177                .await
11178                .expect("capability test route.open succeeds")
11179        });
11180        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
11181            .await
11182            .expect("capability test route.open must reach route.bind")
11183            .expect("target control receiver stays open");
11184        let bind_request: ModuleControlRequest =
11185            serde_json::from_slice(&bind.body).expect("route.bind decodes");
11186        handler
11187            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
11188            .await
11189            .expect("capability test route.bind ACK succeeds");
11190        assert!(route_task.await.expect("route.open task joins").is_empty());
11191        let opened = client_rx
11192            .recv()
11193            .await
11194            .expect("successful route.open publishes a response");
11195        assert!(matches!(
11196            serde_json::from_slice::<ClientControlResponse>(&opened.body),
11197            Ok(ClientControlResponse::RouteOpen { .. })
11198        ));
11199        (client_rx, bind_request)
11200    }
11201
11202    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
11203        assert_eq!(frame.header.ty, FrameType::Push);
11204        assert_eq!(frame.header.channel, 0);
11205        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
11206            .expect("route.closed control push decodes");
11207        let ClientControlPush::RouteClosed { channels, .. } = &push else {
11208            panic!("expected route.closed");
11209        };
11210        assert_eq!(channels.len(), 1, "exactly one violating route closed");
11211        let channels = channels.clone();
11212        assert_eq!(
11213            push,
11214            ClientControlPush::RouteClosed {
11215                module_id: target_module_id.to_string(),
11216                channels,
11217                reason: RouteCloseReason::CapabilityDenied,
11218                drained: false,
11219                abandoned: 0,
11220                excluded_subscriptions: 0,
11221                terminal: Some(false),
11222            }
11223        );
11224    }
11225
11226    #[tokio::test]
11227    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
11228        let registry = Arc::new(Registry::default());
11229        let forwarding = Arc::new(ForwardingTable::default());
11230        let supervisor = SupervisorHandle::new();
11231        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11232        let handler =
11233            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11234                .with_supervisor(supervisor);
11235        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
11236        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
11237        register_capability_manifest(
11238            &handler,
11239            &target_ctx,
11240            &mut target_rx,
11241            capability_manifest("target", &["credentials-provider/v1"], &[]),
11242            1,
11243        )
11244        .await;
11245        register_capability_manifest(
11246            &handler,
11247            &opener_ctx,
11248            &mut opener_rx,
11249            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11250            2,
11251        )
11252        .await;
11253
11254        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
11255        let replies = handler
11256            .handle_control_frame(
11257                &client_ctx,
11258                route_open_frame_with_admission_facts(
11259                    3,
11260                    "target",
11261                    unique_project_root("admission-facts"),
11262                    Some(ConsumerIdentity {
11263                        module_id: "opener".to_string(),
11264                        launch_nonce: "opener-nonce".to_string(),
11265                    }),
11266                    None,
11267                ),
11268            )
11269            .await
11270            .expect("denied route.open returns a typed frame");
11271        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11272        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11273        assert!(
11274            target_rx.try_recv().is_err(),
11275            "forbidden route.open must not relay route.bind"
11276        );
11277    }
11278
11279    #[tokio::test]
11280    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
11281        let registry = Arc::new(Registry::default());
11282        let forwarding = Arc::new(ForwardingTable::default());
11283        let supervisor = SupervisorHandle::new();
11284        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11285        let handler =
11286            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11287                .with_supervisor(supervisor);
11288        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
11289        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
11290        register_capability_manifest(
11291            &handler,
11292            &target_ctx,
11293            &mut target_rx,
11294            capability_manifest("target", &["credentials-provider/v1"], &[]),
11295            1,
11296        )
11297        .await;
11298        register_capability_manifest(
11299            &handler,
11300            &old_opener_ctx,
11301            &mut old_opener_rx,
11302            capability_manifest("opener", &[], &[]),
11303            2,
11304        )
11305        .await;
11306        let (mut client_rx, _) = open_route_for_capability_test(
11307            &handler,
11308            &target_ctx,
11309            &mut target_rx,
11310            712,
11311            3,
11312            "target",
11313            Some(ConsumerIdentity {
11314                module_id: "opener".to_string(),
11315                launch_nonce: "opener-nonce".to_string(),
11316            }),
11317        )
11318        .await;
11319        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11320
11321        handler
11322            .cleanup_connection(old_opener_ctx.connection_id)
11323            .expect("old opener registration cleans up");
11324        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
11325        register_capability_manifest(
11326            &handler,
11327            &new_opener_ctx,
11328            &mut new_opener_rx,
11329            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11330            4,
11331        )
11332        .await;
11333
11334        assert_capability_denied_push(
11335            client_rx
11336                .try_recv()
11337                .expect("HELLO deny addition must emit route.closed")
11338                .frame,
11339            "target",
11340        );
11341        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11342        assert!(matches!(
11343            target_rx.try_recv(),
11344            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11345        ));
11346    }
11347
11348    #[tokio::test]
11349    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
11350        let registry = Arc::new(Registry::default());
11351        let forwarding = Arc::new(ForwardingTable::default());
11352        let supervisor = SupervisorHandle::new();
11353        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11354        let handler =
11355            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11356                .with_supervisor(supervisor);
11357        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
11358        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
11359        register_capability_manifest(
11360            &handler,
11361            &target_ctx,
11362            &mut target_rx,
11363            capability_manifest("target", &[], &[]),
11364            1,
11365        )
11366        .await;
11367        register_capability_manifest(
11368            &handler,
11369            &opener_ctx,
11370            &mut opener_rx,
11371            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11372            2,
11373        )
11374        .await;
11375        let (mut client_rx, _) = open_route_for_capability_test(
11376            &handler,
11377            &target_ctx,
11378            &mut target_rx,
11379            722,
11380            3,
11381            "target",
11382            Some(ConsumerIdentity {
11383                module_id: "opener".to_string(),
11384                launch_nonce: "opener-nonce".to_string(),
11385            }),
11386        )
11387        .await;
11388        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11389
11390        let replies = handler
11391            .handle_control_frame(
11392                &target_ctx,
11393                catalog_update_with_capabilities_frame(
11394                    4,
11395                    CapabilityDeclarations {
11396                        provides: vec!["credentials-provider/v1".to_string()],
11397                        requires: Vec::new(),
11398                        must_never_reach: Vec::new(),
11399                    },
11400                ),
11401            )
11402            .await
11403            .expect("claim catalog.update succeeds");
11404        assert!(matches!(
11405            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
11406            Ok(ModuleControlResponseToModule::CatalogUpdate {})
11407        ));
11408        assert_capability_denied_push(
11409            client_rx
11410                .try_recv()
11411                .expect("claim addition must emit route.closed")
11412                .frame,
11413            "target",
11414        );
11415        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11416        assert!(matches!(
11417            target_rx.try_recv(),
11418            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11419        ));
11420    }
11421
11422    #[tokio::test]
11423    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
11424        let registry = Arc::new(Registry::default());
11425        let forwarding = Arc::new(ForwardingTable::default());
11426        let supervisor = SupervisorHandle::new();
11427        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11428        let handler =
11429            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11430                .with_supervisor(supervisor);
11431        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11432        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11433        register_capability_manifest(
11434            &handler,
11435            &target_ctx,
11436            &mut target_rx,
11437            capability_manifest("target", &["credentials-provider/v1"], &[]),
11438            1,
11439        )
11440        .await;
11441        register_capability_manifest(
11442            &handler,
11443            &opener_ctx,
11444            &mut opener_rx,
11445            capability_manifest("opener", &[], &[]),
11446            2,
11447        )
11448        .await;
11449        let (mut client_rx, _) = open_route_for_capability_test(
11450            &handler,
11451            &target_ctx,
11452            &mut target_rx,
11453            732,
11454            3,
11455            "target",
11456            Some(ConsumerIdentity {
11457                module_id: "opener".to_string(),
11458                launch_nonce: "opener-nonce".to_string(),
11459            }),
11460        )
11461        .await;
11462        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11463
11464        handler
11465            .handle_control_frame(
11466                &target_ctx,
11467                catalog_update_with_capabilities_frame(
11468                    4,
11469                    CapabilityDeclarations {
11470                        provides: Vec::new(),
11471                        requires: Vec::new(),
11472                        must_never_reach: Vec::new(),
11473                    },
11474                ),
11475            )
11476            .await
11477            .expect("claim removal catalog.update succeeds");
11478        assert_eq!(
11479            forwarding.active_binding_count().unwrap(),
11480            1,
11481            "removing an attested target claim must leave the route census unchanged"
11482        );
11483        assert!(
11484            client_rx.try_recv().is_err(),
11485            "claim removal must not emit route.closed capability_denied"
11486        );
11487        assert!(
11488            target_rx.try_recv().is_err(),
11489            "claim removal must not send the target a route GOODBYE"
11490        );
11491    }
11492
11493    /// A direct client may open a route to a denied capability provider; this
11494    /// policy applies only to attested supervised module origins, not to direct clients.
11495    #[tokio::test]
11496    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11497        let registry = Arc::new(Registry::default());
11498        let forwarding = Arc::new(ForwardingTable::default());
11499        let supervisor = SupervisorHandle::new();
11500        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11501        let handler =
11502            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11503                .with_supervisor(supervisor);
11504        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11505        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11506        register_capability_manifest(
11507            &handler,
11508            &target_ctx,
11509            &mut target_rx,
11510            capability_manifest("target", &["credentials-provider/v1"], &[]),
11511            1,
11512        )
11513        .await;
11514        register_capability_manifest(
11515            &handler,
11516            &opener_ctx,
11517            &mut opener_rx,
11518            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11519            2,
11520        )
11521        .await;
11522
11523        let (_client_rx, bind) = open_route_for_capability_test(
11524            &handler,
11525            &target_ctx,
11526            &mut target_rx,
11527            742,
11528            3,
11529            "target",
11530            None,
11531        )
11532        .await;
11533        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11534            panic!("direct scope-honesty route must bind");
11535        };
11536        assert_eq!(principal, Some(Principal::Direct));
11537        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11538    }
11539
11540    /// A module that denies a capability receives no self-route exemption when it
11541    /// also attestedly provides that capability.
11542    #[tokio::test]
11543    async fn must_never_reach_self_route_is_capability_forbidden() {
11544        let registry = Arc::new(Registry::default());
11545        let forwarding = Arc::new(ForwardingTable::default());
11546        let supervisor = SupervisorHandle::new();
11547        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11548        let handler =
11549            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11550                .with_supervisor(supervisor);
11551        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11552        register_capability_manifest(
11553            &handler,
11554            &self_ctx,
11555            &mut self_rx,
11556            capability_manifest(
11557                "self-provider",
11558                &["credentials-provider/v1"],
11559                &["credentials-provider/v1"],
11560            ),
11561            1,
11562        )
11563        .await;
11564
11565        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11566        let replies = handler
11567            .handle_control_frame(
11568                &client_ctx,
11569                route_open_frame_with_admission_facts(
11570                    2,
11571                    "self-provider",
11572                    unique_project_root("admission-facts"),
11573                    Some(ConsumerIdentity {
11574                        module_id: "self-provider".to_string(),
11575                        launch_nonce: "self-nonce".to_string(),
11576                    }),
11577                    None,
11578                ),
11579            )
11580            .await
11581            .expect("self-route refusal returns a typed frame");
11582        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11583        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11584        assert!(
11585            self_rx.try_recv().is_err(),
11586            "self denial must not relay route.bind"
11587        );
11588    }
11589
11590    #[test]
11591    fn unsupported_channel_zero_frame_returns_error() {
11592        let handler = ControlHandler::default();
11593        let request = Frame::build(
11594            FrameType::Request,
11595            control_flags(),
11596            0,
11597            0,
11598            21,
11599            b"opaque".to_vec(),
11600        )
11601        .unwrap();
11602
11603        let response = handler
11604            .handle_control(ConnectionId::new(1), request)
11605            .unwrap();
11606
11607        assert_eq!(response[0].header.ty, FrameType::Error);
11608        assert_eq!(
11609            parse_error(&response[0])["code"],
11610            "unsupported_control_frame"
11611        );
11612    }
11613
11614    /// Blue/green swap at the control-plane boundary. The supervisor that opens
11615    /// a swap is not wired yet, so the candidate is registered here directly
11616    /// into the registry and forwarding candidate slots, the way the swap's
11617    /// HELLO admission will.
11618    mod swap {
11619        use super::*;
11620
11621        const INCUMBENT: ConnectionId = ConnectionId::new(30);
11622        const CANDIDATE: ConnectionId = ConnectionId::new(40);
11623
11624        struct Swap {
11625            registry: Arc<Registry>,
11626            forwarding: Arc<ForwardingTable>,
11627            handler: ControlHandler,
11628            incumbent_ctx: RouteCtx,
11629            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11630            candidate_ctx: RouteCtx,
11631            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11632        }
11633
11634        async fn swap_with_incumbent() -> Swap {
11635            let registry = Arc::new(Registry::default());
11636            let forwarding = Arc::new(ForwardingTable::default());
11637            let handler =
11638                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11639            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11640            hello_via_sink(
11641                &handler,
11642                &incumbent_ctx,
11643                &mut incumbent_rx,
11644                hello_frame("aft", PROTOCOL_VERSION, 7),
11645            )
11646            .await;
11647            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11648            Swap {
11649                registry,
11650                forwarding,
11651                handler,
11652                incumbent_ctx,
11653                incumbent_rx,
11654                candidate_ctx,
11655                candidate_rx,
11656            }
11657        }
11658
11659        fn register_candidate(swap: &Swap, ready: Option<bool>) {
11660            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11661            candidate_manifest.ready = ready;
11662            let registration = swap
11663                .registry
11664                .register_candidate_with_control_ops(
11665                    candidate_manifest,
11666                    PROTOCOL_VERSION,
11667                    CANDIDATE,
11668                    module_baseline_control_ops(),
11669                )
11670                .unwrap();
11671            swap.forwarding
11672                .register_candidate_module_connection(
11673                    CANDIDATE,
11674                    "aft".to_string(),
11675                    PROTOCOL_VERSION,
11676                    manifest_concurrency(&registration.manifest),
11677                    swap.candidate_ctx.egress.clone(),
11678                )
11679                .unwrap();
11680        }
11681
11682        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11683            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11684            swap.registry.promote_candidate("aft").unwrap().unwrap();
11685            cutover.incumbent.unwrap()
11686        }
11687
11688        fn keyed_total(counters: &Value, key: &str) -> u64 {
11689            counters[key]
11690                .as_object()
11691                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11692                .unwrap_or(0)
11693        }
11694
11695        /// An ack from the incumbent for a bind it was sent before cutover,
11696        /// arriving before the incumbent is drained. The incumbent is the live
11697        /// connection carrying every other client's routes, so the ack must
11698        /// not end it: the waiting client is told to retry, the reservation is
11699        /// given back, and the incumbent is told to drop just that binding.
11700        #[tokio::test]
11701        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11702            let mut swap = swap_with_incumbent().await;
11703            let handler = swap.handler.clone();
11704
11705            // A co-tenant route, bound on the incumbent before the swap.
11706            let cotenant = ConnectionId::new(31);
11707            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11708            let (cotenant_task, cotenant_bind) = relay_route_open(
11709                &handler,
11710                cotenant,
11711                &cotenant_ctx.egress,
11712                &mut swap.incumbent_rx,
11713                100,
11714                "aft",
11715                "swap-cotenant",
11716            )
11717            .await;
11718            handler
11719                .handle_control_frame(
11720                    &swap.incumbent_ctx,
11721                    route_bind_ack(cotenant_bind.header.corr),
11722                )
11723                .await
11724                .unwrap();
11725            assert!(cotenant_task.await.unwrap().is_empty());
11726            let (cotenant_channel, cotenant_epoch) =
11727                published_route(&cotenant_rx.recv().await.unwrap());
11728
11729            // A second route.open, relayed to the incumbent and not yet acked.
11730            let caller = ConnectionId::new(32);
11731            let (caller_ctx, mut caller_rx) = route_ctx(caller);
11732            let (caller_task, caller_bind) = relay_route_open(
11733                &handler,
11734                caller,
11735                &caller_ctx.egress,
11736                &mut swap.incumbent_rx,
11737                101,
11738                "aft",
11739                "swap-caller",
11740            )
11741            .await;
11742            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11743
11744            register_candidate(&swap, None);
11745            cutover(&swap);
11746
11747            // The incumbent acks after promotion and before any drain.
11748            let ack = handler
11749                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11750                .await;
11751            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11752            if module_loop_error.is_some() {
11753                // What the connection loop does with an untranslated router
11754                // error: end the connection, releasing every route on it.
11755                handler.cleanup_connection(INCUMBENT).unwrap();
11756            }
11757
11758            // 1. The incumbent's other routes survive.
11759            assert!(
11760                cotenant_rx.try_recv().is_err(),
11761                "the co-tenant route on the incumbent was torn down by one late ack: \
11762                 {module_loop_error:?}"
11763            );
11764            assert!(matches!(
11765                swap.forwarding
11766                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11767                    .unwrap(),
11768                DataRoute::Client(DataRouteState::Bound(_))
11769            ));
11770            assert_eq!(module_loop_error, None);
11771            assert!(swap
11772                .registry
11773                .get_module_by_connection(INCUMBENT)
11774                .unwrap()
11775                .is_some());
11776
11777            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
11778            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11779                .await
11780                .expect("the incumbent is told to drop the abandoned binding")
11781                .unwrap()
11782                .frame;
11783            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11784            assert_eq!(goodbye.header.channel, abandoned_channel);
11785            assert_eq!(goodbye.header.epoch, abandoned_epoch);
11786            assert!(swap.incumbent_rx.try_recv().is_err());
11787
11788            // 3. The waiting client gets a retryable refusal and no route.
11789            let response = caller_task.await.unwrap();
11790            assert_eq!(response.len(), 1);
11791            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11792            assert!(caller_rx.try_recv().is_err());
11793
11794            // 4. The reservation pair is given back, and the pending bind
11795            //    settled exactly once: one accepted open (the co-tenant) and one
11796            //    refused open (the caller), nothing counted twice.
11797            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11798            let counters = handler.counters().snapshot();
11799            assert_eq!(
11800                keyed_total(&counters, "route_open_accepted_by_principal"),
11801                1
11802            );
11803            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11804            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11805        }
11806
11807        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
11808        /// id would resolve to the promoted candidate and every new route.open
11809        /// would be refused as reloading, leaving neither process routable.
11810        #[tokio::test]
11811        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11812            let mut swap = swap_with_incumbent().await;
11813            register_candidate(&swap, None);
11814            let incumbent = cutover(&swap);
11815            swap.forwarding
11816                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11817                .unwrap()
11818                .expect("the incumbent is still registered");
11819
11820            let client = ConnectionId::new(33);
11821            let (client_ctx, mut client_rx) = route_ctx(client);
11822            let route_handler = swap.handler.clone();
11823            let open_ctx = RouteCtx {
11824                connection_id: client,
11825                egress: client_ctx.egress.clone(),
11826            };
11827            let mut route_task = tokio::spawn(async move {
11828                route_handler
11829                    .handle_control_frame(
11830                        &open_ctx,
11831                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11832                    )
11833                    .await
11834                    .unwrap()
11835            });
11836            let bind = tokio::select! {
11837                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11838                response = &mut route_task => {
11839                    let response = response.unwrap();
11840                    panic!(
11841                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
11842                        parse_error(&response[0])["code"]
11843                    );
11844                }
11845            };
11846            swap.handler
11847                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11848                .await
11849                .unwrap();
11850            assert!(route_task.await.unwrap().is_empty());
11851            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11852            match swap
11853                .forwarding
11854                .lookup_data_route(client, channel, epoch)
11855                .unwrap()
11856            {
11857                DataRoute::Client(DataRouteState::Bound(route)) => {
11858                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11859                }
11860                other => panic!("expected a bound route on the candidate, got {other:?}"),
11861            }
11862            assert!(swap.incumbent_rx.try_recv().is_err());
11863        }
11864
11865        /// A candidate declares itself ready with `catalog.update` on its own
11866        /// connection. If the connection-keyed registry lookups searched only the
11867        /// active slot, this would answer `not_registered` and the candidate
11868        /// would never become ready.
11869        #[tokio::test]
11870        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11871            let swap = swap_with_incumbent().await;
11872            register_candidate(&swap, Some(false));
11873            let update = Frame::build(
11874                FrameType::Request,
11875                control_flags(),
11876                0,
11877                0,
11878                55,
11879                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11880                    provides: manifest("aft", PROTOCOL_VERSION).provides,
11881                    capabilities: None,
11882                    ready: Some(true),
11883                })
11884                .unwrap(),
11885            )
11886            .unwrap();
11887
11888            let replies = swap
11889                .handler
11890                .handle_control_frame(&swap.candidate_ctx, update)
11891                .await
11892                .unwrap();
11893
11894            assert_eq!(replies.len(), 1);
11895            assert_eq!(
11896                replies[0].header.ty,
11897                FrameType::Response,
11898                "candidate catalog.update was refused: {:?}",
11899                serde_json::from_slice::<Value>(&replies[0].body).ok()
11900            );
11901            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11902            assert_eq!(
11903                swap.registry
11904                    .get_module("aft")
11905                    .unwrap()
11906                    .unwrap()
11907                    .connection_id,
11908                INCUMBENT
11909            );
11910        }
11911    }
11912
11913    /// The HELLO gate while the supervisor has a swap open: only the nonce it
11914    /// minted for the candidate admits a second process, into the candidate
11915    /// slot, and that check runs ahead of the reserved-module gate.
11916    mod swap_admission {
11917        use super::*;
11918
11919        const INCUMBENT_NONCE: &str = "incumbent-nonce";
11920        const CANDIDATE_NONCE: &str = "candidate-nonce";
11921
11922        fn handler_with_incumbent(
11923            module_id: &str,
11924            reserved: bool,
11925        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11926            let registry = Arc::new(Registry::default());
11927            let supervisor = SupervisorHandle::new();
11928            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11929            if reserved {
11930                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11931            }
11932            let handler =
11933                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
11934            let incumbent = handler
11935                .handle_control(
11936                    ConnectionId::new(1),
11937                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11938                )
11939                .unwrap();
11940            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11941            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11942            (registry, supervisor, handler)
11943        }
11944
11945        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
11946        /// admits every nonce, so while a swap is open the swap gate is the only
11947        /// thing between a key-holder and the candidate slot. A nonce the
11948        /// supervisor did not mint, or none at all, is refused, and neither the
11949        /// incumbent's registration nor the candidate slot moves.
11950        #[test]
11951        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11952            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11953
11954            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11955                let replies = handler
11956                    .handle_control(
11957                        ConnectionId::new(connection),
11958                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11959                    )
11960                    .unwrap();
11961                assert_eq!(replies[0].header.ty, FrameType::Error);
11962                assert_eq!(
11963                    parse_error(&replies[0])["code"],
11964                    "swap_token_invalid",
11965                    "nonce {nonce:?}"
11966                );
11967            }
11968            assert!(registry.get_candidate("aft").unwrap().is_none());
11969            assert_eq!(
11970                registry.get_module("aft").unwrap().unwrap().connection_id,
11971                ConnectionId::new(1)
11972            );
11973
11974            // Control: the minted token is admitted, into the candidate slot,
11975            // and only once.
11976            let admitted = handler
11977                .handle_control(
11978                    ConnectionId::new(4),
11979                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
11980                )
11981                .unwrap();
11982            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
11983            assert_eq!(
11984                registry
11985                    .get_candidate("aft")
11986                    .unwrap()
11987                    .unwrap()
11988                    .connection_id,
11989                ConnectionId::new(4)
11990            );
11991            assert_eq!(
11992                registry.get_module("aft").unwrap().unwrap().connection_id,
11993                ConnectionId::new(1),
11994                "the candidate must not take the active slot"
11995            );
11996            let replayed = handler
11997                .handle_control(
11998                    ConnectionId::new(5),
11999                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12000                )
12001                .unwrap();
12002            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12003
12004            // The case only this gate covers: the incumbent has died mid-swap,
12005            // so its duplicate refusal is gone too, and without the gate a
12006            // key-holder would take the id's ACTIVE slot.
12007            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12008            let squatter = handler
12009                .handle_control(
12010                    ConnectionId::new(6),
12011                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12012                )
12013                .unwrap();
12014            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12015            assert!(
12016                registry.get_module("aft").unwrap().is_none(),
12017                "a squatter took the active slot of an id being swapped"
12018            );
12019        }
12020
12021        /// Design mutation arm (iii). A reserved module's candidate presents a
12022        /// nonce the reserved gate has never seen (that gate holds the
12023        /// incumbent's), so the swap gate must run first or the candidate is
12024        /// refused `reserved_module` and a reserved module can never be swapped.
12025        #[test]
12026        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12027            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12028
12029            let replies = handler
12030                .handle_control(
12031                    ConnectionId::new(2),
12032                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12033                )
12034                .unwrap();
12035
12036            assert_eq!(
12037                replies[0].header.ty,
12038                FrameType::HelloAck,
12039                "reserved candidate refused: {:?}",
12040                serde_json::from_slice::<Value>(&replies[0].body).ok()
12041            );
12042            assert_eq!(
12043                registry
12044                    .get_candidate("vault")
12045                    .unwrap()
12046                    .unwrap()
12047                    .connection_id,
12048                ConnectionId::new(2)
12049            );
12050        }
12051
12052        /// With no swap open the gate is inert: the incumbent's reserved gate
12053        /// and duplicate refusal behave exactly as before.
12054        #[test]
12055        fn without_an_open_swap_the_ordinary_gates_decide() {
12056            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12057            supervisor.close_swap("vault");
12058
12059            let candidate = handler
12060                .handle_control(
12061                    ConnectionId::new(2),
12062                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12063                )
12064                .unwrap();
12065            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12066            let duplicate = handler
12067                .handle_control(
12068                    ConnectionId::new(3),
12069                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12070                )
12071                .unwrap();
12072            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12073            assert!(registry.get_candidate("vault").unwrap().is_none());
12074        }
12075    }
12076
12077    /// `scope.sync` and `scope.describe` through the real control handler: who
12078    /// may sync is decided by the registration and launch nonce of the module
12079    /// connection, never by the request body.
12080    mod scopes {
12081        use subc_protocol::scope::{
12082            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12083            ScopeStatus,
12084        };
12085
12086        use super::*;
12087
12088        const OWNER: &str = "prefrontal-core";
12089
12090        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12091            ScopeRecord {
12092                scope_ref: scope_ref.to_string(),
12093                scope_epoch,
12094                kind: ScopeKind::Head,
12095                parent: None,
12096                child_owners: Vec::new(),
12097                carriers: Vec::new(),
12098                attributes: Default::default(),
12099            }
12100        }
12101
12102        async fn call(
12103            handler: &ControlHandler,
12104            ctx: &RouteCtx,
12105            request: &ModuleControlRequestFromModule,
12106        ) -> Frame {
12107            let body = serde_json::to_vec(request).unwrap();
12108            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12109            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12110            assert_eq!(replies.len(), 1, "{replies:?}");
12111            replies.pop().unwrap()
12112        }
12113
12114        async fn sync(
12115            handler: &ControlHandler,
12116            ctx: &RouteCtx,
12117            generation: u64,
12118            scopes: Vec<ScopeRecord>,
12119        ) -> Result<ModuleControlResponseToModule, String> {
12120            let reply = call(
12121                handler,
12122                ctx,
12123                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
12124            )
12125            .await;
12126            match reply.header.ty {
12127                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
12128                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
12129            }
12130        }
12131
12132        async fn describe(
12133            handler: &ControlHandler,
12134            ctx: &RouteCtx,
12135            owner: &str,
12136            scope_ref: &str,
12137        ) -> ModuleControlResponseToModule {
12138            let reply = call(
12139                handler,
12140                ctx,
12141                &ModuleControlRequestFromModule::ScopeDescribe {
12142                    owner: Principal::Reserved {
12143                        module_id: owner.to_string(),
12144                    },
12145                    scope_ref: scope_ref.to_string(),
12146                },
12147            )
12148            .await;
12149            assert_eq!(
12150                reply.header.ty,
12151                FrameType::Response,
12152                "{:?}",
12153                parse_error(&reply)
12154            );
12155            serde_json::from_slice(&reply.body).unwrap()
12156        }
12157
12158        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
12159        async fn module(
12160            handler: &ControlHandler,
12161            connection: u64,
12162            module_id: &str,
12163            nonce: Option<&str>,
12164        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12165            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
12166            hello_via_sink(
12167                handler,
12168                &ctx,
12169                &mut rx,
12170                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
12171            )
12172            .await;
12173            (ctx, rx)
12174        }
12175
12176        /// `direct` and every other client connection has no registration, so
12177        /// it can neither sync nor own a scope.
12178        #[tokio::test]
12179        async fn a_client_connection_cannot_sync_or_describe() {
12180            let handler = ControlHandler::new(Arc::new(Registry::default()));
12181            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
12182            for request in [
12183                ModuleControlRequestFromModule::ScopeSync {
12184                    generation: 1,
12185                    scopes: vec![head("s", 1)],
12186                },
12187                ModuleControlRequestFromModule::ScopeDescribe {
12188                    owner: Principal::Direct,
12189                    scope_ref: "s".to_string(),
12190                },
12191            ] {
12192                let reply = call(&handler, &ctx, &request).await;
12193                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
12194            }
12195            assert!(
12196                !handler
12197                    .scopes
12198                    .read()
12199                    .unwrap()
12200                    .describe(
12201                        &Principal::Reserved {
12202                            module_id: OWNER.to_string()
12203                        },
12204                        "s"
12205                    )
12206                    .owner_synced
12207            );
12208        }
12209
12210        /// A module the supervisor did not spawn registers without a launch
12211        /// nonce, so it is never an owner's current launch.
12212        #[tokio::test]
12213        async fn a_module_without_a_supervised_launch_cannot_sync() {
12214            let handler = ControlHandler::new(Arc::new(Registry::default()));
12215            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
12216            assert_eq!(
12217                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
12218                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12219            );
12220        }
12221
12222        #[tokio::test]
12223        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
12224            let supervisor = SupervisorHandle::new();
12225            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12226            let handler = ControlHandler::new(Arc::new(Registry::default()))
12227                .with_supervisor(supervisor.clone());
12228            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12229            sync(&handler, &incumbent, 1, vec![head("s", 1)])
12230                .await
12231                .expect("the current launch syncs");
12232
12233            // A swap candidate registers with the swap token and is refused
12234            // while the incumbent keeps syncing.
12235            supervisor.open_swap(OWNER, "n2".to_string());
12236            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
12237            assert_eq!(
12238                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12239                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12240            );
12241            sync(&handler, &incumbent, 2, vec![head("s", 1)])
12242                .await
12243                .expect("the serving owner syncs during the swap");
12244
12245            // The swap fails and is rolled back. The candidate never held sync
12246            // authority, and still cannot sync.
12247            supervisor.close_swap(OWNER);
12248            assert_eq!(
12249                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12250                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12251            );
12252            sync(&handler, &incumbent, 3, vec![head("s", 1)])
12253                .await
12254                .expect("the serving owner syncs after the rollback");
12255            handler.cleanup_connection(candidate.connection_id).unwrap();
12256
12257            // A swap that cuts over. Promotion records the candidate's nonce as
12258            // the module's spawn nonce, which is what `set_spawn_nonce` does
12259            // here; the promoted connection then takes authority at any
12260            // generation and the superseded incumbent is refused.
12261            supervisor.open_swap(OWNER, "n3".to_string());
12262            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
12263            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
12264            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
12265                .await
12266                .expect("the promoted launch takes authority");
12267            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12268                panic!("unexpected reply {reply:?}");
12269            };
12270            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
12271            assert_eq!(
12272                sync(&handler, &incumbent, 4, Vec::new()).await,
12273                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12274            );
12275        }
12276
12277        /// Authority dies with its connection: the cleanup path releases it,
12278        /// so the owner's next connection takes it at any generation.
12279        #[tokio::test]
12280        async fn closing_the_authority_connection_frees_sync_authority() {
12281            let supervisor = SupervisorHandle::new();
12282            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12283            let handler = ControlHandler::new(Arc::new(Registry::default()))
12284                .with_supervisor(supervisor.clone());
12285            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12286            sync(&handler, &first, 10, vec![head("s", 1)])
12287                .await
12288                .unwrap();
12289            handler.cleanup_connection(first.connection_id).unwrap();
12290
12291            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
12292            sync(&handler, &second, 1, vec![head("s", 1)])
12293                .await
12294                .expect("the next connection takes the released authority");
12295        }
12296
12297        #[tokio::test]
12298        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
12299            let supervisor = SupervisorHandle::new();
12300            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12301            let handler =
12302                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
12303            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
12304            sync(&handler, &first, 10, vec![head("s", 1)])
12305                .await
12306                .unwrap();
12307            handler
12308                .handle_control_frame(
12309                    &first,
12310                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
12311                )
12312                .await
12313                .unwrap();
12314            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
12315            sync(&handler, &second, 1, vec![head("s", 1)])
12316                .await
12317                .expect("GOODBYE releases authority even if the old socket remains open");
12318        }
12319
12320        #[tokio::test]
12321        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
12322            let registry = Arc::new(Registry::default());
12323            let supervisor_handle = SupervisorHandle::new();
12324            let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
12325                .with_handle(supervisor_handle.clone())
12326                .with_daemon_incarnation("incarnation-7".to_string());
12327            // Configured with enabled: false, so the supervisor lists the
12328            // module without spawning a process for it.
12329            supervisor
12330                .supervise_configured(
12331                    ModuleSpec {
12332                        module_id: OWNER.to_string(),
12333                        program: PathBuf::from("/nonexistent/prefrontal-core"),
12334                        args: Vec::new(),
12335                        env: Vec::new(),
12336                        reserved: false,
12337                        reserved_prefixes: Vec::new(),
12338                        protocol: ModuleProtocol::Subc,
12339                        overlap: Default::default(),
12340                    },
12341                    false,
12342                )
12343                .unwrap();
12344            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
12345            let handler =
12346                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12347            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
12348
12349            // Configured but not yet synced: a reader waits for the owner.
12350            let ModuleControlResponseToModule::ScopeDescribe {
12351                status,
12352                daemon_incarnation,
12353                owner_synced,
12354                owner_configured,
12355                scope,
12356                ..
12357            } = describe(&handler, &reader, OWNER, "s").await
12358            else {
12359                panic!("not a describe reply");
12360            };
12361            assert_eq!(status, ScopeStatus::NotLive);
12362            assert_eq!(daemon_incarnation, "incarnation-7");
12363            assert!(!owner_synced);
12364            assert!(owner_configured);
12365            assert!(scope.is_none());
12366
12367            // Not a supervised module: the owner will never sync, and a reader
12368            // refuses rather than waits.
12369            let ModuleControlResponseToModule::ScopeDescribe {
12370                status,
12371                owner_configured,
12372                ..
12373            } = describe(&handler, &reader, "ghost", "s").await
12374            else {
12375                panic!("not a describe reply");
12376            };
12377            assert_eq!(status, ScopeStatus::NotLive);
12378            assert!(!owner_configured);
12379
12380            // Live, with the stamp fields and the computed owner_authorized.
12381            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
12382            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
12383            let ModuleControlResponseToModule::ScopeDescribe {
12384                status,
12385                scope_epoch,
12386                owner_synced,
12387                scope,
12388                ..
12389            } = describe(&handler, &reader, OWNER, "s").await
12390            else {
12391                panic!("not a describe reply");
12392            };
12393            assert_eq!(status, ScopeStatus::Live);
12394            assert_eq!(scope_epoch, Some(4));
12395            assert!(owner_synced);
12396            let stamp = scope.expect("a live scope carries its stamp");
12397            assert!(
12398                stamp.owner_authorized,
12399                "prefrontal-core is the default authority"
12400            );
12401            assert_eq!(stamp.kind, ScopeKind::Head);
12402        }
12403
12404        #[tokio::test]
12405        async fn scope_authority_owners_decides_owner_authorized() {
12406            let supervisor = SupervisorHandle::new();
12407            supervisor.set_spawn_nonce("broca", "b1".to_string());
12408            let handler = ControlHandler::new(Arc::new(Registry::default()))
12409                .with_supervisor(supervisor)
12410                .with_scope_authority_owners(vec!["broca".to_string()]);
12411            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
12412            let mut gated = head("s", 1);
12413            gated.attributes.agent_id = Some("agent".to_string());
12414            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
12415            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
12416                describe(&handler, &broca, "broca", "s").await
12417            else {
12418                panic!("not a describe reply");
12419            };
12420            assert!(scope.unwrap().owner_authorized);
12421        }
12422
12423        /// With route admission, the stamp, the commit re-check and drains in
12424        /// place, the feature is advertised: the module ops in HELLO_ACK, and
12425        /// `scopes/v1` in HELLO_ACK and `server.describe`.
12426        #[tokio::test]
12427        async fn scope_ops_and_the_scopes_capability_are_advertised() {
12428            let handler = ControlHandler::new(Arc::new(Registry::default()));
12429            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
12430            let ack = hello_via_sink(
12431                &handler,
12432                &ctx,
12433                &mut rx,
12434                hello_frame("m", PROTOCOL_VERSION, 1),
12435            )
12436            .await;
12437            let ack = parse_ack(&ack);
12438            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
12439                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
12440            }
12441            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
12442
12443            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
12444            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12445            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12446            let reply = handler
12447                .handle_control_frame(&client, frame)
12448                .await
12449                .unwrap()
12450                .pop()
12451                .unwrap();
12452            let ClientControlResponse::ServerDescribe { capabilities, .. } =
12453                serde_json::from_slice(&reply.body).unwrap()
12454            else {
12455                panic!("not a server.describe reply");
12456            };
12457            assert!(
12458                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12459                "{capabilities:?}"
12460            );
12461        }
12462
12463        // ---- route admission, stamps, commit re-check and drains ----------
12464
12465        const PLEXUS: &str = "plexus";
12466        const OTHER: &str = "other";
12467        const AFT: &str = "aft";
12468        const BROCA: &str = "broca";
12469        const MAGIC: &str = "magic-context";
12470
12471        fn nonce(module_id: &str) -> String {
12472            format!("nonce-{module_id}")
12473        }
12474
12475        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12476            let (tx, rx) = mpsc::channel(64);
12477            (
12478                RouteCtx {
12479                    connection_id: ConnectionId::new(connection),
12480                    egress: FrameSink::new(tx),
12481                },
12482                rx,
12483            )
12484        }
12485
12486        /// A daemon with a configured owner (prefrontal-core) registered on its
12487        /// own module connection, two routable targets (plexus, other), and
12488        /// launch nonces minted for the modules that open routes as carriers.
12489        struct Rig {
12490            handler: ControlHandler,
12491            forwarding: Arc<ForwardingTable>,
12492            owner: RouteCtx,
12493            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12494            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12495            generation: u64,
12496            next_connection: u64,
12497            _supervisor: Supervisor,
12498        }
12499
12500        async fn rig() -> Rig {
12501            let registry = Arc::new(Registry::default());
12502            let forwarding = Arc::new(ForwardingTable::default());
12503            let supervisor_handle = SupervisorHandle::new();
12504            let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
12505                .with_handle(supervisor_handle.clone());
12506            supervisor
12507                .supervise_configured(
12508                    ModuleSpec {
12509                        module_id: OWNER.to_string(),
12510                        program: PathBuf::from("/nonexistent/prefrontal-core"),
12511                        args: Vec::new(),
12512                        env: Vec::new(),
12513                        reserved: false,
12514                        reserved_prefixes: Vec::new(),
12515                        protocol: ModuleProtocol::Subc,
12516                        overlap: Default::default(),
12517                    },
12518                    false,
12519                )
12520                .unwrap();
12521            for module_id in [OWNER, AFT, BROCA, MAGIC] {
12522                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12523            }
12524            let handler =
12525                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12526                    .with_supervisor(supervisor_handle);
12527            let (owner, mut owner_rx) = wide_ctx(1);
12528            hello_via_sink(
12529                &handler,
12530                &owner,
12531                &mut owner_rx,
12532                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12533            )
12534            .await;
12535            let mut modules = BTreeMap::new();
12536            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12537                let (ctx, mut rx) = wide_ctx(connection);
12538                hello_via_sink(
12539                    &handler,
12540                    &ctx,
12541                    &mut rx,
12542                    hello_frame(module_id, PROTOCOL_VERSION, connection),
12543                )
12544                .await;
12545                modules.insert(module_id.to_string(), (ctx, rx));
12546            }
12547            Rig {
12548                handler,
12549                forwarding,
12550                owner,
12551                _owner_rx: owner_rx,
12552                modules,
12553                generation: 0,
12554                next_connection: 100,
12555                _supervisor: supervisor,
12556            }
12557        }
12558
12559        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12560            ScopeCarrier {
12561                principal: Principal::Reserved {
12562                    module_id: module_id.to_string(),
12563                },
12564                targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12565            }
12566        }
12567
12568        /// The scope most tests open under: aft carries to any module, broca
12569        /// only to plexus and other, and the owner delegates as agent-1.
12570        fn session(scope_epoch: u64) -> ScopeRecord {
12571            let mut record = head("s", scope_epoch);
12572            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12573            record.attributes.agent_id = Some("agent-1".to_string());
12574            record.attributes.delegates = true;
12575            record
12576        }
12577
12578        impl Rig {
12579            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12580                self.generation += 1;
12581                sync(&self.handler, &self.owner, self.generation, scopes)
12582                    .await
12583                    .expect("the owner's sync is accepted");
12584            }
12585
12586            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12587                ScopeSelector {
12588                    owner: Principal::Reserved {
12589                        module_id: OWNER.to_string(),
12590                    },
12591                    scope_ref: scope_ref.to_string(),
12592                    scope_epoch,
12593                }
12594            }
12595
12596            fn open_frame(
12597                &mut self,
12598                opener: Option<&str>,
12599                target: &str,
12600                scope: Option<ScopeSelector>,
12601            ) -> (
12602                RouteCtx,
12603                mpsc::Receiver<crate::router::OutboundFrame>,
12604                Frame,
12605            ) {
12606                self.next_connection += 1;
12607                let (ctx, rx) = wide_ctx(self.next_connection);
12608                let root = unique_project_root("scoped-open");
12609                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12610                    target: RouteTarget::ToolProvider {
12611                        module_id: target.to_string(),
12612                    },
12613                    identity: BindIdentity::new(
12614                        root.path().to_path_buf(),
12615                        "unit".to_string(),
12616                        "session".to_string(),
12617                    ),
12618                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
12619                        module_id: module_id.to_string(),
12620                        launch_nonce: nonce(module_id),
12621                    }),
12622                    consumer_capabilities: None,
12623                    role_versions: None,
12624                    admission_facts: None,
12625                    scope,
12626                })
12627                .unwrap();
12628                let frame = Frame::build(
12629                    FrameType::Request,
12630                    control_flags(),
12631                    0,
12632                    0,
12633                    self.next_connection,
12634                    body,
12635                )
12636                .unwrap();
12637                (ctx, rx, frame)
12638            }
12639
12640            /// Open and expect a refusal before anything is relayed.
12641            async fn refused(
12642                &mut self,
12643                opener: Option<&str>,
12644                target: &str,
12645                scope: Option<ScopeSelector>,
12646            ) -> String {
12647                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12648                let replies = self
12649                    .handler
12650                    .handle_control_frame(&ctx, frame)
12651                    .await
12652                    .unwrap();
12653                assert_eq!(replies.len(), 1, "{replies:?}");
12654                assert_eq!(replies[0].header.ty, FrameType::Error);
12655                let (_, module_rx) = self.modules.get_mut(target).unwrap();
12656                assert!(
12657                    module_rx.try_recv().is_err(),
12658                    "a refused open relays nothing"
12659                );
12660                parse_error(&replies[0])["code"]
12661                    .as_str()
12662                    .unwrap()
12663                    .to_string()
12664            }
12665
12666            /// Start an open and return its task and the bind the target got.
12667            async fn relayed(
12668                &mut self,
12669                opener: Option<&str>,
12670                target: &str,
12671                scope: Option<ScopeSelector>,
12672            ) -> Relayed {
12673                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12674                let handler = self.handler.clone();
12675                let task_ctx = ctx.clone();
12676                let task = tokio::spawn(async move {
12677                    handler
12678                        .handle_control_frame(&task_ctx, frame)
12679                        .await
12680                        .unwrap()
12681                });
12682                let (_, module_rx) = self.modules.get_mut(target).unwrap();
12683                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12684                    .await
12685                    .expect("the target receives the relayed route.bind")
12686                    .unwrap()
12687                    .frame;
12688                Relayed {
12689                    target: target.to_string(),
12690                    client: ctx,
12691                    client_rx: rx,
12692                    task,
12693                    bind,
12694                }
12695            }
12696
12697            async fn ack(&self, relayed: &Relayed) {
12698                let (module, _) = &self.modules[&relayed.target];
12699                self.handler
12700                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
12701                    .await
12702                    .unwrap();
12703            }
12704
12705            /// Open, ack and return the bound route.
12706            async fn bound(
12707                &mut self,
12708                opener: Option<&str>,
12709                target: &str,
12710                scope: Option<ScopeSelector>,
12711            ) -> Bound {
12712                let relayed = self.relayed(opener, target, scope).await;
12713                self.ack(&relayed).await;
12714                let Relayed {
12715                    target,
12716                    client,
12717                    mut client_rx,
12718                    task,
12719                    bind,
12720                } = relayed;
12721                assert!(
12722                    task.await.unwrap().is_empty(),
12723                    "the open is answered by commit"
12724                );
12725                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12726                Bound {
12727                    target,
12728                    client,
12729                    client_rx,
12730                    channel,
12731                    epoch,
12732                    bind,
12733                }
12734            }
12735
12736            fn live(&self, route: &Bound) -> bool {
12737                matches!(
12738                    self.forwarding
12739                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12740                        .unwrap(),
12741                    DataRoute::Client(DataRouteState::Bound(_))
12742                )
12743            }
12744        }
12745
12746        struct Relayed {
12747            target: String,
12748            client: RouteCtx,
12749            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12750            task: tokio::task::JoinHandle<Vec<Frame>>,
12751            bind: Frame,
12752        }
12753
12754        struct Bound {
12755            target: String,
12756            client: RouteCtx,
12757            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12758            channel: u16,
12759            epoch: u32,
12760            bind: Frame,
12761        }
12762
12763        impl Bound {
12764            /// The reason of the `route.closed` this client was sent, after
12765            /// checking it also got a GOODBYE on exactly this route.
12766            fn closed_reason(&mut self) -> RouteCloseReason {
12767                let mut reason = None;
12768                let mut goodbye = false;
12769                while let Ok(outbound) = self.client_rx.try_recv() {
12770                    let frame = outbound.frame;
12771                    match frame.header.ty {
12772                        FrameType::Goodbye => {
12773                            assert_eq!(
12774                                (frame.header.channel, frame.header.epoch),
12775                                (self.channel, self.epoch)
12776                            );
12777                            goodbye = true;
12778                        }
12779                        FrameType::Push => {
12780                            let ClientControlPush::RouteClosed {
12781                                reason: r,
12782                                module_id,
12783                                ..
12784                            } = serde_json::from_slice(&frame.body).unwrap()
12785                            else {
12786                                panic!("unexpected push");
12787                            };
12788                            assert_eq!(module_id, self.target);
12789                            reason = Some(r);
12790                        }
12791                        other => panic!("unexpected frame {other:?}"),
12792                    }
12793                }
12794                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12795                reason.expect("the client is told why the route closed")
12796            }
12797
12798            fn untouched(&mut self) -> bool {
12799                self.client_rx.try_recv().is_err()
12800            }
12801
12802            fn stamp(&self) -> Option<ScopeStamp> {
12803                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12804                    ModuleControlRequest::RouteBind { scope, .. } => scope,
12805                    other => panic!("expected a route.bind, got {other:?}"),
12806                }
12807            }
12808        }
12809
12810        #[tokio::test]
12811        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12812        ) {
12813            let mut rig = rig().await;
12814            let mut record = session(1);
12815            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12816            record.child_owners = vec![Principal::Reserved {
12817                module_id: MAGIC.to_string(),
12818            }];
12819            rig.sync(vec![record]).await;
12820            let scope = || Some(rig_selector("s", Some(1)));
12821
12822            // Admitted: the owner, a bare carrier to any module, a targeted
12823            // carrier to its listed module.
12824            rig.bound(Some(OWNER), PLEXUS, scope()).await;
12825            rig.bound(Some(AFT), OTHER, scope()).await;
12826            rig.bound(Some(BROCA), PLEXUS, scope()).await;
12827
12828            // Refused scope_not_carrier: a targeted carrier to an unlisted
12829            // module, a module that is not listed at all (a child owner is not
12830            // a carrier), and a direct key-holder.
12831            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12832                assert_eq!(
12833                    rig.refused(opener, target, scope()).await,
12834                    error_codes::SCOPE_NOT_CARRIER,
12835                    "{opener:?} -> {target}"
12836                );
12837            }
12838        }
12839
12840        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12841            ScopeSelector {
12842                owner: Principal::Reserved {
12843                    module_id: OWNER.to_string(),
12844                },
12845                scope_ref: scope_ref.to_string(),
12846                scope_epoch,
12847            }
12848        }
12849
12850        #[tokio::test]
12851        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12852            let mut rig = rig().await;
12853            rig.sync(vec![session(1)]).await;
12854            for opener in [OWNER, AFT] {
12855                assert_eq!(
12856                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12857                        .await,
12858                    error_codes::SCOPE_EPOCH_REQUIRED,
12859                    "{opener}"
12860                );
12861            }
12862        }
12863
12864        #[tokio::test]
12865        async fn admission_separates_not_synced_not_live_and_ended() {
12866            let mut rig = rig().await;
12867            // Before the configured owner's first sync: retryable.
12868            let code = rig
12869                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12870                .await;
12871            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12872            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12873
12874            // An owner that is not configured will never sync: terminal.
12875            let ghost = ScopeSelector {
12876                owner: Principal::Reserved {
12877                    module_id: "ghost".to_string(),
12878                },
12879                scope_ref: "s".to_string(),
12880                scope_epoch: Some(1),
12881            };
12882            assert_eq!(
12883                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12884                error_codes::SCOPE_NOT_LIVE
12885            );
12886
12887            rig.sync(vec![session(2)]).await;
12888            assert_eq!(
12889                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12890                    .await,
12891                error_codes::SCOPE_NOT_LIVE
12892            );
12893            for epoch in [1, 3] {
12894                assert_eq!(
12895                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12896                        .await,
12897                    error_codes::SCOPE_ENDED,
12898                    "epoch {epoch}"
12899                );
12900            }
12901            // Control: the live epoch is admitted.
12902            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12903                .await;
12904        }
12905
12906        #[tokio::test]
12907        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12908            let mut rig = rig().await;
12909            rig.sync(vec![session(1)]).await;
12910            let route = rig
12911                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12912                .await;
12913            let stamp = route.stamp().expect("a scoped bind carries the stamp");
12914            assert_eq!(stamp.scope_ref, "s");
12915            assert_eq!(stamp.scope_epoch, 1);
12916            assert_eq!(stamp.kind, ScopeKind::Head);
12917            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12918            assert!(stamp.attributes.delegates);
12919            assert!(stamp.owner_authorized);
12920            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12921            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12922
12923            // broca owns a scope of its own on its own module connection; it is
12924            // not in scope_authority_owners, so its stamp is not authorized.
12925            let (broca, mut broca_rx) = wide_ctx(50);
12926            hello_via_sink(
12927                &rig.handler,
12928                &broca,
12929                &mut broca_rx,
12930                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
12931            )
12932            .await;
12933            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
12934                .await
12935                .unwrap();
12936            let own = ScopeSelector {
12937                owner: Principal::Reserved {
12938                    module_id: BROCA.to_string(),
12939                },
12940                scope_ref: "b".to_string(),
12941                scope_epoch: Some(1),
12942            };
12943            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
12944            assert!(!route.stamp().unwrap().owner_authorized);
12945        }
12946
12947        /// The owner's sync lands between admission and the module's ack. The
12948        /// open is refused by name, the module's other routes stay up, and the
12949        /// reserved pair is released. Changed content is retryable; an ended
12950        /// scope is not.
12951        #[tokio::test]
12952        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
12953            let mut rig = rig().await;
12954            rig.sync(vec![session(1)]).await;
12955            let mut cotenant = rig
12956                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
12957                .await;
12958
12959            let mut changed = session(1);
12960            changed.child_owners.push(Principal::Reserved {
12961                module_id: MAGIC.to_string(),
12962            });
12963            let mut ended = None;
12964            for (code, next) in [
12965                (error_codes::SCOPE_CHANGED, vec![changed]),
12966                (error_codes::SCOPE_ENDED, Vec::new()),
12967            ] {
12968                let relayed = rig
12969                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12970                    .await;
12971                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
12972                ended = Some(next.is_empty());
12973                rig.sync(next).await;
12974                rig.ack(&relayed).await;
12975                let replies = relayed.task.await.unwrap();
12976                assert_eq!(replies.len(), 1, "{replies:?}");
12977                assert_eq!(parse_error(&replies[0])["code"], code);
12978                assert_eq!(
12979                    subc_protocol::error_codes::is_retryable_route_open(code),
12980                    code == error_codes::SCOPE_CHANGED
12981                );
12982                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
12983                // The module is told to drop just the binding it created.
12984                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12985                // Collected, because ending the scope also closes the co-tenant
12986                // route, whose GOODBYE comes first.
12987                let mut goodbyes = Vec::new();
12988                while let Ok(outbound) = plexus_rx.try_recv() {
12989                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
12990                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
12991                }
12992                assert!(
12993                    goodbyes.contains(&(bind_channel, bind_epoch)),
12994                    "{goodbyes:?}"
12995                );
12996                assert!(rig
12997                    .handler
12998                    .registry
12999                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
13000                    .unwrap()
13001                    .is_some());
13002            }
13003            assert_eq!(ended, Some(true));
13004            // The co-tenant stayed up through the change, and closed only when
13005            // the scope ended, by the drain rule rather than by the commit.
13006            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
13007        }
13008
13009        /// Each row of the drain table on one set of routes: the owner's, a
13010        /// bare carrier's, and a targeted carrier's to each of its targets.
13011        #[tokio::test]
13012        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
13013            struct Case {
13014                name: &'static str,
13015                change: fn(&mut ScopeRecord),
13016                /// Closed routes by index: owner->plexus, aft->plexus,
13017                /// broca->plexus, broca->other.
13018                closed: [Option<RouteCloseReason>; 4],
13019            }
13020            use RouteCloseReason::*;
13021            let cases = [
13022                Case {
13023                    name: "a carrier entry removed",
13024                    change: |r| {
13025                        r.carriers.retain(|c| {
13026                            c.principal
13027                                != Principal::Reserved {
13028                                    module_id: AFT.to_string(),
13029                                }
13030                        })
13031                    },
13032                    closed: [None, Some(ScopeCarrierRemoved), None, None],
13033                },
13034                Case {
13035                    name: "a target removed from a carrier",
13036                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
13037                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
13038                },
13039                Case {
13040                    name: "a bare carrier narrowed to targets",
13041                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
13042                    closed: [None, Some(ScopeCarrierRemoved), None, None],
13043                },
13044                Case {
13045                    name: "delegates turned off",
13046                    change: |r| r.attributes.delegates = false,
13047                    closed: [Some(ScopeDelegationChanged); 4],
13048                },
13049                Case {
13050                    name: "agent_id changed",
13051                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
13052                    closed: [Some(ScopeDelegationChanged); 4],
13053                },
13054                Case {
13055                    name: "a carrier added, child owners changed, the record re-sent",
13056                    change: |r| {
13057                        r.carriers.push(carrier(MAGIC, None));
13058                        r.child_owners.push(Principal::Reserved {
13059                            module_id: MAGIC.to_string(),
13060                        });
13061                    },
13062                    closed: [None; 4],
13063                },
13064                Case {
13065                    name: "a target added",
13066                    change: |r| {
13067                        r.carriers[1]
13068                            .targets
13069                            .as_mut()
13070                            .unwrap()
13071                            .push("third".to_string())
13072                    },
13073                    closed: [None; 4],
13074                },
13075                Case {
13076                    name: "delegates turned on",
13077                    change: |r| r.attributes.delegates = true,
13078                    closed: [None; 4],
13079                },
13080            ];
13081            for case in cases {
13082                let mut rig = rig().await;
13083                rig.sync(vec![session(1)]).await;
13084                let scope = || Some(rig_selector("s", Some(1)));
13085                let mut routes = [
13086                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
13087                    rig.bound(Some(AFT), PLEXUS, scope()).await,
13088                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
13089                    rig.bound(Some(BROCA), OTHER, scope()).await,
13090                ];
13091                let mut record = session(1);
13092                (case.change)(&mut record);
13093                rig.sync(vec![record]).await;
13094                for (index, expected) in case.closed.iter().enumerate() {
13095                    let route = &mut routes[index];
13096                    match expected {
13097                        Some(reason) => {
13098                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
13099                            assert_eq!(
13100                                route.closed_reason(),
13101                                *reason,
13102                                "{}: route {index}",
13103                                case.name
13104                            );
13105                        }
13106                        None => {
13107                            assert!(rig.live(route), "{}: route {index} closed", case.name);
13108                            assert!(
13109                                route.untouched(),
13110                                "{}: route {index} was told something",
13111                                case.name
13112                            );
13113                        }
13114                    }
13115                }
13116            }
13117        }
13118
13119        #[tokio::test]
13120        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
13121            // Removed, and replaced by a higher epoch.
13122            for next in [Vec::new(), vec![session(2)]] {
13123                let mut rig = rig().await;
13124                rig.sync(vec![session(1)]).await;
13125                let mut route = rig
13126                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13127                    .await;
13128                rig.sync(next).await;
13129                assert!(!rig.live(&route));
13130                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
13131            }
13132
13133            // A child whose parent ends: its routes close as parent-ended, the
13134            // child stays live, and routes under the parent close as ended.
13135            let mut rig = rig().await;
13136            let mut child = session(1);
13137            child.scope_ref = "child".to_string();
13138            child.kind = ScopeKind::Worker;
13139            child.parent = Some(ScopeParent {
13140                owner: Principal::Reserved {
13141                    module_id: OWNER.to_string(),
13142                },
13143                scope_ref: "s".to_string(),
13144                scope_epoch: 1,
13145            });
13146            rig.sync(vec![session(1), child.clone()]).await;
13147            let mut child_route = rig
13148                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
13149                .await;
13150            assert_eq!(
13151                child_route.stamp().unwrap().parent_state,
13152                Some(ParentState::Linked)
13153            );
13154            rig.sync(vec![child]).await;
13155            assert!(!rig.live(&child_route));
13156            assert_eq!(
13157                child_route.closed_reason(),
13158                RouteCloseReason::ScopeParentEnded
13159            );
13160        }
13161
13162        #[tokio::test]
13163        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
13164        ) {
13165            let mut rig = rig().await;
13166            rig.sync(vec![session(1)]).await;
13167            let mut route = rig
13168                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13169                .await;
13170            let before = rig.forwarding.published_scope_tag(OWNER, "s");
13171            rig.sync(vec![session(1)]).await;
13172            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13173            assert!(rig.live(&route) && route.untouched());
13174
13175            // A call in flight on the route when another carrier is added. A
13176            // forwarded REQUEST holds one credit on the route's flow until the
13177            // module answers; the router takes it exactly like this.
13178            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
13179                .forwarding
13180                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13181                .unwrap()
13182            else {
13183                panic!("the route is bound");
13184            };
13185            binding.flow.acquire_tagged(9, false).await.unwrap();
13186            let mut widened = session(1);
13187            widened.carriers.push(carrier(MAGIC, None));
13188            rig.sync(vec![widened]).await;
13189            assert!(rig.live(&route) && route.untouched());
13190            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13191            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
13192            // The call's credit is still held on an open flow, so its answer
13193            // will be delivered: closing the route would have closed the flow.
13194            assert_eq!(binding.flow.in_flight(), 1);
13195            binding
13196                .flow
13197                .acquire_tagged(10, false)
13198                .await
13199                .expect("the flow is still open");
13200        }
13201
13202        /// A swap's superseded endpoint keeps its routes until drained; ending
13203        /// the scope closes them there too.
13204        #[tokio::test]
13205        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
13206            let mut rig = rig().await;
13207            rig.sync(vec![session(1)]).await;
13208            let mut on_incumbent = rig
13209                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13210                .await;
13211
13212            // Swap plexus: register a candidate and cut over, leaving the
13213            // incumbent superseded with the route still on it.
13214            let (candidate, _candidate_rx) = wide_ctx(9);
13215            let registration = rig
13216                .handler
13217                .registry
13218                .register_candidate_with_control_ops(
13219                    manifest(PLEXUS, PROTOCOL_VERSION),
13220                    PROTOCOL_VERSION,
13221                    candidate.connection_id,
13222                    module_baseline_control_ops(),
13223                )
13224                .unwrap();
13225            rig.forwarding
13226                .register_candidate_module_connection(
13227                    candidate.connection_id,
13228                    PLEXUS.to_string(),
13229                    PROTOCOL_VERSION,
13230                    manifest_concurrency(&registration.manifest),
13231                    candidate.egress.clone(),
13232                )
13233                .unwrap();
13234            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
13235            rig.handler
13236                .registry
13237                .promote_candidate(PLEXUS)
13238                .unwrap()
13239                .unwrap();
13240            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
13241
13242            rig.sync(Vec::new()).await;
13243            assert!(!rig.live(&on_incumbent));
13244            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
13245            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13246            let goodbye = incumbent_rx
13247                .try_recv()
13248                .expect("the superseded endpoint is told")
13249                .frame;
13250            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13251        }
13252    }
13253}
13254
13255#[cfg(test)]
13256mod concurrency_default_exposure_tests {
13257    use super::*;
13258
13259    fn hello_body(role_json: &str) -> Vec<u8> {
13260        format!(
13261            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
13262        )
13263        .into_bytes()
13264    }
13265
13266    fn manifest_from(body: &[u8]) -> ModuleManifest {
13267        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
13268        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
13269            .expect("manifest parses")
13270    }
13271
13272    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
13273
13274    #[test]
13275    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
13276        let body = hello_body(&format!(
13277            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
13278        ));
13279        let manifest = manifest_from(&body);
13280        // Precondition: serde really resolved it to the default, so the typed
13281        // manifest alone cannot answer the question this probe exists for.
13282        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13283        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
13284    }
13285
13286    #[test]
13287    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
13288        let body = hello_body(&format!(
13289            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
13290        ));
13291        let manifest = manifest_from(&body);
13292        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13293        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13294    }
13295
13296    #[test]
13297    fn non_management_roles_are_never_reported() {
13298        let body = hello_body(
13299            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
13300        );
13301        let manifest = manifest_from(&body);
13302        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13303    }
13304}