Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::{Path, PathBuf},
5    sync::{Arc, Mutex, RwLock},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20    error_codes,
21    manifest::{
22        validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24        ProviderRole,
25    },
26    scope::{
27        ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29    },
30    session::{
31        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33        MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34    },
35    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36    Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42    capability_requirements::{
43        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45        RuntimeModule,
46    },
47    daemon_config::RestartRequiredSection,
48    forwarding::{
49        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52    },
53    observability::{
54        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55    },
56    provenance::{
57        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58    },
59    registry::{ChannelState, ConnectionId, Registry, RegistryError},
60    router::{RouteCtx, RouterError},
61    scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63    stderr_tail::{CaptureState, TailEntry},
64    supervise::{
65        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66        SupervisorHandle, SwapHelloAdmission,
67    },
68    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71/// Lowest envelope version this subc build will negotiate.
72///
73/// Module HELLO negotiation is exact: peers must use the daemon's locked
74/// protocol version. Older and newer peers receive `version_unsupported` and
75/// are not registered.
76pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85    ops::SERVER_DESCRIBE,
86    ops::CATALOG_LIST,
87    ops::ROUTE_OPEN,
88    ops::ROUTE_POLL,
89    ops::ROUTE_CLOSING,
90    ops::ROUTE_CLOSED,
91    ops::SUPERVISOR_LIST,
92    ops::SUPERVISOR_RESTART,
93    ops::SUPERVISOR_SWAP,
94    ops::SUPERVISOR_RELOAD,
95    ops::SUPERVISOR_RESCAN,
96    ops::SUPERVISOR_RELEASE_RESERVED,
97    ops::SUPERVISOR_SET_ENABLED,
98    ops::SUPERVISOR_HEALTH_PROBE,
99    ops::SUPERVISOR_HEALTH,
100    ops::SUPERVISOR_STDERR_TAIL,
101    ops::SUPERVISOR_TERMINALS,
102    ops::SUPERVISOR_ROUTES,
103    ops::SUPERVISOR_PROVENANCE,
104    ops::SUPERVISOR_SPAWN_SNAPSHOT,
105    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110    "supervisor.live_roots",
111    SCOPE_SYNC_OP,
112    SCOPE_DESCRIBE_OP,
113];
114
115/// Module-originated ops the daemon answers but does not advertise in
116/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
117/// belongs to is incomplete, so no module is told it works before it does.
118const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122/// How long subc waits for a module to ack a relayed route.bind before returning
123/// `module_timeout`. The ack waits on the module's own configure, which for AFT
124/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
125/// and DB-open work — on a cold page cache or a large repo that legitimately
126/// exceeds a couple of seconds. The default is generous because rejecting a VALID
127/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
128/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
129pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131/// How many CONSECUTIVE full-budget relay timeouts against one target module
132/// open that module's bind-relay breaker.
133///
134/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
135/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
136/// so three consecutive full-budget timeouts take ~36s to observe: every client
137/// whose open contributed to opening the breaker had already given up on its
138/// own. That is what makes opening the breaker unable to turn a call that would
139/// have succeeded into a refusal — it can only make an already-failing module
140/// fail faster.
141///
142/// Two would be reachable inside one default deadline. One would convict a
143/// module on a single cold-cache bind, which is exactly the valid-but-slow case
144/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
145pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147/// How long a module's bind-relay breaker stays open before exactly one
148/// `route.open` is let through as a probe.
149///
150/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
151/// re-pays a full-budget stall almost continuously, and the breaker stops being
152/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
153/// deadline: a client that starts retrying after the module recovers has to get
154/// a probe opportunity inside its own deadline, or the breaker converts a
155/// recovered module into a failed call — the failure it exists to prevent,
156/// pointed the other way.
157///
158/// 20s sits between those with room on both sides, and it caps what a wedged
159/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
160/// rather than one per `route.open` per connection. The stall that motivated
161/// this, with its measurements, is written up in
162/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
163/// waited the whole budget out.
164pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170    configured: &Path,
171    spawned_from: Option<&Path>,
172    image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174    let path = match spawned_from {
175        Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176        Some(spawned_from) => ReloadPathAgreement::Mismatch {
177            configured: configured.to_path_buf(),
178            spawned_from: spawned_from.to_path_buf(),
179        },
180        None => ReloadPathAgreement::Unavailable {
181            reason: if matches!(
182                image,
183                subc_control::RunningImageAgreement::Unavailable {
184                    reason: subc_control::RunningImageUnavailableReason::NotRunning
185                }
186            ) {
187                ReloadPathUnavailableReason::NotRunning
188            } else {
189                ReloadPathUnavailableReason::SpawnedPathUnavailable
190            },
191        },
192    };
193    PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198    build: DaemonBuildProvenance,
199    pid: Option<u32>,
200    started_at_ms: Option<u64>,
201    start_clock: Option<crate::clock::StartClock>,
202    executable_path: Option<PathBuf>,
203    executable_identity: Option<SpawnedFileIdentity>,
204    process_start_time: Option<u64>,
205    probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209    fn default() -> Self {
210        Self {
211            build: DaemonBuildProvenance {
212                build_git_sha: None,
213                build_lock_digest: None,
214            },
215            pid: None,
216            started_at_ms: None,
217            start_clock: None,
218            executable_path: None,
219            executable_identity: None,
220            process_start_time: None,
221            probe: ExecutableIdentityProbe::default(),
222        }
223    }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228    supervisor: Supervisor,
229    config_path: PathBuf,
230    configured_port: Option<u16>,
231    storage_config: Option<crate::daemon_config::StorageConfig>,
232    admission_facts_carrier_module_id: Option<String>,
233    admission_facts_targets: Option<Vec<String>>,
234    scope_authority_owners: Vec<String>,
235}
236
237/// Refusal labels passed to `observe_route_open_refusal` that mean the target
238/// module is not serving right now, and so open or extend an outage in the
239/// route outage tracker. Every one of them is only reachable after the target
240/// was found in the registry, which is what keeps an arbitrary client-chosen
241/// id from ever creating tracker state.
242///
243/// Deliberately absent: `not_registered` and `removed` (the id may be
244/// anything a client sent, and a removed module is gone on purpose),
245/// `protocol_none` (such a module never serves routes, so nothing is out),
246/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
247/// capability and admission-facts refusals (they refuse the caller, not a
248/// module outage), and `relay_reservation_failed` (its code ranges over
249/// capacity limits as well as a vanished connection). Capacity, breaker,
250/// relay-timeout and module-rejection refusals do not pass through that
251/// function at all; the breaker logs its own transitions.
252///
253/// The two not-serving refusals that bypass that function record themselves
254/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
255/// `required_capability_unprovided` is not tracked: the module itself is up,
256/// and the outage belongs to the missing provider.
257const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258    "reloading",
259    "supervisor_not_live",
260    "registration_not_active",
261    "no_forwarding_connection",
262    "relay_send_failed",
263];
264
265/// Real channel-0 control handler for subc itself.
266#[derive(Clone)]
267pub struct ControlHandler {
268    registry: Arc<Registry>,
269    forwarding: Arc<ForwardingTable>,
270    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271    supervisor: SupervisorHandle,
272    subc_capabilities: Arc<[String]>,
273    /// Daemon-wide route.bind relay budget. Used as the fallback when the
274    /// target module has no per-module override in
275    /// `route_bind_relay_timeouts`.
276    route_bind_relay_timeout: Duration,
277    /// Per-module route.bind relay budget overrides, keyed by module id. When
278    /// `handle_route_open` resolves the deadline for a target module, a
279    /// per-module entry wins over the daemon-wide value above.
280    route_bind_relay_timeouts: BTreeMap<String, Duration>,
281    /// Per-target-module bind-relay breaker state. Shared with the forwarding
282    /// table, which is where a new module connection resets it.
283    route_bind_breakers: RouteBindBreakers,
284    /// Live relay admissions keyed by target module. Shared through the
285    /// forwarding table so cloned or separately built handlers enforce one cap.
286    route_bind_concurrency: RouteBindConcurrency,
287    /// Start and end of each module's not-serving period as seen by
288    /// `route.open`, so an outage gets one line at each edge instead of only
289    /// the per-refusal INFO lines. Taken from the forwarding table, so every
290    /// handler built over one table shares it.
291    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292    /// Consecutive relay timeouts that open a module's breaker.
293    route_bind_breaker_threshold: u32,
294    /// How long a breaker stays open before one probe is admitted.
295    route_bind_breaker_cooldown: Duration,
296    health_probe_timeout: Duration,
297    /// Central storage policy. When set, each registering module receives its
298    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
299    storage_config: Option<crate::daemon_config::StorageConfig>,
300    /// The machine id established at boot, served on every HELLO_ACK and on
301    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
302    /// changes the file, never this value. `None` serves no id.
303    machine_id: Option<crate::machine_id::MachineId>,
304    admission_facts_carrier_module_id: Option<String>,
305    admission_facts_targets: Option<Vec<String>>,
306    /// Scope records with their sync authorities and tombstones; see
307    /// `crate::scopes`. Shared by clones of this handler, so every connection
308    /// reads and writes one table.
309    scopes: Arc<RwLock<ScopeTable>>,
310    /// The configured `scope_authority_owners`, kept so a rescan can report a
311    /// changed value as needing a daemon restart; rescan never applies it.
312    scope_authority_owners: Vec<String>,
313    /// The launch nonce each module connection presented at HELLO, which is how
314    /// a `scope.sync` is matched to the owner's current launch.
315    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316    rescan: Option<SupervisorRescanContext>,
317    connected_clients: ConnectedClients,
318    counters: DaemonCounters,
319    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320    daemon_provenance: DaemonProvenanceFacts,
321    #[cfg(test)]
322    control_dispatch_delay: Option<Duration>,
323    #[cfg(test)]
324    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325    /// Holds `route.open` after the route.bind is queued to the module and
326    /// before it is marked enqueued, so a test can make the module's reply (or
327    /// a client teardown) land exactly inside that window.
328    #[cfg(test)]
329    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
330}
331
332impl fmt::Debug for ControlHandler {
333    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
334        f.debug_struct("ControlHandler")
335            .field("registry", &self.registry)
336            .field("forwarding", &self.forwarding)
337            .field("process_liveness", &self.process_liveness.is_some())
338            .field("supervisor", &self.supervisor)
339            .field("subc_capabilities", &self.subc_capabilities)
340            .finish()
341    }
342}
343
344struct RouteOpenRequest {
345    target: RouteTarget,
346    identity: BindIdentity,
347    consumer_identity: Option<ConsumerIdentity>,
348    consumer_capabilities: Option<Vec<String>>,
349    role_versions: Option<BTreeMap<String, String>>,
350    admission_facts: Option<serde_json::Value>,
351    scope: Option<ScopeSelector>,
352}
353
354struct RouteBindReservationGuard {
355    forwarding: Arc<ForwardingTable>,
356    endpoint: ModuleEndpointId,
357    relay_corr: u64,
358    armed: bool,
359}
360
361struct ModuleControlRpcGuard {
362    forwarding: Arc<ForwardingTable>,
363    endpoint: ModuleEndpointId,
364    corr: u64,
365    armed: bool,
366}
367
368impl ModuleControlRpcGuard {
369    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
370        Self {
371            forwarding,
372            endpoint,
373            corr,
374            armed: true,
375        }
376    }
377
378    fn disarm(&mut self) {
379        self.armed = false;
380    }
381}
382
383impl Drop for ModuleControlRpcGuard {
384    fn drop(&mut self) {
385        if self.armed {
386            let _ = self
387                .forwarding
388                .cancel_module_control_rpc(self.endpoint, self.corr);
389        }
390    }
391}
392
393impl RouteBindReservationGuard {
394    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
395        Self {
396            forwarding,
397            endpoint,
398            relay_corr,
399            armed: true,
400        }
401    }
402
403    fn release_and_disarm(&mut self) {
404        if !self.armed {
405            return;
406        }
407        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
408            self.endpoint,
409            self.relay_corr,
410            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
411        ) {
412            send_goodbye_target_best_effort(
413                &self.forwarding.counters(),
414                &target,
415                "canceled route.bind",
416            );
417        }
418        self.armed = false;
419    }
420
421    fn disarm(&mut self) {
422        self.armed = false;
423    }
424}
425
426impl Drop for RouteBindReservationGuard {
427    fn drop(&mut self) {
428        self.release_and_disarm();
429    }
430}
431
432/// Per-target-module circuit breaker around the `route.bind` relay.
433///
434/// The connection reader is serial per connection, so a module whose `on_bind`
435/// sits on the ack blocks every LATER frame on the connections that call it,
436/// including calls to unrelated modules. This does not make any module's bind
437/// fast; it stops the daemon paying the full budget again and again for a
438/// condition it has already observed.
439///
440/// State is keyed by TARGET MODULE and shared by every connection: a wedged
441/// module wedges everyone, so what one connection learned should protect the
442/// rest.
443///
444/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
445/// relay to that module has actually timed out, and is removed again when a
446/// relay is accepted or the module reconnects, so it cannot grow with traffic
447/// or with modules that behave.
448///
449/// # Why a `std` mutex here is not the head-of-line defect again
450///
451/// Acquisition never awaits. The critical section is a hash lookup plus a few
452/// integer updates, with no I/O and no `.await` inside it, so a reader task
453/// cannot be descheduled behind it the way it can behind
454/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
455/// primitive, held for the same kind of work, as the refusal counter this very
456/// path already increments.
457///
458/// It is also NOT on the data-plane splice path: only `route.open` and module
459/// registration touch it, so bound-route frames gain no state check and no
460/// contention.
461#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindBreakers {
463    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
464}
465
466#[derive(Debug, Clone, Default)]
467pub(crate) struct RouteBindConcurrency {
468    modules: Arc<Mutex<HashMap<String, usize>>>,
469}
470
471struct RouteBindConcurrencyGuard {
472    concurrency: RouteBindConcurrency,
473    module_id: String,
474}
475
476impl RouteBindConcurrency {
477    /// Admit without waiting. Waiting here would move the bind stall from the
478    /// module reply to a semaphore and restore reader head-of-line blocking.
479    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
480        let mut modules = self
481            .modules
482            .lock()
483            .expect("route.bind concurrency mutex poisoned");
484        let in_flight = modules.entry(module_id.to_string()).or_default();
485        if *in_flight >= limit {
486            return Err(*in_flight);
487        }
488        *in_flight += 1;
489        Ok(RouteBindConcurrencyGuard {
490            concurrency: self.clone(),
491            module_id: module_id.to_string(),
492        })
493    }
494}
495
496impl Drop for RouteBindConcurrencyGuard {
497    fn drop(&mut self) {
498        let mut modules = self
499            .concurrency
500            .modules
501            .lock()
502            .expect("route.bind concurrency mutex poisoned");
503        let remove = {
504            let in_flight = modules
505                .get_mut(&self.module_id)
506                .expect("admitted route.bind has a concurrency entry");
507            *in_flight -= 1;
508            *in_flight == 0
509        };
510        if remove {
511            modules.remove(&self.module_id);
512        }
513    }
514}
515
516#[derive(Debug, Default)]
517struct ModuleBreakerState {
518    /// Relay timeouts observed with no accepted relay in between.
519    consecutive_timeouts: u32,
520    /// `Some` while the breaker is open: the instant the cooldown expires and
521    /// the next arrival may probe. `None` means closed.
522    cooldown_until: Option<Instant>,
523    /// A half-open probe has been admitted and has not settled yet. This is
524    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
525    /// that read the cooldown, so concurrent opens arriving at the moment the
526    /// cooldown expires cannot all decide that they are the probe.
527    probe_in_flight: Option<Arc<()>>,
528}
529
530/// What the breaker decided for one `route.open`, before any relay work.
531enum RouteBindAdmission<'a> {
532    Admitted {
533        guard: RouteBindBreakerGuard<'a>,
534        /// This open is the single half-open probe, so the transition is worth
535        /// one log line.
536        probe: bool,
537    },
538    Refused {
539        consecutive_timeouts: u32,
540        /// What is left of the cooldown. Zero when the refusal is because the
541        /// one probe is already in flight rather than because the cooldown has
542        /// not elapsed.
543        retry_in: Duration,
544        probe_in_flight: bool,
545    },
546}
547
548/// An outstanding admission, which must be told how its relay settled.
549///
550/// `Drop` settles it as inconclusive, so an early return between admission and
551/// the relay -- or the whole handler being cancelled when the client
552/// disconnects -- releases a half-open probe slot instead of leaving the
553/// breaker wedged half-open with no further probes.
554struct RouteBindBreakerGuard<'a> {
555    breakers: RouteBindBreakers,
556    module_id: &'a str,
557    probe_token: Option<Arc<()>>,
558    settled: bool,
559}
560
561impl RouteBindBreakerGuard<'_> {
562    /// The module answered within the budget and took the bind. THE ONLY
563    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
564    /// breaker, which is a transition worth logging.
565    fn record_accepted(&mut self) -> bool {
566        self.settled = true;
567        self.breakers.record_accepted(self.module_id)
568    }
569
570    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
571    /// COUNTS TOWARD OPENING.
572    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
573        self.settled = true;
574        self.breakers.record_timeout(
575            self.module_id,
576            self.probe_token.as_ref(),
577            threshold,
578            cooldown,
579        )
580    }
581
582    /// Everything else: the module REJECTED the bind, its connection went away
583    /// mid-relay, or the waiter was cancelled.
584    ///
585    /// None of these is evidence that a module is slow, and each already has
586    /// its own refusal with its own code. A module that rejects a bind in
587    /// microseconds is healthy and must never be convicted for it; a module
588    /// that died has said nothing about the module that replaces it. So these
589    /// neither increment nor reset the count -- they only release a probe slot.
590    fn record_inconclusive(&mut self) {
591        self.settled = true;
592        self.breakers
593            .record_inconclusive(self.module_id, self.probe_token.as_ref());
594    }
595}
596
597impl Drop for RouteBindBreakerGuard<'_> {
598    fn drop(&mut self) {
599        if !self.settled {
600            self.breakers
601                .record_inconclusive(self.module_id, self.probe_token.as_ref());
602        }
603    }
604}
605
606/// The breaker moved to open, reported so the caller can log it outside the
607/// lock. Opening is rare and load-bearing; the refusals that follow are
608/// frequent and are counted rather than logged.
609struct BreakerOpened {
610    consecutive_timeouts: u32,
611    /// True when a failed probe re-opened an already-open breaker, which reads
612    /// very differently in a log from a first opening.
613    reopened_after_probe: bool,
614}
615
616impl RouteBindBreakers {
617    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
618        self.modules
619            .lock()
620            .expect("route.bind breaker mutex poisoned")
621    }
622
623    /// Decide whether this `route.open` may attempt its relay. Takes the map
624    /// lock and nothing else, and never awaits.
625    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
626        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
627            probe: probe_token.is_some(),
628            guard: RouteBindBreakerGuard {
629                breakers: self.clone(),
630                module_id,
631                probe_token,
632                settled: false,
633            },
634        };
635
636        let mut modules = self.lock();
637        let Some(state) = modules.get_mut(module_id) else {
638            return admitted(None);
639        };
640        let Some(cooldown_until) = state.cooldown_until else {
641            return admitted(None);
642        };
643        if state.probe_in_flight.is_some() {
644            return RouteBindAdmission::Refused {
645                consecutive_timeouts: state.consecutive_timeouts,
646                retry_in: Duration::ZERO,
647                probe_in_flight: true,
648            };
649        }
650        let now = Instant::now();
651        if now < cooldown_until {
652            return RouteBindAdmission::Refused {
653                consecutive_timeouts: state.consecutive_timeouts,
654                retry_in: cooldown_until - now,
655                probe_in_flight: false,
656            };
657        }
658        let token = Arc::new(());
659        state.probe_in_flight = Some(Arc::clone(&token));
660        admitted(Some(token))
661    }
662
663    fn record_accepted(&self, module_id: &str) -> bool {
664        self.lock()
665            .remove(module_id)
666            .is_some_and(|state| state.cooldown_until.is_some())
667    }
668
669    fn record_timeout(
670        &self,
671        module_id: &str,
672        probe_token: Option<&Arc<()>>,
673        threshold: u32,
674        cooldown: Duration,
675    ) -> Option<BreakerOpened> {
676        let mut modules = self.lock();
677        let state = modules.entry(module_id.to_string()).or_default();
678        let was_open = state.cooldown_until.is_some();
679        let was_probe = Self::owns_probe(state, probe_token);
680        if was_probe {
681            state.probe_in_flight = None;
682        }
683        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
684        if state.consecutive_timeouts < threshold {
685            return None;
686        }
687        state.cooldown_until = Some(Instant::now() + cooldown);
688        Some(BreakerOpened {
689            consecutive_timeouts: state.consecutive_timeouts,
690            reopened_after_probe: was_open && was_probe,
691        })
692    }
693
694    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
695        // A relay can finish after the breaker was reset or after it opened
696        // again and started a new probe. Only the guard whose token matches the
697        // active probe may release it, so a late relay never frees a newer probe.
698        state
699            .probe_in_flight
700            .as_ref()
701            .zip(token)
702            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
703    }
704
705    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
706        if let Some(state) = self.lock().get_mut(module_id) {
707            if Self::owns_probe(state, probe_token) {
708                state.probe_in_flight = None;
709            }
710        }
711    }
712
713    /// Discard what was learned about a module, because the process it was
714    /// learned about is gone. Returns the discarded count when it was non-zero.
715    ///
716    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
717    /// `module_id` is a configuration identity that outlives any particular
718    /// child; what the breaker observed was the process behind the module
719    /// connection of the moment. When a new connection registers under that id
720    /// the verdict's subject no longer exists, so the verdict is stale by
721    /// construction rather than merely likely to be wrong. Keeping it would
722    /// apply a dead process's record to a live one, which is the same defect
723    /// class this breaker exists to stop the daemon committing.
724    ///
725    /// A half-open probe in flight is discarded with the rest: it was a
726    /// question about the old process.
727    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
728        self.lock()
729            .remove(module_id)
730            .map(|state| state.consecutive_timeouts)
731            .filter(|discarded| *discarded > 0)
732    }
733
734    /// Open breakers, for the `server.describe` counters object. `None` when
735    /// none is open, so the key stays absent rather than present-and-empty.
736    ///
737    /// This is the operator's answer to "is this module refusing instantly or
738    /// is it fine?", which look identical from a client that retries and then
739    /// succeeds.
740    fn open_snapshot(&self) -> Option<serde_json::Value> {
741        let now = Instant::now();
742        let modules = self.lock();
743        let open = modules
744            .iter()
745            .filter_map(|(module_id, state)| {
746                let cooldown_until = state.cooldown_until?;
747                Some((
748                    module_id.clone(),
749                    serde_json::json!({
750                        "consecutive_timeouts": state.consecutive_timeouts,
751                        "cooldown_remaining_ms":
752                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
753                        "probe_in_flight": state.probe_in_flight.is_some(),
754                    }),
755                ))
756            })
757            .collect::<serde_json::Map<String, serde_json::Value>>();
758        (!open.is_empty()).then_some(serde_json::Value::Object(open))
759    }
760}
761
762impl ControlHandler {
763    pub fn new(registry: Arc<Registry>) -> Self {
764        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
765    }
766
767    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
768        let counters = forwarding.counters();
769        // Taken from the forwarding table rather than created here, so that the
770        // breaker a `route.open` consults is the same one a module's
771        // registration resets, however many handlers are built over one table.
772        let route_bind_breakers = forwarding.route_bind_breakers();
773        let route_bind_concurrency = forwarding.route_bind_concurrency();
774        let route_outages = forwarding.route_outages();
775        Self {
776            registry,
777            forwarding,
778            process_liveness: None,
779            supervisor: SupervisorHandle::new(),
780            subc_capabilities: Arc::from([
781                CAP_MANIFEST_REGISTRATION.to_string(),
782                CAP_CHANNEL_LIFECYCLE.to_string(),
783                CAP_PING_PONG.to_string(),
784                CAP_SESSION_ATTACH.to_string(),
785                CAP_ADMISSION_FACTS_RELAY.to_string(),
786                CAP_SCOPES_V1.to_string(),
787                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
788            ]),
789            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
790            route_bind_relay_timeouts: BTreeMap::new(),
791            route_bind_breakers,
792            route_bind_concurrency,
793            route_outages,
794            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
795            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
796            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
797            storage_config: None,
798            machine_id: None,
799            admission_facts_carrier_module_id: None,
800            admission_facts_targets: None,
801            scopes: Arc::new(RwLock::new(ScopeTable::new(
802                crate::daemon_config::default_scope_authority_owners(),
803            ))),
804            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
805            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
806            rescan: None,
807            connected_clients: ConnectedClients::new(),
808            counters,
809            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
810            daemon_provenance: DaemonProvenanceFacts::default(),
811            #[cfg(test)]
812            control_dispatch_delay: None,
813            #[cfg(test)]
814            provenance_probe_override: None,
815            #[cfg(test)]
816            route_bind_relay_sent_hold: None,
817        }
818    }
819
820    /// Set the central storage policy: registering modules then receive their
821    /// resolved storage descriptor in HELLO_ACK.
822    pub fn with_storage_config(
823        mut self,
824        storage_config: Option<crate::daemon_config::StorageConfig>,
825    ) -> Self {
826        self.storage_config = storage_config;
827        self
828    }
829
830    /// Set the machine id served to every registering module (HELLO_ACK) and on
831    /// `server.describe`.
832    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
833        self.machine_id = machine_id;
834        self
835    }
836
837    /// Configure the exact reserved module and target ids permitted to relay
838    /// opaque admission facts. Config-file loading validates this authority;
839    /// this builder keeps the same policy available to embedded test daemons.
840    pub fn with_admission_facts_config(
841        mut self,
842        carrier_module_id: Option<String>,
843        targets: Option<Vec<String>>,
844    ) -> Self {
845        self.admission_facts_carrier_module_id = carrier_module_id;
846        self.admission_facts_targets = targets;
847        self
848    }
849
850    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
851    /// Replaces the scope table with an empty one under the new list, so call it
852    /// while building the handler, before any module can sync.
853    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
854        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
855        self.scope_authority_owners = owners;
856        self
857    }
858
859    /// Override the route.bind relay timeout. Used by tests that assert the
860    /// timeout path so they don't block on the production-safe default.
861    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
862        self.route_bind_relay_timeout = timeout;
863        self
864    }
865
866    /// Install per-module route.bind relay budget overrides. A module id
867    /// listed here wins over the daemon-wide default set via
868    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
869    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
870    /// the same `Duration` the bind path will use.
871    pub fn with_route_bind_relay_timeouts(
872        mut self,
873        timeouts: impl IntoIterator<Item = (String, Duration)>,
874    ) -> Self {
875        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
876        self
877    }
878
879    /// Resolve the route.bind relay budget for a specific target module id.
880    /// Per-module overrides win; the daemon-wide value (set via
881    /// `with_route_bind_relay_timeout` or the built-in default) is the
882    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
883    /// the same resolution `handle_route_open` will use.
884    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
885        self.route_bind_relay_timeouts
886            .get(module_id)
887            .copied()
888            .unwrap_or(self.route_bind_relay_timeout)
889    }
890
891    /// Override the per-module bind-relay breaker policy.
892    ///
893    /// Used by tests, which cannot spend three production budgets opening a
894    /// breaker or twenty seconds waiting for its cooldown. The production
895    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
896    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
897    /// reasoning for the numbers.
898    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
899        self.route_bind_breaker_threshold = threshold.max(1);
900        self.route_bind_breaker_cooldown = cooldown;
901        self
902    }
903
904    #[cfg(test)]
905    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
906        self.health_probe_timeout = timeout;
907        self
908    }
909
910    #[cfg(test)]
911    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
912        self.control_dispatch_delay = Some(delay);
913        self
914    }
915
916    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
917    /// announce it is parked, once to be released. Build the barrier for two
918    /// parties, the test being the other one.
919    #[cfg(test)]
920    pub(crate) fn with_route_bind_relay_sent_hold(
921        mut self,
922        hold: Arc<tokio::sync::Barrier>,
923    ) -> Self {
924        self.route_bind_relay_sent_hold = Some(hold);
925        self
926    }
927
928    pub fn with_process_liveness(
929        mut self,
930        process_liveness: Arc<dyn ModuleProcessLiveness>,
931    ) -> Self {
932        self.process_liveness = Some(process_liveness);
933        self
934    }
935
936    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
937        self.supervisor = supervisor;
938        self
939    }
940
941    pub fn with_daemon_provenance(
942        mut self,
943        pid: u32,
944        started_at_ms: u64,
945        executable_path: Option<PathBuf>,
946        build_git_sha: Option<String>,
947        build_lock_digest: Option<String>,
948    ) -> Self {
949        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
950        let process_start_time = process_start_time(pid);
951        self.daemon_provenance = DaemonProvenanceFacts {
952            build: DaemonBuildProvenance {
953                build_git_sha,
954                build_lock_digest,
955            },
956            pid: Some(pid),
957            started_at_ms: Some(started_at_ms),
958            start_clock: None,
959            executable_path,
960            executable_identity,
961            process_start_time,
962            probe: ExecutableIdentityProbe::default(),
963        };
964        self
965    }
966
967    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
968        self.daemon_provenance.start_clock = Some(clock);
969        self
970    }
971
972    #[cfg(test)]
973    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
974        self.provenance_probe_override = Some(result);
975        self
976    }
977
978    /// Install the configured module set and its reserved capability bindings.
979    /// Bindings are configuration-scoped and may point at a provider that has not
980    /// been installed yet, so this does not require the bound module to exist.
981    pub fn with_capability_config(
982        self,
983        modules: impl IntoIterator<Item = (String, bool)>,
984        reserved_capabilities: BTreeMap<String, String>,
985    ) -> Self {
986        self.capability_evaluator
987            .configure(modules, reserved_capabilities);
988        self
989    }
990
991    pub fn with_supervisor_rescan(
992        mut self,
993        supervisor: Supervisor,
994        config_path: impl Into<PathBuf>,
995        configured_port: Option<u16>,
996    ) -> Self {
997        self.rescan = Some(SupervisorRescanContext {
998            supervisor,
999            config_path: config_path.into(),
1000            configured_port,
1001            storage_config: self.storage_config.clone(),
1002            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1003            admission_facts_targets: self.admission_facts_targets.clone(),
1004            scope_authority_owners: self.scope_authority_owners.clone(),
1005        });
1006        self
1007    }
1008
1009    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1010        self.connected_clients = connected_clients;
1011        self
1012    }
1013
1014    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1015        Arc::clone(&self.forwarding)
1016    }
1017
1018    pub(crate) fn counters(&self) -> DaemonCounters {
1019        self.counters.clone()
1020    }
1021
1022    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1023    /// requirement event without depending on an operator polling a status command.
1024    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1025        tokio::spawn(async move {
1026            loop {
1027                self.capability_evaluator
1028                    .wait_for_change_or_deadline()
1029                    .await;
1030                self.refresh_capability_requirements();
1031            }
1032        });
1033    }
1034
1035    fn runtime_capability_snapshot(
1036        &self,
1037    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1038        let runtime = self
1039            .supervisor
1040            .list()
1041            .into_iter()
1042            .map(|module| {
1043                let status = module.status().map_err(|err| {
1044                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1045                })?;
1046                Ok(RuntimeModule {
1047                    module_id: status.module_id,
1048                    state: status.state,
1049                    enabled: status.enabled,
1050                })
1051            })
1052            .collect::<Result<Vec<_>, RouterError>>()?;
1053        let (_, registrations) = self.registry.list_modules().map_err(|err| {
1054            RouterError::backend(
1055                0,
1056                0,
1057                format!("failed to list capability registrations: {err}"),
1058            )
1059        })?;
1060        let registrations = registrations
1061            .into_iter()
1062            .map(|registration| RegisteredModule {
1063                module_id: registration.manifest.module_id,
1064                module_version: registration.manifest.module_version,
1065                capabilities: registration.manifest.capabilities,
1066            })
1067            .collect();
1068        Ok((runtime, registrations))
1069    }
1070
1071    /// The capability side effects of a module becoming the active registration
1072    /// for its id: cache its manifest (warning if its claims drifted), run the
1073    /// deny census when its declarations call for one, and recompute the
1074    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1075    /// candidate's does not, and the supervisor does it at promotion instead,
1076    /// through [`crate::supervise::SwapPromotionObserver`].
1077    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1078        let cached_registration = RegisteredModule {
1079            module_id: registration.manifest.module_id.clone(),
1080            module_version: registration.manifest.module_version.clone(),
1081            capabilities: registration.manifest.capabilities.clone(),
1082        };
1083        if self.capability_evaluator.record_hello(&cached_registration) {
1084            warn!(
1085                module_id = %cached_registration.module_id,
1086                "capability claims drifted from the cached manifest"
1087            );
1088        }
1089        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1090            self.enforce_capability_denies();
1091        }
1092        self.refresh_capability_requirements();
1093    }
1094
1095    /// Point the shared supervisor handle at this handler for swap promotions.
1096    /// Called wherever a handler is put behind the `Arc` the router serves, so
1097    /// it can be held weakly.
1098    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1099        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1100            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1101        self.supervisor.set_swap_promotion_observer(observer);
1102    }
1103
1104    pub fn refresh_capability_requirements(&self) {
1105        match self.runtime_capability_snapshot() {
1106            Ok((runtime, registrations)) => {
1107                log_requirement_events(
1108                    self.capability_evaluator
1109                        .evaluate_now(&runtime, &registrations),
1110                );
1111            }
1112            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1113        }
1114    }
1115
1116    /// Reconcile only live, attested route bindings after a capability deny edge
1117    /// or target claim was added. This is deliberately a control-plane census:
1118    /// the opaque forwarding hot path must not grow a per-frame capability check.
1119    fn enforce_capability_denies(&self) {
1120        let (_, registrations) = match self.registry.list_modules() {
1121            Ok(snapshot) => snapshot,
1122            Err(err) => {
1123                warn!(error = %err, "failed to read registrations for capability deny census");
1124                return;
1125            }
1126        };
1127        let manifests = registrations
1128            .into_iter()
1129            .map(|registration| {
1130                (
1131                    registration.manifest.module_id.clone(),
1132                    registration.manifest,
1133                )
1134            })
1135            .collect::<BTreeMap<_, _>>();
1136        let census = match self.forwarding.route_census(None) {
1137            Ok(census) => census,
1138            Err(err) => {
1139                warn!(error = %err, "failed to read route census for capability deny enforcement");
1140                return;
1141            }
1142        };
1143
1144        for (target_module_id, routes) in census {
1145            let Some(target_manifest) = manifests.get(&target_module_id) else {
1146                continue;
1147            };
1148            let mut closed_routes = Vec::new();
1149            let mut module_goodbyes = Vec::new();
1150            for route in routes {
1151                let Principal::Reserved {
1152                    module_id: opening_module_id,
1153                } = &route.principal
1154                else {
1155                    continue;
1156                };
1157                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1158                    continue;
1159                };
1160                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1161                    continue;
1162                };
1163
1164                match self.forwarding.release_client_route(
1165                    route.goodbye_target.connection_id,
1166                    route.goodbye_target.channel,
1167                    route.goodbye_target.epoch,
1168                ) {
1169                    Ok(RouteRelease::Removed(module_goodbye)) => {
1170                        warn!(
1171                            opening_module_id,
1172                            target_module_id,
1173                            capability,
1174                            "force-closing route because an attested capability deny edge now matches"
1175                        );
1176                        closed_routes.push(route);
1177                        module_goodbyes.push(module_goodbye);
1178                    }
1179                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1180                    Err(err) => warn!(
1181                        opening_module_id,
1182                        target_module_id,
1183                        capability,
1184                        error = %err,
1185                        "failed to force-close capability-denied route"
1186                    ),
1187                }
1188            }
1189
1190            if closed_routes.is_empty() {
1191                continue;
1192            }
1193            send_route_control_pushes(
1194                &self.forwarding,
1195                closed_routes,
1196                ClientControlPush::RouteClosed {
1197                    module_id: target_module_id,
1198                    channels: Vec::new(),
1199                    reason: RouteCloseReason::CapabilityDenied,
1200                    drained: false,
1201                    abandoned: 0,
1202                    excluded_subscriptions: 0,
1203                    terminal: Some(false),
1204                },
1205            );
1206            self.emit_route_goodbyes(module_goodbyes);
1207        }
1208    }
1209
1210    /// Why a registered module is not accepting new route binds, or `None` when
1211    /// it is. This is the module's effective readiness: its declared readiness
1212    /// first, then every `need: required` capability it declares evaluating to
1213    /// `provided`. `route.open` and `catalog.list` both read it here so the
1214    /// catalog never reports a module routable that `route.open` would refuse.
1215    fn not_ready_reason(
1216        &self,
1217        registration: &crate::registry::ModuleRegistration,
1218    ) -> Option<NotReadyReason> {
1219        if !registration.ready {
1220            return Some(NotReadyReason {
1221                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1222                capability: None,
1223            });
1224        }
1225        self.first_unprovided_required_capability(registration)
1226            .map(|capability| NotReadyReason {
1227                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1228                capability: Some(capability),
1229            })
1230    }
1231
1232    /// The lexicographically first capability this registration declares
1233    /// `need: required` whose evaluator verdict is not `provided`.
1234    ///
1235    /// The verdicts are the capability evaluator's own; nothing here decides
1236    /// what "provided" means. The evaluator counts a capability provided as
1237    /// soon as a module claiming it has REGISTERED, not once that module is
1238    /// ready. That distinction is what keeps two modules that require each
1239    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1240    /// is ready", each would wait for the other to become ready first and
1241    /// neither ever would. Do not tighten it to readiness.
1242    ///
1243    /// A required capability with no verdict at all means this registration's
1244    /// HELLO or catalog.update landed after the last recompute; recompute once
1245    /// rather than let a missing verdict read as either answer. If it is still
1246    /// missing (the recompute itself failed) the capability counts as
1247    /// unprovided: the refusal is retryable, and routing a module whose
1248    /// required provider is unknown is the outcome this check exists to stop.
1249    fn first_unprovided_required_capability(
1250        &self,
1251        registration: &crate::registry::ModuleRegistration,
1252    ) -> Option<String> {
1253        let required = registration
1254            .manifest
1255            .capabilities
1256            .iter()
1257            .flat_map(|declarations| declarations.requires.iter())
1258            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1259            .map(|requirement| requirement.capability.as_str())
1260            .collect::<BTreeSet<_>>();
1261        if required.is_empty() {
1262            return None;
1263        }
1264        let module_id = registration.manifest.module_id.as_str();
1265        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1266        if required
1267            .iter()
1268            .any(|capability| verdict(capability).is_none())
1269        {
1270            self.refresh_capability_requirements();
1271        }
1272        required
1273            .into_iter()
1274            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1275            .map(str::to_string)
1276    }
1277
1278    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1279        self.capability_evaluator
1280            .statuses()
1281            .into_iter()
1282            .map(capability_requirement_status)
1283            .collect()
1284    }
1285
1286    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1287    /// registration-release watch. The signal is what the supervisor waits on
1288    /// before spawning a replacement, so it must only fire once forwarding
1289    /// teardown is also done (see [`Self::cleanup_connection`] /
1290    /// [`Self::handle_goodbye`]). Used directly only where there is no forwarding
1291    /// state to tear down (a HELLO that failed before module registration).
1292    fn deregister_connection(
1293        &self,
1294        connection_id: ConnectionId,
1295    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1296        self.registry.deregister_connection(connection_id)
1297    }
1298
1299    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1300        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1301            return None;
1302        }
1303        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1304            parse_client_control_request(&frame.body)
1305        else {
1306            return None;
1307        };
1308        Some(target_module_id(&target).to_string())
1309    }
1310
1311    pub(crate) fn route_open_capacity_refusal(
1312        &self,
1313        ctx: &RouteCtx,
1314        frame: &Frame,
1315        target_module_id: &str,
1316        in_flight: usize,
1317        limit: usize,
1318    ) -> Result<Frame, RouterError> {
1319        self.route_open_admission_refusal_frame(
1320            ctx,
1321            frame,
1322            target_module_id,
1323            "open_admission_full",
1324            (in_flight, limit),
1325            format!(
1326                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1327            ),
1328        )
1329    }
1330
1331    fn route_open_target_capacity_refusal(
1332        &self,
1333        ctx: &RouteCtx,
1334        frame: &Frame,
1335        target_module_id: &str,
1336        in_flight: usize,
1337    ) -> Result<Frame, RouterError> {
1338        self.route_open_admission_refusal_frame(
1339            ctx,
1340            frame,
1341            target_module_id,
1342            "target_binds_full",
1343            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1344            format!(
1345                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1346            ),
1347        )
1348    }
1349
1350    /// Admission pressure clears as existing binds settle, so its refusal must
1351    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1352    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1353    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1354    /// cannot currently reach its target; `module_timeout` would falsely claim
1355    /// that a wait expired. A new, cleaner code would be terminal to deployed
1356    /// clients, so it requires a client-tolerance rollout before daemon emission.
1357    fn route_open_admission_refusal_frame(
1358        &self,
1359        ctx: &RouteCtx,
1360        frame: &Frame,
1361        target_module_id: &str,
1362        reason: &'static str,
1363        (in_flight, limit): (usize, usize),
1364        message: impl Into<String>,
1365    ) -> Result<Frame, RouterError> {
1366        let code = error_codes::TARGET_UNAVAILABLE;
1367        self.counters.increment_route_open_refused(code);
1368        info!(
1369            target: "control",
1370            code,
1371            reason,
1372            module_id = ?target_module_id,
1373            connection_id = ctx.connection_id.get(),
1374            in_flight,
1375            limit,
1376            "route.open refused"
1377        );
1378        control_error_frame(frame, code, message.into())
1379    }
1380
1381    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1382    ///
1383    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1384    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1385    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1386    #[cfg(test)]
1387    pub fn handle_control(
1388        &self,
1389        connection_id: ConnectionId,
1390        frame: Frame,
1391    ) -> Result<Vec<Frame>, RouterError> {
1392        match frame.header.ty {
1393            FrameType::Ping => Ok(vec![pong(&frame)?]),
1394            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1395            FrameType::Goodbye => self.handle_goodbye(connection_id),
1396            ty => Ok(vec![control_error_frame(
1397                &frame,
1398                "unsupported_control_frame",
1399                format!("unsupported channel-0 frame {ty:?}"),
1400            )?]),
1401        }
1402    }
1403
1404    pub async fn handle_control_frame(
1405        &self,
1406        ctx: &RouteCtx,
1407        frame: Frame,
1408    ) -> Result<Vec<Frame>, RouterError> {
1409        self.handle_control_frame_timed(ctx, frame, None).await
1410    }
1411
1412    pub(crate) async fn handle_control_frame_timed(
1413        &self,
1414        ctx: &RouteCtx,
1415        frame: Frame,
1416        dispatch_started_at: Option<StdInstant>,
1417    ) -> Result<Vec<Frame>, RouterError> {
1418        match frame.header.ty {
1419            FrameType::Ping => Ok(vec![pong(&frame)?]),
1420            FrameType::Hello => {
1421                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1422            }
1423            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1424            FrameType::Cancel => {
1425                if self
1426                    .supervisor
1427                    .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1428                {
1429                    Ok(Vec::new())
1430                } else {
1431                    Ok(vec![control_error_frame(
1432                        &frame,
1433                        "unknown_subscription",
1434                        "no supervisor spawn subscription has this correlation id",
1435                    )?])
1436                }
1437            }
1438            FrameType::Request => {
1439                if self
1440                    .forwarding
1441                    .module_endpoint_for_connection(ctx.connection_id)
1442                    .map_err(RouterError::Forwarding)?
1443                    .is_some()
1444                {
1445                    if !is_known_module_request_op(&frame.body) {
1446                        return Ok(vec![control_error_frame(
1447                            &frame,
1448                            "unsupported_control_frame",
1449                            "module-originated channel-0 REQUEST is not supported",
1450                        )?]);
1451                    }
1452                    let request = match parse_module_control_request_from_module(&frame.body) {
1453                        Ok(request) => request,
1454                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1455                            return Ok(vec![control_error_frame(
1456                                &frame,
1457                                "unsupported_control_frame",
1458                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1459                            )?])
1460                        }
1461                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1462                            return Ok(vec![control_error_frame(
1463                                &frame,
1464                                "invalid_control_body",
1465                                format!("malformed module control body: {err}"),
1466                            )?])
1467                        }
1468                    };
1469                    let op = module_control_request_op(&request);
1470                    let corr = frame.header.corr;
1471                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1472                    let result =
1473                        self.handle_module_control_request(ctx.connection_id, frame, request);
1474                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1475                    return result;
1476                }
1477
1478                if is_known_module_request_op(&frame.body) {
1479                    return Ok(vec![control_error_frame(
1480                        &frame,
1481                        "not_registered",
1482                        "catalog.update requires an active module registration owned by this connection",
1483                    )?]);
1484                }
1485
1486                let request = match parse_client_control_request(&frame.body) {
1487                    Ok(request) => request,
1488                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1489                        return Ok(vec![control_error_frame(
1490                            &frame,
1491                            "unknown_control_op",
1492                            format!("unknown client control op: {err}"),
1493                        )?])
1494                    }
1495                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1496                        return Ok(vec![control_error_frame(
1497                            &frame,
1498                            "invalid_control_body",
1499                            format!("malformed client control body: {err}"),
1500                        )?])
1501                    }
1502                };
1503                let op = client_control_request_op(&request);
1504                let corr = frame.header.corr;
1505                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1506                #[cfg(test)]
1507                if let Some(delay) = self.control_dispatch_delay {
1508                    tokio::time::sleep(delay).await;
1509                }
1510                let result = self
1511                    .handle_client_control_request(ctx, frame, request)
1512                    .await;
1513                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1514                result
1515            }
1516            FrameType::Push => {
1517                let Some(endpoint) = self
1518                    .forwarding
1519                    .module_endpoint_for_connection(ctx.connection_id)
1520                    .map_err(RouterError::Forwarding)?
1521                else {
1522                    return Ok(vec![control_error_frame(
1523                        &frame,
1524                        "unsupported_control_frame",
1525                        "client-originated channel-0 PUSH is not supported",
1526                    )?]);
1527                };
1528                self.handle_status_update(endpoint, frame)
1529            }
1530            FrameType::Response | FrameType::Error
1531                if self
1532                    .forwarding
1533                    .module_endpoint_for_connection(ctx.connection_id)
1534                    .map_err(RouterError::Forwarding)?
1535                    .is_some() =>
1536            {
1537                self.handle_module_relay_response(ctx.connection_id, frame)
1538            }
1539            ty => Ok(vec![control_error_frame(
1540                &frame,
1541                "unsupported_control_frame",
1542                format!("unsupported channel-0 frame {ty:?}"),
1543            )?]),
1544        }
1545    }
1546
1547    pub fn cleanup_connection(
1548        &self,
1549        connection_id: ConnectionId,
1550    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1551        let crash_closed = self
1552            .registry
1553            .get_module_by_connection(connection_id)?
1554            .and_then(|registration| {
1555                self.forwarding
1556                    .module_endpoint_for_connection(connection_id)
1557                    .ok()
1558                    .flatten()
1559                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1560                    .map(|routes| (registration.manifest.module_id, routes))
1561            });
1562        let crash_closed = crash_closed.map(|(module_id, routes)| {
1563            let terminal = match self.supervisor.get(&module_id) {
1564                None => false,
1565                Some(module) => match module.will_recover_after_connection_loss() {
1566                    Ok(will_recover) => !will_recover,
1567                    Err(err) => {
1568                        warn!(
1569                            %module_id,
1570                            error = %err,
1571                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1572                        );
1573                        false
1574                    }
1575                },
1576            };
1577            // The forwarding table gates all providers at the start of daemon
1578            // shutdown, before their connections are closed. An ordinary
1579            // module disconnect still reports crash if that gate is not set.
1580            let reason = match self.forwarding.is_daemon_draining() {
1581                Ok(true) => RouteCloseReason::Restart,
1582                Ok(false) => RouteCloseReason::Crash,
1583                Err(err) => {
1584                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1585                    RouteCloseReason::Crash
1586                }
1587            };
1588            (module_id, routes, reason, terminal)
1589        });
1590        let registrations = self.deregister_connection(connection_id);
1591        let cleanup = if crash_closed.is_some() {
1592            self.forwarding.cleanup_connection_counted(connection_id)
1593        } else {
1594            // A module connection's teardown needs the count of abandoned
1595            // route.bind relays for its route.closed notice below. Any other
1596            // connection, such as a client's, sends no such notice and needs
1597            // only its routes released, so it uses the route-only wrapper and
1598            // reports zero.
1599            self.forwarding
1600                .cleanup_connection(connection_id)
1601                .map(|released| crate::forwarding::ConnectionCleanup {
1602                    released,
1603                    abandoned_relays: 0,
1604                })
1605        };
1606        // The route.closed push waits for forwarding teardown because only
1607        // teardown knows how many pending route.bind relays it aborted. It still
1608        // goes out before the GOODBYEs for the released routes, and its targets
1609        // were captured above, before teardown removed those routes.
1610        if let Some((module_id, routes, reason, terminal)) = crash_closed {
1611            let abandoned = cleanup
1612                .as_ref()
1613                .map_or(0, |cleanup| cleanup.abandoned_relays);
1614            send_route_control_pushes(
1615                &self.forwarding,
1616                routes,
1617                ClientControlPush::RouteClosed {
1618                    module_id,
1619                    channels: Vec::new(),
1620                    reason,
1621                    drained: false,
1622                    abandoned,
1623                    excluded_subscriptions: 0,
1624                    terminal: Some(terminal),
1625                },
1626            );
1627        }
1628        if let Ok(cleanup) = cleanup {
1629            self.emit_route_goodbyes(cleanup.released);
1630        }
1631        // Signal the registration-release watch only now that BOTH registry and
1632        // forwarding teardown are done, so a supervisor waiting to spawn a
1633        // replacement never observes release while old routes still exist.
1634        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1635            crate::supervise::notify_registration_release();
1636            self.capability_evaluator.wake_deadline_loop();
1637            self.refresh_capability_requirements();
1638        }
1639        self.supervisor.remove_spawn_subscribers(connection_id);
1640        // Sync authority dies with its connection, so the owner's next
1641        // connection can take it; the owner's scopes stay as they are.
1642        self.hello_launch_nonces
1643            .lock()
1644            .unwrap_or_else(|poisoned| poisoned.into_inner())
1645            .forget(connection_id);
1646        self.scopes
1647            .write()
1648            .unwrap_or_else(|poisoned| poisoned.into_inner())
1649            .release_connection(connection_id);
1650        registrations
1651    }
1652
1653    pub(crate) fn handle_route_goodbye(
1654        &self,
1655        connection_id: ConnectionId,
1656        route_channel: u16,
1657        route_epoch: u32,
1658    ) -> Result<bool, RouterError> {
1659        debug!(
1660            connection_id = connection_id.get(),
1661            route_channel, route_epoch, "handling route GOODBYE"
1662        );
1663        let RouteRelease::Removed(released_route) = self
1664            .forwarding
1665            .release_client_route(connection_id, route_channel, route_epoch)
1666            .map_err(RouterError::Forwarding)?
1667        else {
1668            return Ok(false);
1669        };
1670        self.emit_route_goodbyes(vec![released_route]);
1671        Ok(true)
1672    }
1673
1674    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1675        for released in released_routes {
1676            let frame = match Frame::build_with_version(
1677                released.negotiated_ver,
1678                FrameType::Goodbye,
1679                control_flags(),
1680                released.channel,
1681                released.epoch,
1682                0,
1683                Vec::new(),
1684            ) {
1685                Ok(frame) => frame,
1686                Err(err) => {
1687                    warn!(
1688                        route_channel = released.channel,
1689                        error = %err,
1690                        "failed to build route GOODBYE frame"
1691                    );
1692                    continue;
1693                }
1694            };
1695            if !released.close_on_delivery_failure() {
1696                crate::forwarding::send_module_route_goodbye(
1697                    &self.counters,
1698                    &released.sink,
1699                    frame,
1700                    released.module_id.as_deref(),
1701                    "client route released",
1702                );
1703                continue;
1704            }
1705            if let Err(err) = released.sink.try_send(frame) {
1706                warn!(
1707                    target_connection_id = released.connection_id.get(),
1708                    route_channel = released.channel,
1709                    error = %err,
1710                    "route GOODBYE was not delivered to client; closing target connection"
1711                );
1712                if self
1713                    .forwarding
1714                    .escalate_client_delivery_failure(
1715                        released.connection_id,
1716                        released.channel,
1717                        released.epoch,
1718                        CloseReason::new(
1719                            "route_goodbye_delivery_failed",
1720                            format!(
1721                                "failed to enqueue route GOODBYE for channel {}: {err}",
1722                                released.channel
1723                            ),
1724                        ),
1725                        crate::forwarding::UndeliveredFrame {
1726                            module_id: released.module_id.as_deref(),
1727                            sink: &released.sink,
1728                        },
1729                    )
1730                    .unwrap_or(false)
1731                {
1732                    self.counters.increment_goodbye_relay_client_failed();
1733                }
1734            }
1735        }
1736    }
1737
1738    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1739    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1740    /// own commit failed after the module had already accepted). Without this, a
1741    /// module that accepts late keeps a binding subc has torn down, so a later
1742    /// frame on that module channel could misdeliver if the channel is reused.
1743    ///
1744    /// Never closes the shared module connection on failure: a dropped notification
1745    /// only wastes a bounded amount of warm module-side state, which the module's
1746    /// own idle reaper reclaims. Only call this once the route.bind relay was
1747    /// actually enqueued to the module — if the relay send itself failed, the
1748    /// module never created a binding and there is nothing to tear down.
1749    fn send_abandoned_route_bind_goodbye(
1750        &self,
1751        module_sink: &crate::FrameSink,
1752        negotiated_ver: u8,
1753        module_channel: u16,
1754        module_epoch: u32,
1755    ) {
1756        let frame = match Frame::build_with_version(
1757            negotiated_ver,
1758            FrameType::Goodbye,
1759            control_flags(),
1760            module_channel,
1761            module_epoch,
1762            0,
1763            Vec::new(),
1764        ) {
1765            Ok(frame) => frame,
1766            Err(err) => {
1767                warn!(
1768                    route_channel = module_channel,
1769                    error = %err,
1770                    "failed to build GOODBYE for abandoned route.bind"
1771                );
1772                return;
1773            }
1774        };
1775        crate::forwarding::send_module_route_goodbye(
1776            &self.counters,
1777            module_sink,
1778            frame,
1779            None,
1780            "abandoned route.bind",
1781        );
1782    }
1783
1784    fn handle_hello(
1785        &self,
1786        connection_id: ConnectionId,
1787        sink: Option<crate::FrameSink>,
1788        frame: Frame,
1789    ) -> Result<Vec<Frame>, RouterError> {
1790        debug!(
1791            connection_id = connection_id.get(),
1792            corr = frame.header.corr,
1793            "handling HELLO"
1794        );
1795        // A module connection has one identity for its entire lifetime. A second
1796        // registration would leave the old registry owner behind while replacing
1797        // its forwarding endpoint and launch nonce.
1798        if self
1799            .registry
1800            .get_module_by_connection(connection_id)
1801            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1802            .is_some()
1803        {
1804            return Ok(vec![control_error_frame(
1805                &frame,
1806                "invalid_hello",
1807                "connection is already registered as a module",
1808            )?]);
1809        }
1810        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1811            Ok(value) => value,
1812            Err(err) => {
1813                return Ok(vec![control_error_frame(
1814                    &frame,
1815                    "invalid_hello",
1816                    format!("malformed HELLO body: {err}"),
1817                )?])
1818            }
1819        };
1820        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1821            return Ok(vec![control_error_frame(
1822                &frame,
1823                "invalid_capability_grammar",
1824                err.to_string(),
1825            )?]);
1826        }
1827        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1828            return Ok(vec![control_error_frame(
1829                &frame,
1830                "invalid_manifest",
1831                err.to_string(),
1832            )?]);
1833        }
1834        if let Some(provenance) = hello_value
1835            .get("manifest")
1836            .and_then(|manifest| manifest.get("provenance"))
1837        {
1838            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1839                return Ok(vec![control_error_frame(
1840                    &frame,
1841                    "invalid_manifest",
1842                    format!("malformed manifest provenance: {err}"),
1843                )?]);
1844            }
1845        }
1846        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1847            Ok(hello) => hello,
1848            Err(err) => {
1849                return Ok(vec![control_error_frame(
1850                    &frame,
1851                    "invalid_hello",
1852                    format!("malformed HELLO body: {err}"),
1853                )?])
1854            }
1855        };
1856
1857        if hello.protocol_ver != hello.manifest.protocol_ver {
1858            return Ok(vec![control_error_frame(
1859                &frame,
1860                "invalid_manifest",
1861                format!(
1862                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1863                    hello.protocol_ver, hello.manifest.protocol_ver
1864                ),
1865            )?]);
1866        }
1867
1868        if hello.manifest.module_id.trim().is_empty() {
1869            return Ok(vec![control_error_frame(
1870                &frame,
1871                "invalid_manifest",
1872                "manifest module_id must not be empty",
1873            )?]);
1874        }
1875
1876        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1877            Ok(negotiated_ver) => negotiated_ver,
1878            Err(message) => {
1879                return Ok(vec![control_error_frame(
1880                    &frame,
1881                    "version_unsupported",
1882                    message,
1883                )?])
1884            }
1885        };
1886
1887        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
1888        // swap is open for this id, the only HELLO admitted as a second process
1889        // is the one carrying the candidate's launch nonce (the swap token), and
1890        // it registers into the candidate slot rather than being refused as a
1891        // duplicate. Run after the reserved gate, a reserved module's candidate
1892        // would be refused `reserved_module` for presenting a nonce that gate
1893        // does not know. See `SupervisorHandle::swap_hello_admission`.
1894        let swap_admission = self
1895            .supervisor
1896            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1897        if swap_admission == SwapHelloAdmission::Refused {
1898            warn!(
1899                module_id = %hello.manifest.module_id,
1900                connection_id = connection_id.get(),
1901                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1902            );
1903            return Ok(vec![control_error_frame(
1904                &frame,
1905                "swap_token_invalid",
1906                format!(
1907                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1908                    hello.manifest.module_id
1909                ),
1910            )?]);
1911        }
1912        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1913
1914        // Reserved-module identity gate: a module_id configured `reserved` may be
1915        // registered ONLY by the process subc spawned for it, proven by echoing the
1916        // one-time launch nonce subc injected. A non-reserved id has no recorded
1917        // nonce and always passes. This blocks a key-holder from impersonating a
1918        // security-boundary module (e.g. the credential vault) while the real one is
1919        // down/restarting and its registration slot is momentarily free. A swap
1920        // candidate has already proven the same thing with its own nonce above.
1921        if let Some(rejection) = (!swap_candidate)
1922            .then(|| {
1923                self.supervisor.reserved_hello_rejection(
1924                    &hello.manifest.module_id,
1925                    hello.launch_nonce.as_deref(),
1926                )
1927            })
1928            .flatten()
1929        {
1930            let message = match rejection {
1931                ReservedHelloRejection::Exact { module_id } => format!(
1932                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1933                ),
1934                ReservedHelloRejection::Prefix {
1935                    prefix,
1936                    owner_module_id,
1937                } => format!(
1938                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1939                    hello.manifest.module_id
1940                ),
1941            };
1942            return Ok(vec![control_error_frame(
1943                &frame,
1944                "reserved_module",
1945                message,
1946            )?]);
1947        }
1948
1949        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1950            &hello.manifest.module_id,
1951            hello.manifest.capabilities.as_ref(),
1952        );
1953        if let Some(refusal) = reserved_capability_refusals.first() {
1954            let capability = refusal.capability.clone();
1955            let bound_module = refusal.claimants[0].clone();
1956            log_duplicate_claim_events(reserved_capability_refusals);
1957            return Ok(vec![control_error_frame(
1958                &frame,
1959                "reserved_capability",
1960                format!(
1961                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1962                    capability, bound_module, hello.manifest.module_id
1963                ),
1964            )?]);
1965        }
1966
1967        // A connection that already opened client routes must not also register as
1968        // a module: cleanup would then release only one side and leak the other.
1969        if self
1970            .forwarding
1971            .connection_has_client_routes(connection_id)
1972            .map_err(RouterError::Forwarding)?
1973        {
1974            return Ok(vec![control_error_frame(
1975                &frame,
1976                "invalid_hello",
1977                "connection has open client routes and cannot also register as a module",
1978            )?]);
1979        }
1980
1981        // Kept for scope sync authority, which goes only to the connection that
1982        // presented the module's current launch nonce. Recorded before the
1983        // registration is attempted: a connection whose registration then fails
1984        // has no registration, so it cannot sync anyway, and cleanup forgets it.
1985        self.hello_launch_nonces
1986            .lock()
1987            .unwrap_or_else(|poisoned| poisoned.into_inner())
1988            .record(connection_id, hello.launch_nonce.as_deref());
1989        let control_ops = effective_module_control_ops(hello.control_ops);
1990        // Built before anything is registered so an encoding failure leaves no
1991        // registry or forwarding state behind.
1992        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1993        if swap_candidate {
1994            return self.register_swap_candidate(
1995                connection_id,
1996                sink,
1997                &frame,
1998                hello.manifest,
1999                negotiated_ver,
2000                control_ops,
2001                hello_ack,
2002            );
2003        }
2004        let registration = match self.registry.register_with_control_ops(
2005            hello.manifest,
2006            negotiated_ver,
2007            connection_id,
2008            control_ops,
2009        ) {
2010            Ok(registration) => registration,
2011            Err(RegistryError::DuplicateModuleId { module_id }) => {
2012                return Ok(vec![control_error_frame(
2013                    &frame,
2014                    "duplicate_module_id",
2015                    format!(
2016                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2017                    ),
2018                )?])
2019            }
2020            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2021                return Ok(vec![control_error_frame(
2022                    &frame,
2023                    "invalid_module_id",
2024                    err.to_string(),
2025                )?])
2026            }
2027            Err(err) => {
2028                return Ok(vec![control_error_frame(
2029                    &frame,
2030                    "registry_error",
2031                    err.to_string(),
2032                )?])
2033            }
2034        };
2035
2036        let reply = if let Some(sink) = sink {
2037            // The forwarding table's module store is also the daemon-to-module
2038            // control-RPC lane, so every HELLO gets a live endpoint even when the
2039            // manifest has no routable provider role. Non-routable modules still
2040            // cannot receive route.bind in production: `handle_route_open` checks
2041            // the registry manifest with `target_has_required_role` before the
2042            // only production call to `begin_route_bind_relay_for` below that
2043            // route.open path. The remaining direct relay callers are unit tests
2044            // and benchmark harnesses that construct forwarding state explicitly.
2045            //
2046            // The HELLO_ACK is queued by the forwarding table itself, before the
2047            // endpoint becomes visible, and is NOT returned as a reply. A module
2048            // reads HELLO_ACK first and exits on anything else; a reply is only
2049            // written after this handler returns, by which time a route.open on
2050            // another connection could already have queued a route.bind request
2051            // for this module ahead of it.
2052            let concurrency = manifest_concurrency(&registration.manifest);
2053            if let Err(err) = self.forwarding.register_module_connection_acked(
2054                connection_id,
2055                registration.manifest.module_id.clone(),
2056                negotiated_ver,
2057                concurrency,
2058                sink,
2059                hello_ack,
2060            ) {
2061                // Forwarding registration failed, so there is no forwarding
2062                // state to tear down. Remove the registry entry and signal the
2063                // release watch directly.
2064                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2065                    crate::supervise::notify_registration_release();
2066                }
2067                return Ok(vec![control_error_frame(
2068                    &frame,
2069                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2070                        "invalid_hello"
2071                    } else {
2072                        forwarding_error_code(&err)
2073                    },
2074                    err.to_string(),
2075                )?]);
2076            }
2077            Vec::new()
2078        } else {
2079            // No sink means no forwarding endpoint, so nothing can be routed
2080            // ahead of the ack; it goes out as the reply.
2081            vec![hello_ack]
2082        };
2083
2084        // Exposure over assumption: Concurrency's serde default is pinned to the
2085        // pre-field behavior (ModuleManaged), so a management surface that is
2086        // genuinely Serial and just never declared it inherits concurrent
2087        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2088        // turns "no module has been bitten yet" into the checkable claim "no
2089        // module is exposed" -- one read of the boot log instead of a fleet
2090        // audit. Detected from the raw HELLO bytes because the serde default
2091        // deliberately erases the absent/declared distinction from the type.
2092        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2093            info!(
2094                module_id = %registration.manifest.module_id,
2095                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2096            );
2097        }
2098
2099        self.apply_registration_capabilities(&registration);
2100
2101        info!(
2102            module_id = %registration.manifest.module_id,
2103            module_version = %registration.manifest.module_version,
2104            negotiated_ver,
2105            routable_provider = manifest_provides_routable_role(&registration.manifest),
2106            connection_id = connection_id.get(),
2107            "module registered"
2108        );
2109
2110        Ok(reply)
2111    }
2112
2113    /// Register a HELLO the swap gate admitted into the candidate slot of the
2114    /// registry and of forwarding, where it is reachable over its own
2115    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2116    /// nothing routes to it until the supervisor cuts over.
2117    ///
2118    /// Registry first, then forwarding, the same order as an ordinary HELLO;
2119    /// a forwarding failure removes the registry entry again. The capability
2120    /// census is not run: it describes routable modules, and this one is not
2121    /// routable until promotion.
2122    #[allow(clippy::too_many_arguments)]
2123    fn register_swap_candidate(
2124        &self,
2125        connection_id: ConnectionId,
2126        sink: Option<crate::FrameSink>,
2127        frame: &Frame,
2128        manifest: ModuleManifest,
2129        negotiated_ver: u8,
2130        control_ops: Vec<String>,
2131        hello_ack: Frame,
2132    ) -> Result<Vec<Frame>, RouterError> {
2133        let module_id = manifest.module_id.clone();
2134        let registration = match self.registry.register_candidate_with_control_ops(
2135            manifest,
2136            negotiated_ver,
2137            connection_id,
2138            control_ops,
2139        ) {
2140            Ok(registration) => registration,
2141            Err(RegistryError::DuplicateModuleId { module_id }) => {
2142                return Ok(vec![control_error_frame(
2143                    frame,
2144                    "duplicate_module_id",
2145                    format!(
2146                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2147                    ),
2148                )?])
2149            }
2150            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2151                return Ok(vec![control_error_frame(
2152                    frame,
2153                    "invalid_module_id",
2154                    err.to_string(),
2155                )?])
2156            }
2157            Err(err) => {
2158                return Ok(vec![control_error_frame(
2159                    frame,
2160                    "registry_error",
2161                    err.to_string(),
2162                )?])
2163            }
2164        };
2165        let reply = if let Some(sink) = sink {
2166            // Same ordering as an ordinary HELLO: the forwarding table queues
2167            // the HELLO_ACK before the candidate endpoint is inserted, because
2168            // a module exits if its first frame after HELLO is anything else.
2169            let concurrency = manifest_concurrency(&registration.manifest);
2170            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2171                connection_id,
2172                module_id.clone(),
2173                negotiated_ver,
2174                concurrency,
2175                sink,
2176                hello_ack,
2177            ) {
2178                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2179                    crate::supervise::notify_registration_release();
2180                }
2181                return Ok(vec![control_error_frame(
2182                    frame,
2183                    forwarding_error_code(&err),
2184                    err.to_string(),
2185                )?]);
2186            }
2187            Vec::new()
2188        } else {
2189            vec![hello_ack]
2190        };
2191        self.supervisor.mark_swap_candidate_admitted(&module_id);
2192        info!(
2193            module_id = %module_id,
2194            module_version = %registration.manifest.module_version,
2195            negotiated_ver,
2196            ready = registration.ready,
2197            connection_id = connection_id.get(),
2198            "swap candidate registered; not routable until cutover"
2199        );
2200        Ok(reply)
2201    }
2202
2203    fn build_hello_ack(
2204        &self,
2205        frame: &Frame,
2206        negotiated_ver: u8,
2207        module_id: &str,
2208    ) -> Result<Frame, RouterError> {
2209        let ack = ModuleHelloAckBody {
2210            negotiated_ver,
2211            subc_ops: module_subc_ops(),
2212            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2213            storage: self
2214                .storage_config
2215                .as_ref()
2216                .map(|cfg| cfg.descriptor_for(module_id)),
2217            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2218        };
2219        let body = serde_json::to_vec(&ack).map_err(|err| {
2220            RouterError::backend(
2221                0,
2222                frame.header.corr,
2223                format!("failed to encode HELLO_ACK: {err}"),
2224            )
2225        })?;
2226
2227        Frame::build_with_version(
2228            negotiated_ver,
2229            FrameType::HelloAck,
2230            control_flags(),
2231            0,
2232            0,
2233            frame.header.corr,
2234            body,
2235        )
2236        .map_err(RouterError::FrameBuild)
2237    }
2238
2239    async fn handle_client_control_request(
2240        &self,
2241        ctx: &RouteCtx,
2242        frame: Frame,
2243        request: ClientControlRequest,
2244    ) -> Result<Vec<Frame>, RouterError> {
2245        match request {
2246            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2247            ClientControlRequest::CatalogList { module_id } => {
2248                self.handle_catalog_list(frame, module_id)
2249            }
2250            ClientControlRequest::RouteOpen {
2251                target,
2252                identity,
2253                consumer_identity,
2254                consumer_capabilities,
2255                role_versions,
2256                admission_facts,
2257                scope,
2258            } => {
2259                self.handle_route_open(
2260                    ctx,
2261                    frame,
2262                    RouteOpenRequest {
2263                        target,
2264                        identity,
2265                        consumer_identity,
2266                        consumer_capabilities,
2267                        role_versions,
2268                        admission_facts,
2269                        scope,
2270                    },
2271                )
2272                .await
2273            }
2274            ClientControlRequest::RoutePoll {
2275                route_channel,
2276                route_epoch,
2277                kind,
2278            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2279            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2280            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2281                self.handle_supervisor_spawn_snapshot(frame)
2282            }
2283            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2284                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2285            }
2286            ClientControlRequest::SupervisorRestart {
2287                module_id,
2288                drain_timeout_ms,
2289            } => {
2290                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2291                    .await
2292            }
2293            ClientControlRequest::SupervisorSwap {
2294                module_id,
2295                ready_timeout_ms,
2296            } => {
2297                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2298                    .await
2299            }
2300            ClientControlRequest::SupervisorReload { module_id } => {
2301                self.handle_supervisor_reload(frame, module_id).await
2302            }
2303            ClientControlRequest::SupervisorRescan { preview } => {
2304                self.handle_supervisor_rescan(frame, preview).await
2305            }
2306            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2307                self.handle_supervisor_release_reserved(frame, module_id)
2308                    .await
2309            }
2310            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2311                self.handle_supervisor_set_enabled(frame, module_id, enabled)
2312                    .await
2313            }
2314            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2315                self.handle_supervisor_health_probe(frame, module_id).await
2316            }
2317            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2318            ClientControlRequest::SupervisorRoutes { module_id } => {
2319                self.handle_supervisor_routes(frame, module_id)
2320            }
2321            ClientControlRequest::SupervisorProvenance { module_id } => {
2322                self.handle_supervisor_provenance(frame, module_id).await
2323            }
2324            ClientControlRequest::SupervisorStderrTail {
2325                module_id,
2326                max_lines,
2327                max_bytes,
2328            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2329            ClientControlRequest::SupervisorTerminals { module_id } => {
2330                self.handle_supervisor_terminals(frame, module_id).await
2331            }
2332        }
2333    }
2334
2335    fn handle_module_control_request(
2336        &self,
2337        connection_id: ConnectionId,
2338        frame: Frame,
2339        request: ModuleControlRequestFromModule,
2340    ) -> Result<Vec<Frame>, RouterError> {
2341        match request {
2342            ModuleControlRequestFromModule::CatalogUpdate {
2343                provides,
2344                capabilities,
2345                ready,
2346            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2347            ModuleControlRequestFromModule::LiveRoots {} => {
2348                let registered = self
2349                    .registry
2350                    .get_module_by_connection(connection_id)
2351                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2352                let Some(registration) = registered else {
2353                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2354                };
2355                let response = self
2356                    .forwarding
2357                    .live_roots(&registration.manifest.module_id)
2358                    .map_err(RouterError::Forwarding)?;
2359                Ok(vec![control_response_body_frame(
2360                    &frame,
2361                    &response,
2362                    "ModuleControlResponseToModule::LiveRoots",
2363                )?])
2364            }
2365            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2366                self.handle_scope_sync(connection_id, frame, generation, scopes)
2367            }
2368            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2369                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2370            }
2371        }
2372    }
2373
2374    /// `scope.sync`: the owner is the module registered on this connection.
2375    /// A connection with no registration (every client connection, `direct`
2376    /// included) is refused `not_registered` before the table is consulted.
2377    fn handle_scope_sync(
2378        &self,
2379        connection_id: ConnectionId,
2380        frame: Frame,
2381        generation: u64,
2382        scopes: Vec<ScopeRecord>,
2383    ) -> Result<Vec<Frame>, RouterError> {
2384        let Some(registration) = self
2385            .registry
2386            .get_module_by_connection(connection_id)
2387            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2388        else {
2389            return Ok(vec![control_error_frame(
2390                &frame,
2391                "not_registered",
2392                "scope.sync requires an active module registration owned by this connection",
2393            )?]);
2394        };
2395        let owner = registration.manifest.module_id;
2396        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2397        let is_current_launch = |connection: ConnectionId| {
2398            self.hello_launch_nonces
2399                .lock()
2400                .unwrap_or_else(|poisoned| poisoned.into_inner())
2401                .presented(connection, current_nonce.as_deref())
2402        };
2403        // Lock order is the scope table, then the forwarding table: the new
2404        // tags are published, and the routes the change closes are selected,
2405        // while the scope table is still write-locked, so no admission can read
2406        // a record whose tag is not yet published.
2407        let mut table = self
2408            .scopes
2409            .write()
2410            .unwrap_or_else(|poisoned| poisoned.into_inner());
2411        let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2412        let drained = match &outcome {
2413            Ok(applied) => self
2414                .forwarding
2415                .publish_scope_changes(&applied.tag_changes)
2416                .map_err(RouterError::Forwarding)?,
2417            Err(_) => Vec::new(),
2418        };
2419        drop(table);
2420        match outcome {
2421            Ok(applied) => {
2422                let counts = ScopeOutcomeCounts::of(&applied.results);
2423                info!(
2424                    owner = %owner,
2425                    generation,
2426                    records = applied.results.len(),
2427                    created = counts.created,
2428                    replaced = counts.replaced,
2429                    updated = counts.updated,
2430                    unchanged = counts.unchanged,
2431                    refused = counts.refused,
2432                    ended = applied.ended.len(),
2433                    tag_changes = applied.tag_changes.len(),
2434                    routes_closed = drained.len(),
2435                    "scope sync accepted"
2436                );
2437                // An accepted sync can still refuse individual records, and the
2438                // owner is the only party that sees the reply. Name them here so
2439                // an operator can tell a refused session from a missing one
2440                // without the owner's logs. Capped so a sync that refuses
2441                // thousands cannot flood the log; the count above is complete.
2442                for refused in applied
2443                    .results
2444                    .iter()
2445                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2446                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2447                {
2448                    warn!(
2449                        owner = %owner,
2450                        generation,
2451                        scope_ref = %refused.scope_ref,
2452                        scope_epoch = refused.scope_epoch,
2453                        code = refused.code.as_deref().unwrap_or(""),
2454                        "scope record refused"
2455                    );
2456                }
2457                self.close_scope_drained_routes(drained);
2458                let response = ModuleControlResponseToModule::ScopeSync {
2459                    generation,
2460                    results: applied.results,
2461                    ended: applied.ended,
2462                };
2463                Ok(vec![control_response_body_frame(
2464                    &frame,
2465                    &response,
2466                    "ModuleControlResponseToModule::ScopeSync",
2467                )?])
2468            }
2469            Err(refusal) => {
2470                info!(
2471                    owner = %owner,
2472                    generation,
2473                    code = refusal.code,
2474                    "scope sync refused"
2475                );
2476                Ok(vec![control_error_frame(
2477                    &frame,
2478                    refusal.code,
2479                    refusal.message,
2480                )?])
2481            }
2482        }
2483    }
2484
2485    /// Tell both ends of each route a scope change closed. The module gets a
2486    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
2487    /// the client's route handle. The client also gets `route.closed` with the
2488    /// scope reason, one push per module and reason, so it can tell a revoked
2489    /// route from an ordinary close and not reopen it.
2490    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2491        if drained.is_empty() {
2492            return;
2493        }
2494        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2495            BTreeMap::new();
2496        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2497        for route in drained {
2498            warn!(
2499                module_id = %route.module_id,
2500                reason = ?route.reason,
2501                client_connection_id = route.client.connection_id.get(),
2502                route_channel = route.client.channel,
2503                "closing route because its scope changed"
2504            );
2505            pushes
2506                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2507                .or_insert_with(|| (route.reason, Vec::new()))
2508                .1
2509                .push(EndpointRoute {
2510                    goodbye_target: route.client.clone(),
2511                    principal: Principal::Unverified,
2512                    bound_at: Instant::now(),
2513                    draining: false,
2514                    drain_reason: None,
2515                });
2516            goodbyes.push(route.module);
2517            goodbyes.push(route.client);
2518        }
2519        for ((module_id, _), (reason, routes)) in pushes {
2520            send_route_control_pushes(
2521                &self.forwarding,
2522                routes,
2523                ClientControlPush::RouteClosed {
2524                    module_id,
2525                    channels: Vec::new(),
2526                    reason,
2527                    drained: false,
2528                    abandoned: 0,
2529                    excluded_subscriptions: 0,
2530                    terminal: Some(false),
2531                },
2532            );
2533        }
2534        self.emit_route_goodbyes(goodbyes);
2535    }
2536
2537    /// `scope.describe`: any registered module may read any scope, because a
2538    /// provider must read the scope a route it serves is stamped with.
2539    fn handle_scope_describe(
2540        &self,
2541        connection_id: ConnectionId,
2542        frame: Frame,
2543        owner: Principal,
2544        scope_ref: String,
2545    ) -> Result<Vec<Frame>, RouterError> {
2546        let registered = self
2547            .registry
2548            .get_module_by_connection(connection_id)
2549            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2550        if registered.is_none() {
2551            return Ok(vec![control_error_frame(
2552                &frame,
2553                "not_registered",
2554                "scope.describe requires an active module registration owned by this connection",
2555            )?]);
2556        }
2557        let description = self
2558            .scopes
2559            .read()
2560            .unwrap_or_else(|poisoned| poisoned.into_inner())
2561            .describe(&owner, &scope_ref);
2562        let owner_configured = match &owner {
2563            // Ask whether the owner is configured (`is_configured`), not
2564            // whether it is on the roster (`get(..).is_some()`): a supervised
2565            // module's process can register and describe a scope before the
2566            // supervisor has put it on the roster.
2567            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2568            _ => false,
2569        };
2570        let response = ModuleControlResponseToModule::ScopeDescribe {
2571            status: description.status,
2572            scope_epoch: description.scope_epoch,
2573            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2574            owner_synced: description.owner_synced,
2575            owner_configured,
2576            scope: description.stamp,
2577        };
2578        Ok(vec![control_response_body_frame(
2579            &frame,
2580            &response,
2581            "ModuleControlResponseToModule::ScopeDescribe",
2582        )?])
2583    }
2584
2585    fn handle_catalog_update(
2586        &self,
2587        connection_id: ConnectionId,
2588        frame: Frame,
2589        provides: Vec<ProviderRole>,
2590        capabilities: Option<CapabilityDeclarations>,
2591        ready: Option<bool>,
2592    ) -> Result<Vec<Frame>, RouterError> {
2593        self.refresh_capability_requirements();
2594        let Some(registration) = self
2595            .registry
2596            .get_module_by_connection(connection_id)
2597            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2598        else {
2599            return Ok(vec![control_error_frame(
2600                &frame,
2601                "not_registered",
2602                "catalog.update requires an active module registration owned by this connection",
2603            )?]);
2604        };
2605
2606        if let Some(message) =
2607            catalog_update_frozen_field_message(&registration.manifest, &provides)
2608        {
2609            return Ok(vec![control_error_frame(
2610                &frame,
2611                "catalog_update_frozen_field",
2612                message,
2613            )?]);
2614        }
2615
2616        let mut candidate = registration.manifest.clone();
2617        candidate.provides = provides.clone();
2618        candidate.capabilities = capabilities
2619            .clone()
2620            .or_else(|| registration.manifest.capabilities.clone());
2621        if let Err(err) = candidate.validate_capability_grammar() {
2622            return Ok(vec![control_error_frame(
2623                &frame,
2624                "invalid_capability_grammar",
2625                err.to_string(),
2626            )?]);
2627        }
2628
2629        // Updates must honor the same reserved owner as initial registration;
2630        // otherwise an empty HELLO could acquire the claim after admission.
2631        let mut conflicts = self
2632            .capability_evaluator
2633            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2634        if let Some(conflict) = conflicts.first() {
2635            let message = format!(
2636                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2637                conflict.capability, conflict.claimants[0], candidate.module_id
2638            );
2639            for conflict in &mut conflicts {
2640                conflict.source = DuplicateClaimSource::CatalogUpdate;
2641            }
2642            log_duplicate_claim_events(conflicts);
2643            return Ok(vec![control_error_frame(
2644                &frame,
2645                "reserved_capability",
2646                message,
2647            )?]);
2648        }
2649
2650        let updated = self
2651            .registry
2652            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2653            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2654        if updated.is_none() {
2655            return Ok(vec![control_error_frame(
2656                &frame,
2657                "not_registered",
2658                "catalog.update requires an active module registration owned by this connection",
2659            )?]);
2660        }
2661        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2662            log_duplicate_claim_events(
2663                self.capability_evaluator
2664                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
2665            );
2666        }
2667        if capability_census_trigger(
2668            registration.manifest.capabilities.as_ref(),
2669            updated
2670                .as_ref()
2671                .and_then(|entry| entry.manifest.capabilities.as_ref()),
2672        ) {
2673            self.enforce_capability_denies();
2674        }
2675        self.refresh_capability_requirements();
2676
2677        let response = ModuleControlResponseToModule::CatalogUpdate {};
2678        control_response_body_frame(
2679            &frame,
2680            &response,
2681            "ModuleControlResponseToModule::CatalogUpdate",
2682        )
2683        .map(|frame| vec![frame])
2684    }
2685
2686    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2687        self.refresh_capability_requirements();
2688        // A bare connection count is ambiguous between many clients holding a
2689        // route each and one client accumulating hundreds, so publish the
2690        // concentration alongside it. Route state is best-effort here: a
2691        // diagnostic endpoint must still answer if the forwarding lock is
2692        // contended.
2693        let mut counters = self.counters.snapshot();
2694        if let (Ok((connections_with_routes, max)), Some(obj)) = (
2695            self.forwarding.client_route_concentration(),
2696            counters.as_object_mut(),
2697        ) {
2698            obj.insert(
2699                "client_connections_with_routes".into(),
2700                connections_with_routes.into(),
2701            );
2702            obj.insert("max_routes_on_one_connection".into(), max.into());
2703        }
2704        // A module that is being fast-refused and a module that is fine look
2705        // identical from a client that retries and succeeds, so name the open
2706        // breakers here. This rides the existing free-form counters object
2707        // rather than a new wire field, so no sibling that deserializes
2708        // `ServerDescribe` has to be rebuilt to keep reading it.
2709        if let (Some(open_breakers), Some(obj)) = (
2710            self.route_bind_breakers.open_snapshot(),
2711            counters.as_object_mut(),
2712        ) {
2713            obj.insert("route_bind_breakers_open".into(), open_breakers);
2714        }
2715        let response = ClientControlResponse::ServerDescribe {
2716            protocol_ver: PROTOCOL_VERSION,
2717            subc_ops: subc_ops(),
2718            capabilities: self.subc_capabilities.as_ref().to_vec(),
2719            connected_clients: self.connected_clients.count(),
2720            counters: Some(counters),
2721            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2722            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2723            capability_requirements: self.capability_requirement_statuses(),
2724            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2725        };
2726        Ok(vec![control_response_body_frame(
2727            &frame,
2728            &response,
2729            "ClientControlResponse::ServerDescribe",
2730        )?])
2731    }
2732
2733    fn handle_catalog_list(
2734        &self,
2735        frame: Frame,
2736        module_id: Option<String>,
2737    ) -> Result<Vec<Frame>, RouterError> {
2738        let (generation, modules) = self.registry.list_modules().map_err(|err| {
2739            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2740        })?;
2741        let entries = modules
2742            .into_iter()
2743            .filter(|registration| {
2744                module_id
2745                    .as_deref()
2746                    .map(|wanted| registration.manifest.module_id == wanted)
2747                    .unwrap_or(true)
2748            })
2749            .map(|registration| {
2750                let not_ready = self.not_ready_reason(&registration);
2751                let roles = registration.manifest.provides;
2752                CatalogEntry {
2753                    module_id: registration.manifest.module_id,
2754                    ready: not_ready.is_none(),
2755                    not_ready,
2756                    module_version: Some(registration.manifest.module_version),
2757                    roles,
2758                    control_ops: registration.control_ops,
2759                    capabilities: registration.manifest.capabilities,
2760                    self_signals: registration.manifest.self_signals,
2761                }
2762            })
2763            .collect();
2764        let response = ClientControlResponse::CatalogList {
2765            generation,
2766            modules: entries,
2767            subc_ops: subc_ops(),
2768        };
2769        Ok(vec![control_response_body_frame(
2770            &frame,
2771            &response,
2772            "ClientControlResponse::CatalogList",
2773        )?])
2774    }
2775
2776    fn route_open_principal(
2777        &self,
2778        frame: &Frame,
2779        consumer_identity: Option<ConsumerIdentity>,
2780    ) -> Result<Result<Principal, Frame>, RouterError> {
2781        let Some(consumer_identity) = consumer_identity else {
2782            return Ok(Ok(Principal::Direct));
2783        };
2784
2785        if self.supervisor.spawned_consumer_authorized(
2786            &consumer_identity.module_id,
2787            &consumer_identity.launch_nonce,
2788        ) {
2789            return Ok(Ok(Principal::Reserved {
2790                module_id: consumer_identity.module_id,
2791            }));
2792        }
2793
2794        Ok(Err(control_error_frame(
2795            frame,
2796            "bad_consumer_identity",
2797            format!(
2798                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2799                consumer_identity.module_id
2800            ),
2801        )?))
2802    }
2803
2804    /// Ordinary `route.open` refusals go through here; admission and breaker
2805    /// refusals log separately with their capacity or breaker state. The daemon can
2806    /// attest which code it sent: without the event, a client's "the daemon
2807    /// refused me" and the daemon's own view could only be reconciled by
2808    /// argument. Malformed input (`invalid_project_root`) does not come here;
2809    /// rejecting a request that was never a valid open is not a refusal of one.
2810    fn route_open_refusal_frame(
2811        &self,
2812        ctx: &RouteCtx,
2813        frame: &Frame,
2814        module_id: &str,
2815        reason: &'static str,
2816        code: &'static str,
2817        message: impl Into<String>,
2818    ) -> Result<Frame, RouterError> {
2819        self.observe_route_open_refusal(ctx, module_id, reason, code);
2820        control_error_frame(frame, code, message.into())
2821    }
2822
2823    /// Refuse a `route.open` because the target module's bind-relay breaker is
2824    /// open, without attempting the relay.
2825    ///
2826    /// The wire code is `module_timeout`, which is the truth (the module has
2827    /// not been answering binds) and which both SDKs already classify as
2828    /// retryable with capped backoff. Reusing it is what keeps this change out
2829    /// of both SDKs; the daemon-side distinction lives in the counter key
2830    /// instead.
2831    ///
2832    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
2833    /// While a breaker is open this fires on every open to that module, and the
2834    /// stall written up in `docs/designs/route-open-head-of-line.md` already
2835    /// produced 261 lines about a single module inside 3000 lines of daemon
2836    /// log. The rare transitions are logged at warn/info instead and the volume
2837    /// is carried by the counter, so the evidence survives without the flood.
2838    /// The debug line keeps a per-refusal record reachable for whoever turns
2839    /// the level up.
2840    fn route_open_breaker_refusal_frame(
2841        &self,
2842        ctx: &RouteCtx,
2843        frame: &Frame,
2844        module_id: &str,
2845        consecutive_timeouts: u32,
2846        retry_in: Duration,
2847        probe_in_flight: bool,
2848    ) -> Result<Frame, RouterError> {
2849        self.counters
2850            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2851        debug!(
2852            target: "control",
2853            code = "module_timeout",
2854            module_id = ?module_id,
2855            connection_id = ctx.connection_id.get(),
2856            consecutive_timeouts,
2857            retry_in_ms = retry_in.as_millis() as u64,
2858            probe_in_flight,
2859            "route.open refused by open bind-relay breaker"
2860        );
2861        // Say what a caller can act on. An open bind-relay breaker means the
2862        // module timed out accepting several new routes in a row. The module
2863        // is still running and its established routes keep working; only new
2864        // route.open requests are refused until the cooldown ends and one
2865        // test route (the probe) gets through. A message that only counts
2866        // failed relays reads as "the module is down" to a worker that sees it.
2867        let detail = if probe_in_flight {
2868            "one test route is already being tried; retry once it settles".to_string()
2869        } else {
2870            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
2871        };
2872        control_error_frame(
2873            frame,
2874            "module_timeout",
2875            format!(
2876                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
2877                 timed out in a row); {detail}; its established routes are unaffected"
2878            ),
2879        )
2880    }
2881
2882    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
2883    /// requester's bytes (an unknown target is whatever the client sent) and
2884    /// is Debug-formatted so control characters land in the log escaped
2885    /// rather than as terminal sequences for whoever tails it.
2886    ///
2887    /// `reason` names the check that refused, because one wire code has
2888    /// several senders: after a module registers, `target_unavailable` can
2889    /// come from a missing role, an inactive registration, a supervisor that
2890    /// has not marked the process live, a missing forwarding connection, or a
2891    /// failed relay, and a log that records only the code cannot say which of
2892    /// them fired. It is a static, daemon-chosen label per branch, so it is
2893    /// safe to print plainly and stays a closed set.
2894    fn observe_route_open_refusal(
2895        &self,
2896        ctx: &RouteCtx,
2897        module_id: &str,
2898        reason: &'static str,
2899        code: &'static str,
2900    ) {
2901        self.counters.increment_route_open_refused(code);
2902        info!(
2903            target: "control",
2904            code,
2905            reason,
2906            module_id = ?module_id,
2907            connection_id = ctx.connection_id.get(),
2908            "route.open refused"
2909        );
2910        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2911            self.route_outages.record_not_serving(module_id, reason);
2912        }
2913    }
2914
2915    /// Record an ACCEPTED route.open.
2916    ///
2917    /// Refusals have been logged and counted since the attestation work; accepts
2918    /// were invisible, so the daemon knew every principal it stamped and wrote
2919    /// none of them down. The party that attests the identity was the only party
2920    /// not recording it, which left a credential vault unable to name the sender
2921    /// of a call that reached it (claustrum #43) and left the launch-nonce
2922    /// concurrency question unanswerable from the outside.
2923    ///
2924    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
2925    /// `code`/`module_id`/`connection_id` returns both directions of the same
2926    /// decision rather than two shapes a reader has to join by hand.
2927    ///
2928    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
2929    /// and the difference carries information rather than being an
2930    /// inconsistency. This line is only reachable after a successful bind to a
2931    /// REGISTERED module, so the value has already passed HELLO validation
2932    /// including the path-hazard refusal and cannot contain control bytes. A
2933    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
2934    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
2935    ///
2936    /// Bare is also what every other daemon line already emits (`module
2937    /// registered`, `configured module supervised`). Shipping `?module_id` here
2938    /// made this instrument the only one in the file whose ids did not answer
2939    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
2940    /// line whose whole purpose is being grepped beside its sibling.
2941    ///
2942    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
2943    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
2944    /// forwards every field type through it and a bare `&str` and a `?`-escaped
2945    /// one are recorded identically. A test written against that harness passes
2946    /// either way -- I wrote one, measured it, and deleted it rather than ship a
2947    /// green assertion that cannot fail. The same limit applies to the escaping
2948    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
2949    /// it reads as a guard on the Debug escaping and cannot detect its removal.
2950    /// Fencing either needs the real formatter, not the capture layer.
2951    ///
2952    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
2953    /// unix-socket options and subc is loopback TCP, so there is no peer identity
2954    /// to record. The ephemeral port would decay within minutes and answer only a
2955    /// live question. The identity question is instead answered by counting
2956    /// distinct live connections presenting one module's `consumer_identity` --
2957    /// "is anyone else holding this secret" rather than "is this the right
2958    /// process".
2959    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2960        self.route_outages.record_accepted(module_id);
2961        self.counters.increment_route_open_accepted(principal);
2962        info!(
2963            target: "control",
2964            principal,
2965            module_id,
2966            connection_id = ctx.connection_id.get(),
2967            "route.open accepted"
2968        );
2969    }
2970
2971    fn supervised_absent_route_open_refusal_frame(
2972        &self,
2973        ctx: &RouteCtx,
2974        frame: &Frame,
2975        module_id: &str,
2976        code: &'static str,
2977        status: &crate::supervise::ModuleStatus,
2978    ) -> Result<Frame, RouterError> {
2979        self.counters.increment_route_open_refused(code);
2980        info!(
2981            target: "control",
2982            code,
2983            reason = "supervised_not_registered",
2984            module_id = ?module_id,
2985            connection_id = ctx.connection_id.get(),
2986            state = %status.state,
2987            enabled = status.enabled,
2988            live = status.live,
2989            "route.open refused"
2990        );
2991        // A supervised module whose process has not registered is not
2992        // serving, whatever the reason; the supervisor knows this id, so it is
2993        // safe to track.
2994        self.route_outages
2995            .record_not_serving(module_id, "supervised_not_registered");
2996        control_error_frame(
2997            frame,
2998            code,
2999            format!(
3000                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3001                status.state, status.enabled, status.live
3002            ),
3003        )
3004    }
3005
3006    async fn handle_route_open(
3007        &self,
3008        ctx: &RouteCtx,
3009        frame: Frame,
3010        request: RouteOpenRequest,
3011    ) -> Result<Vec<Frame>, RouterError> {
3012        let RouteOpenRequest {
3013            target,
3014            mut identity,
3015            consumer_identity,
3016            consumer_capabilities,
3017            role_versions,
3018            admission_facts,
3019            scope,
3020        } = request;
3021        let target_module_id = target_module_id(&target).to_string();
3022        if self
3023            .registry
3024            .get_module_by_connection(ctx.connection_id)
3025            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3026            .is_some()
3027        {
3028            return Ok(vec![control_error_frame(
3029                &frame,
3030                "invalid_request",
3031                "module connections cannot open client routes",
3032            )?]);
3033        }
3034        debug!(
3035            connection_id = ctx.connection_id.get(),
3036            corr = frame.header.corr,
3037            module_id = %target_module_id,
3038            "handling route.open"
3039        );
3040
3041        // A malformed declaration is refused first, before anything about the
3042        // target is looked up: the same body would be refused against any
3043        // module, so the caller learns nothing by retrying or waiting. An empty
3044        // map declares nothing and travels as no field at all, so a provider
3045        // only ever sees a missing field or a non-empty one.
3046        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3047        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3048            self.observe_route_open_refusal(
3049                ctx,
3050                &target_module_id,
3051                "invalid_role_versions",
3052                error_codes::INVALID_REQUEST,
3053            );
3054            return Ok(vec![control_error_body_frame(
3055                &frame,
3056                ErrorBody {
3057                    code: error_codes::INVALID_REQUEST.to_string(),
3058                    message: error.to_string(),
3059                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3060                },
3061            )?]);
3062        }
3063
3064        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
3065        // opposite. Below, a caller learns whether a module is unregistered,
3066        // supervised-but-down (with state/enabled/live), or registered without the
3067        // requested role. Elsewhere that is an enumeration leak: a probe learning
3068        // the shape of a fleet it cannot otherwise see.
3069        //
3070        // It is not one here, and the reason is the ACCESS MODEL rather than
3071        // anything about these errors. Reaching route.open requires the
3072        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
3073        // connection file, so any caller who completes it already runs as this
3074        // user -- and can read subc.jsonc for the module list and `ck module
3075        // status` for live state. The reply discloses nothing the caller cannot
3076        // read more easily from disk, while the precision is load-bearing:
3077        // `unknown_module` is retryable and a missing role is not.
3078        //
3079        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
3080        // remote transport, a sandboxed caller, a shared-host mode -- THAT
3081        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
3082        let Some(registration) = self
3083            .registry
3084            .get_module(&target_module_id)
3085            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3086        else {
3087            if let Some((status, warming)) =
3088                self.supervisor_status(&target_module_id, frame.header.corr)?
3089            {
3090                // BEFORE the two availability codes below, because for a module
3091                // that speaks no subc wire both of them are false comfort: they
3092                // say "not right now" and are retried, and this module will
3093                // never register no matter how long the caller waits. The
3094                // absence here is the declaration being honoured, not a module
3095                // that is late.
3096                if status.protocol == ModuleProtocol::None {
3097                    return Ok(vec![self.route_open_refusal_frame(
3098                        ctx,
3099                        &frame,
3100                        &target_module_id,
3101                        "protocol_none",
3102                        error_codes::MODULE_NO_PROTOCOL,
3103                        format!(
3104                            "module_id '{target_module_id}' is declared protocol: none; \
3105                             it speaks no subc wire and serves no routes"
3106                        ),
3107                    )?]);
3108                }
3109                let code = if warming {
3110                    "module_warming"
3111                } else {
3112                    "target_unavailable"
3113                };
3114                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3115                    ctx,
3116                    &frame,
3117                    &target_module_id,
3118                    code,
3119                    &status,
3120                )?]);
3121            }
3122            if let Some(removed_ago_ms) =
3123                self.supervisor.removal_tombstone_age_ms(&target_module_id)
3124            {
3125                return Ok(vec![self.route_open_refusal_frame(
3126                    ctx,
3127                    &frame,
3128                    &target_module_id,
3129                    "removed",
3130                    error_codes::MODULE_REMOVED,
3131                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3132                )?]);
3133            }
3134            return Ok(vec![self.route_open_refusal_frame(
3135                ctx,
3136                &frame,
3137                &target_module_id,
3138                "not_registered",
3139                error_codes::UNKNOWN_MODULE,
3140                format!("module_id '{target_module_id}' is not registered"),
3141            )?]);
3142        };
3143
3144        // Best-effort only: registry readiness and forwarding reservation use
3145        // different locks, so a module can flip readiness between this read and
3146        // the relay. Modules must still tolerate an `on_bind` while not ready.
3147        if !registration.ready {
3148            self.counters
3149                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3150            info!(
3151                target: "control",
3152                code = error_codes::MODULE_WARMING,
3153                module_id = ?target_module_id,
3154                connection_id = ctx.connection_id.get(),
3155                reason = "declared_not_ready",
3156                "route.open refused"
3157            );
3158            // The module is registered but says it cannot take work, which is
3159            // an outage from the caller's side even though its process is up.
3160            self.route_outages
3161                .record_not_serving(&target_module_id, "declared_not_ready");
3162            return Ok(vec![control_error_body_frame(
3163                &frame,
3164                ErrorBody {
3165                    code: error_codes::MODULE_WARMING.to_string(),
3166                    message: format!(
3167                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3168                    ),
3169                    detail: Some(serde_json::json!({
3170                        "reason": "declared_not_ready"
3171                    })),
3172                },
3173            )?]);
3174        }
3175
3176        // Effective readiness, second half: a module that declares a capability
3177        // `need: required` is not routable while that capability has no
3178        // registered provider. It is enforced HERE, as a retryable routing
3179        // refusal, and deliberately not as spawn ordering or a boot block. The
3180        // module is still started and registered and can make its own calls;
3181        // spawn ordering is a promise that cannot be kept once a provider
3182        // crashes at runtime, and refusing to boot would stop the whole
3183        // machine, including the tools needed to fix its configuration.
3184        //
3185        // "Provided" is the evaluator's verdict, which counts a provider as
3186        // soon as it has REGISTERED, not once it is ready. Two modules that
3187        // require each other's capabilities are therefore both routable once
3188        // both register; counting readiness instead would deadlock them.
3189        //
3190        // Only new opens are refused. Routes already bound when a provider
3191        // goes away stay bound: nothing here tears them down, and the module
3192        // answers them as it can. Like the readiness read above this is
3193        // best-effort against a provider registering or leaving concurrently.
3194        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
3195            self.counters
3196                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3197            info!(
3198                target: "control",
3199                code = error_codes::MODULE_WARMING,
3200                module_id = ?target_module_id,
3201                connection_id = ctx.connection_id.get(),
3202                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3203                capability = %capability,
3204                "route.open refused"
3205            );
3206            return Ok(vec![control_error_body_frame(
3207                &frame,
3208                ErrorBody {
3209                    code: error_codes::MODULE_WARMING.to_string(),
3210                    message: format!(
3211                        "module_id '{target_module_id}' requires capability '{capability}', \
3212                         which no registered module provides; retry"
3213                    ),
3214                    detail: Some(serde_json::json!({
3215                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3216                        "capability": capability,
3217                    })),
3218                },
3219            )?]);
3220        }
3221
3222        if !target_has_required_role(&target, &registration.manifest.provides) {
3223            return Ok(vec![self.route_open_refusal_frame(
3224                ctx,
3225                &frame,
3226                &target_module_id,
3227                "role_not_provided",
3228                "target_unavailable",
3229                format!("module_id '{target_module_id}' does not provide the requested target"),
3230            )?]);
3231        }
3232
3233        if registration.state != ChannelState::Active {
3234            return Ok(vec![self.route_open_refusal_frame(
3235                ctx,
3236                &frame,
3237                &target_module_id,
3238                "registration_not_active",
3239                "target_unavailable",
3240                format!("module_id '{target_module_id}' is not active"),
3241            )?]);
3242        }
3243
3244        if self
3245            .forwarding
3246            .module_is_draining(&target_module_id)
3247            .map_err(RouterError::Forwarding)?
3248        {
3249            return Ok(vec![self.route_open_refusal_frame(
3250                ctx,
3251                &frame,
3252                &target_module_id,
3253                "reloading",
3254                "module_reloading",
3255                format!("module_id '{target_module_id}' is reloading"),
3256            )?]);
3257        }
3258
3259        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3260            process_liveness.process_live(&target_module_id) == Some(false)
3261        }) {
3262            // A module the supervisor is restarting or reloading can still hold
3263            // a registration: the old process before its connection closes, or
3264            // a new one that registered while the supervisor was draining. The
3265            // forwarding table does not see that as draining, but the consumer
3266            // should still be told to retry soon, exactly as for the drain
3267            // above, rather than that the target is unavailable.
3268            if process_liveness.process_replacing(&target_module_id) {
3269                return Ok(vec![self.route_open_refusal_frame(
3270                    ctx,
3271                    &frame,
3272                    &target_module_id,
3273                    "reloading",
3274                    "module_reloading",
3275                    format!("module_id '{target_module_id}' is reloading"),
3276                )?]);
3277            }
3278            return Ok(vec![self.route_open_refusal_frame(
3279                ctx,
3280                &frame,
3281                &target_module_id,
3282                "supervisor_not_live",
3283                "target_unavailable",
3284                format!("module_id '{target_module_id}' is not live"),
3285            )?]);
3286        }
3287
3288        if !self
3289            .forwarding
3290            .has_live_module_connection(&target_module_id)
3291            .map_err(RouterError::Forwarding)?
3292        {
3293            return Ok(vec![self.route_open_refusal_frame(
3294                ctx,
3295                &frame,
3296                &target_module_id,
3297                "no_forwarding_connection",
3298                "target_unavailable",
3299                format!("module_id '{target_module_id}' has no live forwarding connection"),
3300            )?]);
3301        }
3302
3303        if let Some(error) =
3304            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3305        {
3306            self.observe_route_open_refusal(
3307                ctx,
3308                &target_module_id,
3309                "op_not_allowed",
3310                "op_not_allowed",
3311            );
3312            return Ok(vec![error]);
3313        }
3314
3315        let principal = match self.route_open_principal(&frame, consumer_identity)? {
3316            Ok(principal) => principal,
3317            Err(error) => {
3318                self.observe_route_open_refusal(
3319                    ctx,
3320                    &target_module_id,
3321                    "bad_consumer_identity",
3322                    "bad_consumer_identity",
3323                );
3324                return Ok(vec![error]);
3325            }
3326        };
3327
3328        // This is attested, control-plane policy for supervised module origins.
3329        // Keep it before route reservation and out of the opaque forwarding hot
3330        // path: data frames must never acquire a per-frame capability check.
3331        if let Principal::Reserved {
3332            module_id: opening_module_id,
3333        } = &principal
3334        {
3335            if let Some(opening_registration) = self
3336                .registry
3337                .get_module(opening_module_id)
3338                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3339            {
3340                if let Some(capability) =
3341                    denied_capability(&opening_registration.manifest, &registration.manifest)
3342                {
3343                    warn!(
3344                        opening_module_id,
3345                        target_module_id,
3346                        capability,
3347                        "refusing route.open because an attested capability deny edge matches"
3348                    );
3349                    return Ok(vec![self.route_open_refusal_frame(
3350                        ctx,
3351                        &frame,
3352                        &target_module_id,
3353                        "capability_deny_edge",
3354                        "capability_forbidden",
3355                        format!(
3356                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3357                        ),
3358                    )?]);
3359                }
3360            }
3361        }
3362
3363        if admission_facts.is_some() {
3364            let carrier_matches = matches!(
3365                &principal,
3366                Principal::Reserved { module_id }
3367                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3368            );
3369            if !carrier_matches {
3370                return Ok(vec![self.route_open_refusal_frame(
3371                    ctx,
3372                    &frame,
3373                    &target_module_id,
3374                    "admission_facts_carrier_not_permitted",
3375                    "admission_facts_not_permitted",
3376                    "admission facts may only be carried by the configured reserved module",
3377                )?]);
3378            }
3379
3380            let target_allowed = self
3381                .admission_facts_targets
3382                .as_ref()
3383                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3384            if !target_allowed {
3385                return Ok(vec![self.route_open_refusal_frame(
3386                    ctx,
3387                    &frame,
3388                    &target_module_id,
3389                    "admission_facts_target_not_listed",
3390                    "admission_facts_target_not_allowed",
3391                    format!(
3392                        "admission facts are not permitted for target module_id '{target_module_id}'"
3393                    ),
3394                )?]);
3395            }
3396
3397            // Keep the value opaque to subc. The downstream admission validator owns
3398            // schema and semantic checks; this daemon only enforces carrier authority
3399            // and the configured destination allowlist.
3400        }
3401
3402        // Scope admission, on the attested principal above and never on the
3403        // request body. The tag read here travels with the pending bind and is
3404        // compared with the published one at commit, so a sync between here
3405        // and the module's ack refuses the open instead of binding a stamp
3406        // that is no longer true.
3407        let (bound_scope, scope_stamp) = match scope {
3408            None => (None, None),
3409            Some(selector) => {
3410                let owner_configured = match &selector.owner {
3411                    // A reserved owner counts as configured from before its
3412                    // process is spawned (see `SupervisorHandle::is_configured`).
3413                    // So an owner that has not synced its scopes yet is refused
3414                    // as retryable (`scope_not_synced`), not as one that will
3415                    // never sync.
3416                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3417                    _ => false,
3418                };
3419                let admitted = self
3420                    .scopes
3421                    .read()
3422                    .unwrap_or_else(|poisoned| poisoned.into_inner())
3423                    .admit(&principal, &target_module_id, &selector, owner_configured)
3424                    .and_then(|admission| {
3425                        crate::scopes::check_target_flow_support(
3426                            &admission.stamp,
3427                            &target_module_id,
3428                            registration.manifest.capabilities.as_ref(),
3429                        )?;
3430                        Ok(admission)
3431                    });
3432                match admitted {
3433                    Ok(admission) => (
3434                        Some(BoundScope {
3435                            owner: admission.owner,
3436                            scope_ref: admission.stamp.scope_ref.clone(),
3437                            tag: admission.tag,
3438                        }),
3439                        Some(admission.stamp),
3440                    ),
3441                    Err(refusal) => {
3442                        return Ok(vec![self.route_open_refusal_frame(
3443                            ctx,
3444                            &frame,
3445                            &target_module_id,
3446                            refusal.code,
3447                            refusal.code,
3448                            refusal.message,
3449                        )?]);
3450                    }
3451                }
3452            }
3453        };
3454
3455        // Bind admits a root that no longer exists on disk, because refusing here
3456        // closes the only exit from a paused run: cancel needs a bound route, and a
3457        // renamed or reclaimed directory makes that route unopenable forever. The
3458        // run itself is intact and still addressable by its recorded identity.
3459        //
3460        // This does NOT relax the rule the strict constructor protects. That rule is
3461        // that no root is ever aliased into NEW durable state -- a missing component
3462        // can reappear as a symlink elsewhere, which would move the identity and
3463        // split a session's history across two of them. The engine now refuses the
3464        // two operations that create such state (send and import) at admission,
3465        // which is a narrower way to hold the same invariant: reads and terminations
3466        // are admitted, writes are not. That refusal had to ship before this line
3467        // changed, or there is an interval where a send commits under a provisional
3468        // identity -- the exact failure the original policy existed to prevent.
3469        //
3470        // Resolution follows realpath rather than lexical cleanup: the longest
3471        // existing ancestor is canonicalized and the missing tail re-appended, so a
3472        // live root is unchanged and a vanished leaf keeps the identity it was
3473        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
3474        // same caller the moment the directory vanished, which strands the run more
3475        // quietly than refusing it.
3476        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3477            Ok(project_root) => project_root,
3478            Err(err) => {
3479                return Ok(vec![control_error_frame(
3480                    &frame,
3481                    "invalid_project_root",
3482                    err.to_string(),
3483                )?])
3484            }
3485        };
3486        identity.project_root = project_root.as_path().to_path_buf();
3487
3488        // Last gate before any relay work, and deliberately after the cheap
3489        // registry and availability checks above: those name a more precise
3490        // condition (unknown, removed, reloading) and a caller is better served
3491        // by the precise code than by this one.
3492        //
3493        // Everything below this point costs an egress permit, a reserved handle
3494        // pair and, if the module does not answer, the whole relay budget. The
3495        // reader no longer waits for that budget, so cap each target explicitly;
3496        // serial dispatch used to provide the accidental cap of one relay per
3497        // connection. Admission is a mutex-protected count and never waits.
3498        let _concurrency_guard = match self
3499            .route_bind_concurrency
3500            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3501        {
3502            Ok(guard) => guard,
3503            Err(in_flight) => {
3504                return Ok(vec![self.route_open_target_capacity_refusal(
3505                    ctx,
3506                    &frame,
3507                    &target_module_id,
3508                    in_flight,
3509                )?]);
3510            }
3511        };
3512
3513        // A module that has already burned the whole budget `threshold` times
3514        // in a row does not get to charge it again until a probe says it recovered.
3515        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3516            RouteBindAdmission::Admitted { guard, probe } => {
3517                if probe {
3518                    info!(
3519                        module_id = %target_module_id,
3520                        connection_id = ctx.connection_id.get(),
3521                        "route.bind breaker half-open: admitting one probe"
3522                    );
3523                }
3524                guard
3525            }
3526            RouteBindAdmission::Refused {
3527                consecutive_timeouts,
3528                retry_in,
3529                probe_in_flight,
3530            } => {
3531                return Ok(vec![self.route_open_breaker_refusal_frame(
3532                    ctx,
3533                    &frame,
3534                    &target_module_id,
3535                    consecutive_timeouts,
3536                    retry_in,
3537                    probe_in_flight,
3538                )?]);
3539            }
3540        };
3541
3542        // Resolve the per-module budget here so the wait matches the operator's
3543        // intent for this specific target. A per-module override in
3544        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
3545        // daemons) wins over the daemon-wide default.
3546        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3547        let relay_deadline = Instant::now() + route_bind_relay_timeout;
3548        let pending = match self
3549            .forwarding
3550            .begin_route_bind_relay_for(
3551                ctx.connection_id,
3552                ctx.egress.clone(),
3553                response_version(&frame),
3554                frame.header.corr,
3555                &target_module_id,
3556                principal.clone(),
3557                bound_scope,
3558                Some(project_root),
3559                relay_deadline,
3560            )
3561            .await
3562        {
3563            Ok(pending) => pending,
3564            Err(err) => {
3565                return Ok(vec![self.route_open_refusal_frame(
3566                    ctx,
3567                    &frame,
3568                    &target_module_id,
3569                    "relay_reservation_failed",
3570                    forwarding_error_code(&err),
3571                    err.to_string(),
3572                )?])
3573            }
3574        };
3575        let crate::forwarding::PendingRouteBindRelay {
3576            endpoint,
3577            module_sink,
3578            negotiated_ver,
3579            client_channel,
3580            client_epoch,
3581            module_channel,
3582            module_epoch,
3583            corr: relay_corr,
3584            receiver,
3585        } = pending;
3586        let mut reservation =
3587            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3588
3589        // Reserving egress can wait while a module reconnects or a swap cuts
3590        // over. Check the connection the relay actually captured, not the
3591        // earlier by-id lookup: a flow-aware module must not vouch for a
3592        // replacement. The captured sink cannot turn into another connection.
3593        if let Some(stamp) = scope_stamp
3594            .as_ref()
3595            .filter(|stamp| stamp.attributes.flow_id.is_some())
3596        {
3597            let relay_registration = self
3598                .registry
3599                .get_module_by_connection(endpoint.connection_id)
3600                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3601            if let Err(refusal) = crate::scopes::check_target_flow_support(
3602                stamp,
3603                &target_module_id,
3604                relay_registration
3605                    .as_ref()
3606                    .and_then(|registration| registration.manifest.capabilities.as_ref()),
3607            ) {
3608                reservation.release_and_disarm();
3609                return Ok(vec![self.route_open_refusal_frame(
3610                    ctx,
3611                    &frame,
3612                    &target_module_id,
3613                    refusal.code,
3614                    refusal.code,
3615                    refusal.message,
3616                )?]);
3617            }
3618        }
3619
3620        debug!(
3621            connection_id = ctx.connection_id.get(),
3622            client_channel,
3623            client_epoch,
3624            module_channel,
3625            module_epoch,
3626            "reserved route handle pair"
3627        );
3628        // Rendered BEFORE the move into the relay, because the accept arm below
3629        // is where it is logged and the principal is gone by then.
3630        let principal_label = match &principal {
3631            Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3632            Principal::Direct => "direct".to_string(),
3633            other => format!("{other:?}"),
3634        };
3635        let relay = ModuleControlRequest::RouteBind {
3636            route_channel: module_channel,
3637            epoch: module_epoch,
3638            target,
3639            identity,
3640            principal: Some(principal),
3641            consumer_capabilities,
3642            role_versions,
3643            admission_facts,
3644            scope: scope_stamp,
3645        };
3646        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3647            RouterError::backend(
3648                0,
3649                frame.header.corr,
3650                format!("failed to encode route.bind request: {err}"),
3651            )
3652        })?;
3653        let relay_frame = Frame::build_with_version(
3654            negotiated_ver,
3655            FrameType::Request,
3656            control_flags(),
3657            0,
3658            0,
3659            relay_corr,
3660            relay_body,
3661        )
3662        .map_err(RouterError::FrameBuild)?;
3663
3664        if let Err(err) = module_sink.send(relay_frame).await {
3665            reservation.release_and_disarm();
3666            return Ok(vec![self.route_open_refusal_frame(
3667                ctx,
3668                &frame,
3669                &target_module_id,
3670                "relay_send_failed",
3671                "target_unavailable",
3672                err.to_string(),
3673            )?]);
3674        }
3675
3676        #[cfg(test)]
3677        if let Some(hold) = &self.route_bind_relay_sent_hold {
3678            // First wait: tell the test the bind is queued. Second wait: stay
3679            // here until the test has driven whatever it wants to happen
3680            // between queueing the bind and marking it enqueued.
3681            hold.wait().await;
3682            hold.wait().await;
3683        }
3684
3685        // A missing relay is not by itself an abandoned bind: the module may
3686        // already have acked it and the route may be committed. Only a relay
3687        // dropped without any module reply needs this GOODBYE; one a reply
3688        // settled has already had its GOODBYE decided by the reply path.
3689        if self
3690            .forwarding
3691            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3692            .map_err(RouterError::Forwarding)?
3693            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
3694        {
3695            self.send_abandoned_route_bind_goodbye(
3696                &module_sink,
3697                negotiated_ver,
3698                module_channel,
3699                module_epoch,
3700            );
3701        }
3702
3703        match timeout_at(relay_deadline, receiver).await {
3704            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3705                reservation.disarm();
3706                if breaker.record_accepted() {
3707                    info!(
3708                        module_id = %target_module_id,
3709                        "route.bind breaker closed: the probe was accepted"
3710                    );
3711                }
3712                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3713                Ok(Vec::new())
3714            }
3715            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3716                reservation.release_and_disarm();
3717                // A module that says no in microseconds is healthy. Rejection
3718                // is a different condition with its own refusal and must not
3719                // move the breaker.
3720                breaker.record_inconclusive();
3721                // The daemon's own commit re-check refused the bind because the
3722                // scope ended or changed after admission. The module accepted;
3723                // counting it as a module rejection would blame the module.
3724                let scope_code = match body.code.as_str() {
3725                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3726                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3727                    _ => None,
3728                };
3729                if let Some(code) = scope_code {
3730                    self.observe_route_open_refusal(
3731                        ctx,
3732                        &target_module_id,
3733                        "scope_changed_before_commit",
3734                        code,
3735                    );
3736                    return Ok(vec![control_error_body_frame(&frame, body)?]);
3737                }
3738                self.counters
3739                    .increment_route_open_refused("module_rejected");
3740                info!(
3741                    target: "control",
3742                    code = "module_rejected",
3743                    module_code = ?body.code,
3744                    module_id = ?target_module_id,
3745                    connection_id = ctx.connection_id.get(),
3746                    "route.open refused"
3747                );
3748                Ok(vec![control_error_body_frame(&frame, body)?])
3749            }
3750            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3751                reservation.release_and_disarm();
3752                breaker.record_inconclusive();
3753                // Fires when the module's connection closes while a relayed
3754                // bind is pending -- typically a caller racing a module restart
3755                // whose bind was relayed BEFORE the drain mark went up. Logged
3756                // because the caller sees only its own error and the fleet has
3757                // already spent one diagnosis round unable to tell this arm
3758                // from a relay timeout without daemon-side evidence.
3759                tracing::warn!(
3760                    module_id = %target_module_id,
3761                    "route.bind relay abandoned: {message}"
3762                );
3763                Ok(vec![self.route_open_refusal_frame(
3764                    ctx,
3765                    &frame,
3766                    &target_module_id,
3767                    "relay_abandoned",
3768                    "target_unavailable",
3769                    message,
3770                )?])
3771            }
3772            Ok(Err(_)) => {
3773                reservation.release_and_disarm();
3774                breaker.record_inconclusive();
3775                Ok(vec![self.route_open_refusal_frame(
3776                    ctx,
3777                    &frame,
3778                    &target_module_id,
3779                    "relay_waiter_canceled",
3780                    "target_unavailable",
3781                    "route.bind relay waiter was canceled before the module responded",
3782                )?])
3783            }
3784            Err(_) => {
3785                reservation.release_and_disarm();
3786                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
3787                // answer at all is the one condition a fast refusal can
3788                // usefully stand in for; every other arm already answered.
3789                if let Some(opened) = breaker.record_timeout(
3790                    self.route_bind_breaker_threshold,
3791                    self.route_bind_breaker_cooldown,
3792                ) {
3793                    warn!(
3794                        module_id = %target_module_id,
3795                        consecutive_timeouts = opened.consecutive_timeouts,
3796                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3797                        reopened_after_probe = opened.reopened_after_probe,
3798                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3799                    );
3800                }
3801                // The generous budget just burned to no answer: the module is
3802                // registered and its connection is up, but its bind handler sat
3803                // on the ack for the full budget (warm-on-bind, cold configure,
3804                // or a wedged handler). Every earlier unavailability shape
3805                // fast-refuses BEFORE the relay, so this arm firing means the
3806                // slowness is module-side -- log it so the per-module timeline
3807                // is reconstructable without client audit rows.
3808                tracing::warn!(
3809                    module_id = %target_module_id,
3810                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3811                    "route.bind relay timed out: module did not ack within budget"
3812                );
3813                Ok(vec![self.route_open_refusal_frame(
3814                    ctx,
3815                    &frame,
3816                    &target_module_id,
3817                    "relay_timed_out",
3818                    "module_timeout",
3819                    format!(
3820                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
3821                        route_bind_relay_timeout
3822                    ),
3823                )?])
3824            }
3825        }
3826    }
3827
3828    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3829        let response = ClientControlResponse::SupervisorSpawnSnapshot {
3830            snapshot: self.supervisor.spawn_snapshot(),
3831        };
3832        Ok(vec![control_response_body_frame(
3833            &frame,
3834            &response,
3835            "ClientControlResponse::SupervisorSpawnSnapshot",
3836        )?])
3837    }
3838
3839    fn handle_supervisor_spawn_subscribe(
3840        &self,
3841        ctx: &RouteCtx,
3842        frame: Frame,
3843        since: Option<SpawnCursor>,
3844    ) -> Result<Vec<Frame>, RouterError> {
3845        match self.supervisor.subscribe_spawns(
3846            ctx.connection_id,
3847            frame.header.corr,
3848            response_version(&frame),
3849            since,
3850            ctx.egress.clone(),
3851        ) {
3852            Ok(()) => Ok(Vec::new()),
3853            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3854                Ok(vec![control_error_body_frame(
3855                    &frame,
3856                    ErrorBody {
3857                        code: "spawn_cursor_incarnation_mismatch".to_string(),
3858                        message: "spawn cursor belongs to a different daemon incarnation"
3859                            .to_string(),
3860                        detail: Some(serde_json::json!({
3861                            "current_daemon_incarnation": current
3862                        })),
3863                    },
3864                )?])
3865            }
3866            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3867                &frame,
3868                ErrorBody {
3869                    code: "spawn_cursor_too_old".to_string(),
3870                    message: "spawn cursor predates the retained event ring".to_string(),
3871                    detail: Some(serde_json::json!({
3872                        "oldest_retained_cursor": oldest
3873                    })),
3874                },
3875            )?]),
3876            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3877                0,
3878                frame.header.corr,
3879                format!("failed to open supervisor spawn subscription: {error}"),
3880            )),
3881        }
3882    }
3883
3884    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3885        let generation = self
3886            .registry
3887            .generation()
3888            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3889        let mut modules = Vec::new();
3890        for module in self.supervisor.list() {
3891            let status = module.status_for_control("list").map_err(|err| {
3892                RouterError::backend(
3893                    0,
3894                    frame.header.corr,
3895                    format!("failed to read supervisor status: {err}"),
3896                )
3897            })?;
3898            let (configured, _) = module.configuration().map_err(|err| {
3899                RouterError::backend(
3900                    0,
3901                    frame.header.corr,
3902                    format!("failed to read module configuration: {err}"),
3903                )
3904            })?;
3905            // Status and configuration snapshots release their locks before the image probe awaits.
3906            let image = module.running_image_agreement().await;
3907            // Read per request so the figure is current when the operator asks;
3908            // the daemon samples nothing in between.
3909            let resources = Some(module.child_resource_usage());
3910            let pending_reload = Some(reload_verdict(
3911                &configured.program,
3912                status.spawned_from.as_deref(),
3913                image,
3914            ));
3915            modules.push(SupervisorEntry {
3916                // Keep the retired policy field on the wire for one release so
3917                // existing status consumers still receive the platform policy.
3918                launch_nonce_env: Some(!cfg!(unix)),
3919                module_id: status.module_id,
3920                state: status.state.to_string(),
3921                enabled: status.enabled,
3922                live: status.live,
3923                protocol: status.protocol,
3924                health: status.health.status,
3925                pending_reload,
3926                last_probe_ms: status.health.last_probe_ms,
3927                last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3928                last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3929                last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3930                last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3931                restart_count: Some(status.restart_count),
3932                max_restarts: Some(status.max_restarts),
3933                lifetime_restarts: Some(status.lifetime_restarts),
3934                spawn_generation: Some(status.spawn_generation),
3935                restart_window_secs: Some(status.restart_window.as_secs()),
3936                drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3937                restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3938                restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3939                resources,
3940            });
3941        }
3942        let response = ClientControlResponse::SupervisorList {
3943            generation,
3944            modules,
3945        };
3946        Ok(vec![control_response_body_frame(
3947            &frame,
3948            &response,
3949            "ClientControlResponse::SupervisorList",
3950        )?])
3951    }
3952
3953    fn handle_supervisor_stderr_tail(
3954        &self,
3955        frame: Frame,
3956        module_id: String,
3957        max_lines: Option<u32>,
3958        max_bytes: Option<u32>,
3959    ) -> Result<Vec<Frame>, RouterError> {
3960        let Some(module) = self.supervisor.get(&module_id) else {
3961            return Ok(vec![control_error_frame(
3962                &frame,
3963                "unknown_module",
3964                format!("module_id '{module_id}' is not supervised"),
3965            )?]);
3966        };
3967
3968        let snapshot = module.stderr_tail(
3969            max_lines.map(|value| value as usize),
3970            max_bytes.map(|value| value as usize),
3971        );
3972
3973        let response = ClientControlResponse::SupervisorStderrTail {
3974            module_id,
3975            tail: StderrTail {
3976                capture: match snapshot.capture {
3977                    CaptureState::Captured => StderrCaptureState::Captured,
3978                    CaptureState::Incomplete { reason } => {
3979                        StderrCaptureState::Incomplete { reason }
3980                    }
3981                    CaptureState::NotCaptured { reason } => {
3982                        StderrCaptureState::NotCaptured { reason }
3983                    }
3984                },
3985                entries: snapshot
3986                    .entries
3987                    .into_iter()
3988                    .map(|entry| match entry {
3989                        TailEntry::Line {
3990                            text,
3991                            truncated,
3992                            at_ms,
3993                        } => StderrTailEntry::Line {
3994                            text,
3995                            truncated,
3996                            at_ms,
3997                        },
3998                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3999                    })
4000                    .collect(),
4001                dropped_lines: snapshot.dropped_lines,
4002            },
4003        };
4004        Ok(vec![control_response_body_frame(
4005            &frame,
4006            &response,
4007            "ClientControlResponse::SupervisorStderrTail",
4008        )?])
4009    }
4010
4011    async fn handle_supervisor_terminals(
4012        &self,
4013        frame: Frame,
4014        module_id: String,
4015    ) -> Result<Vec<Frame>, RouterError> {
4016        let Some(module) = self.supervisor.get(&module_id) else {
4017            return Ok(vec![control_error_frame(
4018                &frame,
4019                "unknown_module",
4020                format!("module_id '{module_id}' is not supervised"),
4021            )?]);
4022        };
4023
4024        // The journal read runs on a blocking thread: it can be megabytes of
4025        // file I/O and must not occupy a runtime worker.
4026        let terminals = module
4027            .read_durable_terminal_history()
4028            .await
4029            .map_err(|error| {
4030                RouterError::backend(
4031                    0,
4032                    frame.header.corr,
4033                    format!("failed to read terminal history: {error}"),
4034                )
4035            })?;
4036        let response = ClientControlResponse::SupervisorTerminals {
4037            module_id,
4038            terminals,
4039        };
4040        Ok(vec![control_response_body_frame(
4041            &frame,
4042            &response,
4043            "ClientControlResponse::SupervisorTerminals",
4044        )?])
4045    }
4046
4047    fn handle_supervisor_routes(
4048        &self,
4049        frame: Frame,
4050        module_id: Option<String>,
4051    ) -> Result<Vec<Frame>, RouterError> {
4052        let modules = self
4053            .forwarding
4054            .route_census(module_id.as_deref())
4055            .map_err(RouterError::Forwarding)?
4056            .into_iter()
4057            .map(|(module_id, routes)| SupervisorRouteModule {
4058                module_id,
4059                routes: routes
4060                    .into_iter()
4061                    .map(|route| SupervisorRoute {
4062                        consumer: match route.principal {
4063                            Principal::Reserved { module_id } => {
4064                                SupervisorRouteConsumer::Reserved { module_id }
4065                            }
4066                            Principal::Direct | Principal::Unverified => {
4067                                SupervisorRouteConsumer::Direct {
4068                                    connection_id: route.goodbye_target.connection_id.get(),
4069                                }
4070                            }
4071                        },
4072                        age_ms: Instant::now()
4073                            .saturating_duration_since(route.bound_at)
4074                            .as_millis()
4075                            .try_into()
4076                            .unwrap_or(u64::MAX),
4077                        draining: route.draining,
4078                        drain_reason: route.drain_reason,
4079                    })
4080                    .collect(),
4081            })
4082            .collect();
4083        let response = ClientControlResponse::SupervisorRoutes { modules };
4084        Ok(vec![control_response_body_frame(
4085            &frame,
4086            &response,
4087            "ClientControlResponse::SupervisorRoutes",
4088        )?])
4089    }
4090
4091    async fn handle_supervisor_provenance(
4092        &self,
4093        frame: Frame,
4094        module_id: Option<String>,
4095    ) -> Result<Vec<Frame>, RouterError> {
4096        let mut selected = if let Some(module_id) = module_id {
4097            let Some(module) = self.supervisor.get(&module_id) else {
4098                return Ok(vec![control_error_frame(
4099                    &frame,
4100                    "unknown_module",
4101                    format!("module_id '{module_id}' is not supervised"),
4102                )?]);
4103            };
4104            vec![module]
4105        } else {
4106            self.supervisor.list()
4107        };
4108
4109        let mut modules = Vec::with_capacity(selected.len());
4110        for module in selected.drain(..) {
4111            let status = module.status().map_err(|err| {
4112                RouterError::backend(
4113                    0,
4114                    frame.header.corr,
4115                    format!("failed to read supervisor status: {err}"),
4116                )
4117            })?;
4118            let module_declared = self
4119                .registry
4120                .get_module(&status.module_id)
4121                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4122                .and_then(|registration| registration.manifest.provenance)
4123                .map(|build| ModuleDeclaredProvenance::Reported { build })
4124                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4125            #[cfg(test)]
4126            let running_image = match &self.provenance_probe_override {
4127                Some(result) => result.clone(),
4128                None => module.running_image_agreement().await,
4129            };
4130            #[cfg(not(test))]
4131            let running_image = module.running_image_agreement().await;
4132            modules.push(SupervisorModuleProvenance {
4133                module_id: status.module_id,
4134                module_declared,
4135                daemon_observed: SupervisorObservedProcess {
4136                    pid: status.pid,
4137                    spawned_at_ms: status.spawned_at_ms,
4138                    spawned_from: status.spawned_from,
4139                    running_image,
4140                },
4141            });
4142        }
4143        let daemon = SupervisorDaemonProvenance {
4144            daemon_build: self.daemon_provenance.build.clone(),
4145            daemon_observed: DaemonObservedProcess {
4146                pid: self.daemon_provenance.pid,
4147                started_at_ms: self
4148                    .daemon_provenance
4149                    .start_clock
4150                    .map(|clock| clock.started_at_ms())
4151                    .or(self.daemon_provenance.started_at_ms),
4152                running_image: self
4153                    .daemon_provenance
4154                    .probe
4155                    .observe(
4156                        self.daemon_provenance.pid,
4157                        self.daemon_provenance.executable_path.as_deref(),
4158                        self.daemon_provenance.executable_identity,
4159                        self.daemon_provenance.process_start_time,
4160                    )
4161                    .await,
4162            },
4163        };
4164        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4165        Ok(vec![control_response_body_frame(
4166            &frame,
4167            &response,
4168            "ClientControlResponse::SupervisorProvenance",
4169        )?])
4170    }
4171
4172    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4173        self.refresh_capability_requirements();
4174        let generation = self
4175            .registry
4176            .generation()
4177            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4178        let modules = self
4179            .supervisor
4180            .list()
4181            .into_iter()
4182            .map(|module| {
4183                let status = module.status_for_control("health").map_err(|err| {
4184                    RouterError::backend(
4185                        0,
4186                        frame.header.corr,
4187                        format!("failed to read supervisor health: {err}"),
4188                    )
4189                })?;
4190                let module_id = status.module_id;
4191                let capability_detail = self
4192                    .capability_evaluator
4193                    .required_problem_detail(&module_id);
4194                Ok(SupervisorHealthEntry {
4195                    module_id,
4196                    status: status.health.status,
4197                    detail: append_capability_problem_detail(
4198                        status.health.detail,
4199                        capability_detail,
4200                    ),
4201                    metrics: status.health.metrics,
4202                    consecutive_failures: status.health.consecutive_failures,
4203                    late_answer_count: status.health.late_answer_count,
4204                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4205                    last_action: status.health.last_action,
4206                    last_action_ms: status.health.last_action_ms,
4207                    last_probe_ms: status.health.last_probe_ms,
4208                })
4209            })
4210            .collect::<Result<Vec<_>, RouterError>>()?;
4211        let response = ClientControlResponse::SupervisorHealth {
4212            generation,
4213            modules,
4214        };
4215        Ok(vec![control_response_body_frame(
4216            &frame,
4217            &response,
4218            "ClientControlResponse::SupervisorHealth",
4219        )?])
4220    }
4221
4222    async fn handle_supervisor_restart(
4223        &self,
4224        frame: Frame,
4225        module_id: String,
4226        drain_timeout_ms: Option<u64>,
4227    ) -> Result<Vec<Frame>, RouterError> {
4228        let operation_lock = self.supervisor.operation_lock();
4229        let _operation_guard = operation_lock.lock().await;
4230        let Some(module) = self.supervisor.get(&module_id) else {
4231            return Ok(vec![control_error_frame(
4232                &frame,
4233                "unknown_module",
4234                format!("module_id '{module_id}' is not supervised"),
4235            )?]);
4236        };
4237
4238        self.route_outages.mark_operator_action(&module_id);
4239        if let Err(err) = module.restart(drain_timeout_ms).await {
4240            self.route_outages
4241                .operator_action_ended_unrefused(&module_id);
4242            let (code, message) = match err {
4243                crate::supervise::SuperviseError::Disabled { .. } => {
4244                    ("module_disabled", err.to_string())
4245                }
4246                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4247                    ("swap_in_progress", err.to_string())
4248                }
4249                _ => (
4250                    "target_unavailable",
4251                    format!("failed to restart module_id '{module_id}': {err}"),
4252                ),
4253            };
4254            return Ok(vec![control_error_frame(&frame, code, message)?]);
4255        }
4256
4257        let response = ClientControlResponse::SupervisorAck {
4258            module_id,
4259            applied: true,
4260        };
4261        Ok(vec![control_response_body_frame(
4262            &frame,
4263            &response,
4264            "ClientControlResponse::SupervisorAck",
4265        )?])
4266    }
4267
4268    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
4269    /// when the old process has finished draining: a caller whose own lane
4270    /// rides the old process must get its reply before that drain waits on it.
4271    async fn handle_supervisor_swap(
4272        &self,
4273        frame: Frame,
4274        module_id: String,
4275        ready_timeout_ms: Option<u64>,
4276    ) -> Result<Vec<Frame>, RouterError> {
4277        // The daemon-wide operation lock is held only to resolve the handle,
4278        // not across the swap. The swap can take its whole readiness budget,
4279        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
4280        // holding it here would park an operator's stop behind the swap it is
4281        // meant to abort. A rescan or stop that reaches the module during the
4282        // swap is served by the swap itself (see `supervise_swap`).
4283        let module = {
4284            let operation_lock = self.supervisor.operation_lock();
4285            let _operation_guard = operation_lock.lock().await;
4286            self.supervisor.get(&module_id)
4287        };
4288        let Some(module) = module else {
4289            return Ok(vec![control_error_frame(
4290                &frame,
4291                "unknown_module",
4292                format!("module_id '{module_id}' is not supervised"),
4293            )?]);
4294        };
4295
4296        self.route_outages.mark_operator_action(&module_id);
4297        if let Err(err) = module
4298            .swap(ready_timeout_ms.map(Duration::from_millis))
4299            .await
4300        {
4301            self.route_outages
4302                .operator_action_ended_unrefused(&module_id);
4303            use crate::supervise::SuperviseError;
4304            let message = err.to_string();
4305            let error = match err {
4306                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4307                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4308                    code: "swap_refused".to_string(),
4309                    message,
4310                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4311                },
4312                SuperviseError::SwapFailed {
4313                    arm,
4314                    candidate_exit,
4315                    ..
4316                } => ErrorBody {
4317                    code: "swap_failed".to_string(),
4318                    message,
4319                    detail: Some(serde_json::json!({
4320                        "arm": arm.as_str(),
4321                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4322                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4323                    })),
4324                },
4325                _ => ErrorBody::new(
4326                    "target_unavailable",
4327                    format!("failed to swap module_id '{module_id}': {message}"),
4328                ),
4329            };
4330            return Ok(vec![control_error_body_frame(&frame, error)?]);
4331        }
4332        // A completed swap kept the incumbent serving until cutover, so it
4333        // usually opened no outage; a mark left behind would make the next,
4334        // unrelated outage read as requested.
4335        self.route_outages
4336            .operator_action_ended_unrefused(&module_id);
4337
4338        let response = ClientControlResponse::SupervisorAck {
4339            module_id,
4340            applied: true,
4341        };
4342        Ok(vec![control_response_body_frame(
4343            &frame,
4344            &response,
4345            "ClientControlResponse::SupervisorAck",
4346        )?])
4347    }
4348
4349    async fn handle_supervisor_reload(
4350        &self,
4351        frame: Frame,
4352        module_id: String,
4353    ) -> Result<Vec<Frame>, RouterError> {
4354        let operation_lock = self.supervisor.operation_lock();
4355        let _operation_guard = operation_lock.lock().await;
4356        let Some(module) = self.supervisor.get(&module_id) else {
4357            return Ok(vec![control_error_frame(
4358                &frame,
4359                "unknown_module",
4360                format!("module_id '{module_id}' is not supervised"),
4361            )?]);
4362        };
4363
4364        self.route_outages.mark_operator_action(&module_id);
4365        if let Err(err) = module.reload().await {
4366            self.route_outages
4367                .operator_action_ended_unrefused(&module_id);
4368            let (code, message) = match err {
4369                crate::supervise::SuperviseError::Disabled { .. } => {
4370                    ("module_disabled", err.to_string())
4371                }
4372                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4373                    ("swap_in_progress", err.to_string())
4374                }
4375                _ => (
4376                    "reload_failed",
4377                    format!("failed to reload module_id '{module_id}': {err}"),
4378                ),
4379            };
4380            return Ok(vec![control_error_frame(&frame, code, message)?]);
4381        }
4382
4383        let response = ClientControlResponse::SupervisorAck {
4384            module_id,
4385            applied: true,
4386        };
4387        Ok(vec![control_response_body_frame(
4388            &frame,
4389            &response,
4390            "ClientControlResponse::SupervisorAck",
4391        )?])
4392    }
4393
4394    async fn handle_supervisor_rescan(
4395        &self,
4396        frame: Frame,
4397        preview: bool,
4398    ) -> Result<Vec<Frame>, RouterError> {
4399        let Some(context) = self.rescan.clone() else {
4400            return Ok(vec![control_error_frame(
4401                &frame,
4402                "rescan_unavailable",
4403                "the daemon was not started with a reloadable config path".to_string(),
4404            )?]);
4405        };
4406
4407        let operation_lock = self.supervisor.operation_lock();
4408        let _operation_guard = operation_lock.lock().await;
4409        let loaded = match crate::daemon_config::load(&context.config_path) {
4410            Ok(config) => config,
4411            Err(err) => {
4412                return Ok(vec![control_error_frame(
4413                    &frame,
4414                    "invalid_daemon_config",
4415                    format!("supervisor rescan rejected daemon config: {err}"),
4416                )?])
4417            }
4418        };
4419        // `load` reports a missing file as Ok(None), which is correct at boot
4420        // (no config, nothing to supervise) and catastrophic here: rescan treats
4421        // "not in the config" as "remove it", so an absent file would read as an
4422        // empty module list and retire the entire running fleet. An editor
4423        // writing via write-new-then-rename, or a half-finished edit, is enough
4424        // to open that window. Refuse instead: a config that cannot be read
4425        // carries no instruction to remove anything.
4426        let Some(config) = loaded else {
4427            return Ok(vec![control_error_frame(
4428                &frame,
4429                "invalid_daemon_config",
4430                format!(
4431                    "daemon config not found at {}; refusing to rescan (an absent config would \
4432                     retire every supervised module)",
4433                    context.config_path.display()
4434                ),
4435            )?]);
4436        };
4437        let (
4438            configured_port,
4439            storage_config,
4440            admission_facts_carrier_module_id,
4441            admission_facts_targets,
4442            scope_authority_owners,
4443            modules,
4444            reserved_capabilities,
4445        ) = (
4446            config.port,
4447            config.storage,
4448            config.admission_facts_carrier_module_id,
4449            config.admission_facts_targets,
4450            config.scope_authority_owners,
4451            config.modules,
4452            config.reserved_capabilities,
4453        );
4454
4455        // Collect the sections rescan cannot apply, so the REPLY carries them.
4456        //
4457        // The warning below has always been correct and has always gone only to
4458        // the journal -- addressed to whoever reads logs, while the person who
4459        // just edited the config is looking at the CLI. Naming each section
4460        // individually rather than setting a flag: "something outside modules
4461        // changed" sends the operator back to diffing their own file, which is
4462        // the work this is meant to save.
4463        let mut restart_required = Vec::new();
4464        for section in RestartRequiredSection::ALL {
4465            let changed = match section {
4466                RestartRequiredSection::Port => configured_port != context.configured_port,
4467                RestartRequiredSection::Storage => storage_config != context.storage_config,
4468                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4469                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4470                }
4471                RestartRequiredSection::AdmissionFactsTargets => {
4472                    admission_facts_targets != context.admission_facts_targets
4473                }
4474                RestartRequiredSection::ScopeAuthorityOwners => {
4475                    scope_authority_owners != context.scope_authority_owners
4476                }
4477            };
4478            if changed {
4479                restart_required.push(section.label().to_string());
4480            }
4481        }
4482        if !restart_required.is_empty() {
4483            warn!(
4484                config_path = %context.config_path.display(),
4485                sections = %restart_required.join(", "),
4486                "daemon config changed outside the modules section; restart the daemon to apply those changes"
4487            );
4488        }
4489
4490        for configured in &modules {
4491            if let Err(err) = validate_spec(&configured.module_spec()) {
4492                return Ok(vec![control_error_frame(
4493                    &frame,
4494                    "invalid_daemon_config",
4495                    format!("supervisor rescan rejected daemon config: {err}"),
4496                )?]);
4497            }
4498        }
4499
4500        let configured_capabilities = modules
4501            .iter()
4502            .map(|module| (module.module_id.clone(), module.enabled))
4503            .collect::<Vec<_>>();
4504        let preview_capability_warnings = if preview {
4505            let (_, registrations) = self.runtime_capability_snapshot()?;
4506            let current_modules = self
4507                .supervisor
4508                .list()
4509                .into_iter()
4510                .map(|module| module.module_id().to_string())
4511                .collect::<BTreeSet<_>>();
4512            let resulting_modules = configured_capabilities.clone();
4513            let removed = current_modules
4514                .into_iter()
4515                .filter(|module_id| {
4516                    !resulting_modules
4517                        .iter()
4518                        .any(|(configured_id, _)| configured_id == module_id)
4519                })
4520                .collect::<Vec<_>>();
4521            self.capability_evaluator.preview_removal_warnings(
4522                resulting_modules,
4523                &removed,
4524                &registrations,
4525            )
4526        } else {
4527            Vec::new()
4528        };
4529        let result = match self
4530            .reconcile_supervised_modules(&context.supervisor, modules, preview)
4531            .await
4532        {
4533            Ok(result) => result,
4534            Err(message) => {
4535                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4536            }
4537        };
4538        if !preview {
4539            self.capability_evaluator
4540                .configure(configured_capabilities, reserved_capabilities);
4541            self.capability_evaluator.wake_deadline_loop();
4542            self.refresh_capability_requirements();
4543        }
4544        let mut result = result;
4545        result.restart_required = restart_required;
4546        result.capability_warnings = preview_capability_warnings;
4547        let response = ClientControlResponse::SupervisorRescan { result };
4548        Ok(vec![control_response_body_frame(
4549            &frame,
4550            &response,
4551            "ClientControlResponse::SupervisorRescan",
4552        )?])
4553    }
4554
4555    async fn handle_supervisor_release_reserved(
4556        &self,
4557        frame: Frame,
4558        module_id: String,
4559    ) -> Result<Vec<Frame>, RouterError> {
4560        let Some(context) = self.rescan.clone() else {
4561            return Ok(vec![control_error_frame(
4562                &frame,
4563                "release_unavailable",
4564                "reserved-id release requires a daemon started with a reloadable config path",
4565            )?]);
4566        };
4567        let operation_lock = self.supervisor.operation_lock();
4568        let _operation_guard = operation_lock.lock().await;
4569        let loaded = match crate::daemon_config::load(&context.config_path) {
4570            Ok(Some(config)) => config,
4571            Ok(None) => {
4572                return Ok(vec![control_error_frame(
4573                    &frame,
4574                    "invalid_daemon_config",
4575                    format!(
4576                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4577                        context.config_path.display()
4578                    ),
4579                )?])
4580            }
4581            Err(err) => {
4582                return Ok(vec![control_error_frame(
4583                    &frame,
4584                    "invalid_daemon_config",
4585                    format!("unable to verify reserved-id release against daemon config: {err}"),
4586                )?])
4587            }
4588        };
4589        if loaded
4590            .modules
4591            .iter()
4592            .any(|configured| configured.module_id == module_id)
4593        {
4594            return Ok(vec![control_error_frame(
4595                &frame,
4596                "reserved_module_configured",
4597                format!(
4598                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4599                ),
4600            )?]);
4601        }
4602        if !self.supervisor.release_retained_reserved_gate(&module_id) {
4603            return Ok(vec![control_error_frame(
4604                &frame,
4605                "reserved_gate_not_retained",
4606                format!(
4607                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4608                ),
4609            )?]);
4610        }
4611
4612        let response = ClientControlResponse::SupervisorAck {
4613            module_id,
4614            applied: true,
4615        };
4616        Ok(vec![control_response_body_frame(
4617            &frame,
4618            &response,
4619            "ClientControlResponse::SupervisorAck",
4620        )?])
4621    }
4622
4623    /// Reconcile the running module set against the configured one.
4624    ///
4625    /// With `preview` set, the diff is computed and returned WITHOUT applying any
4626    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
4627    /// deliberately shares this function with the executing path rather than
4628    /// computing the same diff somewhere else -- two implementations of one
4629    /// decision agree until they do not, and the whole value of a preview is that
4630    /// it describes the operation that will actually run.
4631    async fn reconcile_supervised_modules(
4632        &self,
4633        supervisor: &Supervisor,
4634        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4635        preview: bool,
4636    ) -> Result<SupervisorRescanResult, String> {
4637        let mut current = BTreeMap::new();
4638        for module in self.supervisor.list() {
4639            let (spec, health) = module.configuration().map_err(|err| {
4640                format!(
4641                    "failed to read configuration for module_id '{}': {err}",
4642                    module.module_id()
4643                )
4644            })?;
4645            let enabled = module
4646                .status()
4647                .map_err(|err| {
4648                    format!(
4649                        "failed to read status for module_id '{}': {err}",
4650                        module.module_id()
4651                    )
4652                })?
4653                .enabled;
4654            current.insert(
4655                module.module_id().to_string(),
4656                (module, spec, health, enabled),
4657            );
4658        }
4659        let configured = configured_modules
4660            .into_iter()
4661            .map(|module| (module.module_id.clone(), module))
4662            .collect::<BTreeMap<_, _>>();
4663
4664        let added = configured
4665            .keys()
4666            .filter(|module_id| !current.contains_key(*module_id))
4667            .cloned()
4668            .collect::<Vec<_>>();
4669        let removed = current
4670            .keys()
4671            .filter(|module_id| !configured.contains_key(*module_id))
4672            .cloned()
4673            .collect::<Vec<_>>();
4674        let mut changed_pending_reload = Vec::new();
4675        let mut configuration_changes = BTreeSet::new();
4676        let mut enabled_changes = BTreeSet::new();
4677        let mut unchanged = 0_u32;
4678
4679        for (module_id, configured_module) in &configured {
4680            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4681            else {
4682                continue;
4683            };
4684            // Compare the whole launch spec so a future launch field cannot
4685            // accidentally become a live-only policy change. Health is stored
4686            // separately and applies live without replacing the process.
4687            let launch_changed = *current_spec != configured_module.module_spec();
4688            let configuration_changed =
4689                launch_changed || *current_health != configured_module.health;
4690            let enabled_changed = *current_enabled != configured_module.enabled;
4691            if configuration_changed {
4692                configuration_changes.insert(module_id.clone());
4693            }
4694            if launch_changed {
4695                changed_pending_reload.push(module_id.clone());
4696            }
4697            if enabled_changed {
4698                enabled_changes.insert(module_id.clone());
4699            }
4700            if !configuration_changed && !enabled_changed {
4701                unchanged = unchanged.saturating_add(1);
4702            }
4703        }
4704
4705        // Everything above this point is pure computation over two snapshots.
4706        // Everything below MUTATES. The preview returns here so the boundary is a
4707        // single early return rather than a condition repeated at each mutation
4708        // site, where one missed guard would apply part of a change the caller was
4709        // told would not happen.
4710        if preview {
4711            return Ok(SupervisorRescanResult {
4712                added,
4713                removed,
4714                changed_pending_reload,
4715                enabled_changes: enabled_changes.iter().cloned().collect(),
4716                unchanged,
4717                preview: true,
4718                // Filled by the caller on both paths, so the preview reports
4719                // restart-required sections identically to an executed rescan --
4720                // the preview is where an operator is most likely to be looking.
4721                restart_required: Vec::new(),
4722                capability_warnings: Vec::new(),
4723            });
4724        }
4725
4726        for module_id in &removed {
4727            let module = &current
4728                .get(module_id)
4729                .expect("removed module came from current supervisor state")
4730                .0;
4731            module.retire().await.map_err(|err| {
4732                format!("failed to retire module_id '{module_id}' during rescan: {err}")
4733            })?;
4734            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
4735            //
4736            // `handle_route_open` resolves an absent module in three steps:
4737            // registry, then supervisor status, then tombstone. Retiring first
4738            // opens a window where ALL THREE ARE ABSENT -- the registry entry
4739            // went with the teardown above, the supervisor entry went with
4740            // `retire`, and the tombstone does not exist yet -- so a route.open
4741            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
4742            // it") for a module that was deliberately removed and whose caller
4743            // should get `module_removed` (TERMINAL, carrying a removal age).
4744            //
4745            // Writing the tombstone first closes it: during the window the
4746            // supervisor entry still answers, so the caller gets
4747            // `target_unavailable` -- retryable, and TRUE, because the module
4748            // is mid-teardown. After both statements it is `module_removed`.
4749            // No instant remains where a removed module reads as one that
4750            // never existed.
4751            //
4752            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
4753            // the absence of a test beside a fix invites deletion: these are
4754            // two sync statements with no await between them, so reaching the
4755            // window needs a second worker thread to land exactly between them
4756            // and there is no hook to force it. MEASURED: the 25 daemon_config
4757            // tests pass identically with the old order and the new one, so
4758            // the existing suite cannot see this and a green run is not
4759            // evidence either way. What the suite does hold is the
4760            // post-condition -- a removed module answers `module_removed` --
4761            // which this preserves.
4762            //
4763            // Found by an Athena panel reading the shipped tree against a
4764            // design note (2026-09-19), as the one concrete instance of that
4765            // note's class that survived contact with source. Direction is
4766            // benign: retryable where terminal was intended, never the reverse.
4767            self.supervisor.record_rescan_removal(module_id);
4768            self.supervisor.retire(module_id);
4769            self.route_outages.forget(module_id);
4770        }
4771
4772        for module_id in configured.keys() {
4773            let Some((module, _, _, _)) = current.get(module_id) else {
4774                continue;
4775            };
4776            let configured_module = configured
4777                .get(module_id)
4778                .expect("configured module id came from configured map");
4779            if configuration_changes.contains(module_id) {
4780                module
4781                    .update_configuration(
4782                        configured_module.module_spec(),
4783                        configured_module.health.clone(),
4784                        configured_module.drain_timeout_ms,
4785                    )
4786                    .await
4787                    .map_err(|err| {
4788                        format!(
4789                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
4790                        )
4791                    })?;
4792            }
4793            if enabled_changes.contains(module_id) {
4794                // A rescan that starts or stops a module applies an operator's
4795                // edit to the config, so the resulting outage was asked for.
4796                self.route_outages.mark_operator_action(module_id);
4797                module
4798                    .set_enabled(configured_module.enabled)
4799                    .await
4800                    .map_err(|err| {
4801                        self.route_outages.operator_action_ended_unrefused(module_id);
4802                        format!(
4803                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4804                            configured_module.enabled
4805                        )
4806                    })?;
4807            }
4808        }
4809
4810        for module_id in &added {
4811            let configured_module = configured
4812                .get(module_id)
4813                .expect("added module id came from configured map");
4814            supervisor
4815                .supervise_configured_with_health(
4816                    configured_module.module_spec(),
4817                    configured_module.enabled,
4818                    configured_module.health.clone(),
4819                    configured_module.drain_timeout_ms,
4820                    configured_module.restart,
4821                )
4822                .map_err(|err| {
4823                    format!("failed to add module_id '{module_id}' during rescan: {err}")
4824                })?;
4825        }
4826
4827        Ok(SupervisorRescanResult {
4828            added,
4829            removed,
4830            changed_pending_reload,
4831            enabled_changes: enabled_changes.iter().cloned().collect(),
4832            unchanged,
4833            preview: false,
4834            // Filled by the caller, which is the only layer that can see the
4835            // previous config to diff against.
4836            restart_required: Vec::new(),
4837            capability_warnings: Vec::new(),
4838        })
4839    }
4840
4841    async fn handle_supervisor_set_enabled(
4842        &self,
4843        frame: Frame,
4844        module_id: String,
4845        enabled: bool,
4846    ) -> Result<Vec<Frame>, RouterError> {
4847        let operation_lock = self.supervisor.operation_lock();
4848        let _operation_guard = operation_lock.lock().await;
4849        let Some(module) = self.supervisor.get(&module_id) else {
4850            return Ok(vec![control_error_frame(
4851                &frame,
4852                "unknown_module",
4853                format!("module_id '{module_id}' is not supervised"),
4854            )?]);
4855        };
4856
4857        // Enabling counts as well as disabling: a module an operator starts
4858        // is refused until it registers, and that wait was asked for.
4859        self.route_outages.mark_operator_action(&module_id);
4860        let applied = match module.set_enabled(enabled).await {
4861            Ok(applied) => applied,
4862            Err(err) => {
4863                self.route_outages
4864                    .operator_action_ended_unrefused(&module_id);
4865                return Ok(vec![control_error_frame(
4866                    &frame,
4867                    "target_unavailable",
4868                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4869                )?]);
4870            }
4871        };
4872        if !applied {
4873            // Already in the requested state: nothing was made unavailable,
4874            // so the mark must not outlive this request.
4875            self.route_outages
4876                .operator_action_ended_unrefused(&module_id);
4877        }
4878
4879        self.capability_evaluator.wake_deadline_loop();
4880        self.refresh_capability_requirements();
4881        let response = ClientControlResponse::SupervisorAck { module_id, applied };
4882        Ok(vec![control_response_body_frame(
4883            &frame,
4884            &response,
4885            "ClientControlResponse::SupervisorAck",
4886        )?])
4887    }
4888
4889    async fn handle_supervisor_health_probe(
4890        &self,
4891        frame: Frame,
4892        module_id: String,
4893    ) -> Result<Vec<Frame>, RouterError> {
4894        self.refresh_capability_requirements();
4895        let Some(registration) = self
4896            .registry
4897            .get_module(&module_id)
4898            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4899        else {
4900            return Ok(vec![control_error_frame(
4901                &frame,
4902                "unknown_module",
4903                format!("module_id '{module_id}' is not registered"),
4904            )?]);
4905        };
4906
4907        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
4908        // named for it. Making `module_registration_grants_op` return false
4909        // unconditionally reddens five tests, and every one is named for something
4910        // else -- capability relay, probe/bind demultiplexing, supervision-only
4911        // probing. They exercise a successful advertisement check on the way to their
4912        // own subject.
4913        //
4914        // Real protection, fragile in a specific way: narrowing any of those tests to
4915        // focus on its stated subject would silently remove coverage nobody knows
4916        // they are carrying. Recorded here rather than as a sixth test, because the
4917        // useful fact is WHICH tests hold the guard up -- a new test would add
4918        // coverage without telling the next person what the existing ones quietly do.
4919        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4920        {
4921            return Ok(vec![control_error_frame(
4922                &frame,
4923                "health_not_advertised",
4924                format!("module_id '{module_id}' did not advertise health.check"),
4925            )?]);
4926        }
4927
4928        let deadline = Instant::now() + self.health_probe_timeout;
4929        let pending = match self.forwarding.begin_module_control_rpc_for(
4930            &module_id,
4931            MODULE_CONTROL_OP_HEALTH_CHECK,
4932            deadline,
4933        ) {
4934            Ok(pending) => pending,
4935            Err(err) => {
4936                return Ok(vec![control_error_frame(
4937                    &frame,
4938                    forwarding_error_code(&err),
4939                    err.to_string(),
4940                )?])
4941            }
4942        };
4943
4944        let PendingModuleControlRpc {
4945            endpoint,
4946            module_sink,
4947            negotiated_ver,
4948            corr: probe_corr,
4949            receiver,
4950        } = pending;
4951        let mut guard =
4952            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4953        let probe_body =
4954            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4955                RouterError::backend(
4956                    0,
4957                    frame.header.corr,
4958                    format!("failed to encode health.check request: {err}"),
4959                )
4960            })?;
4961        let probe_frame = Frame::build_with_version(
4962            negotiated_ver,
4963            FrameType::Request,
4964            control_flags(),
4965            0,
4966            0,
4967            probe_corr,
4968            probe_body,
4969        )
4970        .map_err(RouterError::FrameBuild)?;
4971
4972        if let Err(err) = module_sink.send(probe_frame).await {
4973            return Ok(vec![control_error_frame(
4974                &frame,
4975                "target_unavailable",
4976                err.to_string(),
4977            )?]);
4978        }
4979
4980        match timeout_at(deadline, receiver).await {
4981            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4982                guard.disarm();
4983                let Some(report) = response.health_report() else {
4984                    return Ok(vec![control_error_frame(
4985                        &frame,
4986                        "invalid_control_body",
4987                        "health.check RPC returned a non-health response",
4988                    )?]);
4989                };
4990                // Metrics go out whole here. The supervisor's cached snapshot
4991                // caps this blob (see truncate_health_metrics), and this path
4992                // exists precisely to answer without that cap -- so applying it
4993                // here would leave no way to see what the cached view drops.
4994                let HealthReport {
4995                    status,
4996                    detail,
4997                    metrics,
4998                } = report;
4999                let capability_detail = self
5000                    .capability_evaluator
5001                    .required_problem_detail(&module_id);
5002                let response = ClientControlResponse::SupervisorHealthProbe {
5003                    module_id,
5004                    status,
5005                    detail: append_capability_problem_detail(detail, capability_detail),
5006                    metrics,
5007                };
5008                Ok(vec![control_response_body_frame(
5009                    &frame,
5010                    &response,
5011                    "ClientControlResponse::SupervisorHealthProbe",
5012                )?])
5013            }
5014            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5015                guard.disarm();
5016                Ok(vec![control_error_body_frame(&frame, body)?])
5017            }
5018            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5019                guard.disarm();
5020                Ok(vec![control_error_frame(
5021                    &frame,
5022                    "target_unavailable",
5023                    message,
5024                )?])
5025            }
5026            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5027                guard.disarm();
5028                Ok(vec![control_error_frame(
5029                    &frame,
5030                    "invalid_control_body",
5031                    message,
5032                )?])
5033            }
5034            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5035                guard.disarm();
5036                Ok(vec![control_error_frame(
5037                    &frame,
5038                    "invalid_control_body",
5039                    format!("expected module-control op '{expected}', got '{actual}'"),
5040                )?])
5041            }
5042            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5043                guard.disarm();
5044                Ok(vec![control_error_frame(
5045                    &frame,
5046                    "module_timeout",
5047                    format!(
5048                        "module_id '{module_id}' answered health.check after {:?}",
5049                        self.health_probe_timeout
5050                    ),
5051                )?])
5052            }
5053            Ok(Err(_)) => Ok(vec![control_error_frame(
5054                &frame,
5055                "target_unavailable",
5056                "health.check waiter was canceled before the module responded",
5057            )?]),
5058            Err(_) => Ok(vec![control_error_frame(
5059                &frame,
5060                "module_timeout",
5061                format!(
5062                    "module_id '{module_id}' did not answer health.check within {:?}",
5063                    self.health_probe_timeout
5064                ),
5065            )?]),
5066        }
5067    }
5068
5069    fn supervisor_status(
5070        &self,
5071        module_id: &str,
5072        corr: u64,
5073    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5074        self.supervisor
5075            .get(module_id)
5076            .map(|module| {
5077                let warming = module.is_warming_for_control("status").map_err(|err| {
5078                    RouterError::backend(
5079                        0,
5080                        corr,
5081                        format!(
5082                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
5083                        ),
5084                    )
5085                })?;
5086                module.status_for_control("status").map_err(|err| {
5087                    RouterError::backend(
5088                        0,
5089                        corr,
5090                        format!(
5091                            "failed to read supervisor status for module_id '{module_id}': {err}"
5092                        ),
5093                    )
5094                }).map(|status| (status, warming))
5095            })
5096            .transpose()
5097    }
5098
5099    fn guard_module_control_op(
5100        &self,
5101        frame: &Frame,
5102        module_id: &str,
5103        op: &str,
5104    ) -> Result<Option<Frame>, RouterError> {
5105        if self.module_grants_op(module_id, op, frame.header.corr)? {
5106            return Ok(None);
5107        }
5108
5109        Ok(Some(control_error_frame(
5110            frame,
5111            "op_not_allowed",
5112            format!("module_id '{module_id}' did not grant control op '{op}'"),
5113        )?))
5114    }
5115
5116    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5117        let Some(registration) = self
5118            .registry
5119            .get_module(module_id)
5120            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5121        else {
5122            return Ok(false);
5123        };
5124        Ok(module_registration_grants_op(&registration.control_ops, op))
5125    }
5126
5127    fn handle_status_update(
5128        &self,
5129        endpoint: ModuleEndpointId,
5130        frame: Frame,
5131    ) -> Result<Vec<Frame>, RouterError> {
5132        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5133            Ok(update) => update,
5134            Err(err) => {
5135                // Forward-compat: a newer module may push a channel-0 op this subc
5136                // version doesn't know. The control contract says unknown push ops
5137                // are IGNORED, never answered with an error. Only a malformed body
5138                // for an op we DO know is a real error worth surfacing.
5139                if is_known_module_push_op(&frame.body) {
5140                    return Ok(vec![control_error_frame(
5141                        &frame,
5142                        "invalid_control_body",
5143                        format!("malformed module control push body: {err}"),
5144                    )?]);
5145                }
5146                return Ok(Vec::new());
5147            }
5148        };
5149
5150        match update {
5151            ModuleControlPush::RouteStatus {
5152                route_channel,
5153                route_epoch,
5154                status,
5155            } => {
5156                self.forwarding
5157                    .cache_status(endpoint, route_channel, route_epoch, status)
5158                    .map_err(RouterError::Forwarding)?;
5159            }
5160        }
5161        Ok(Vec::new())
5162    }
5163
5164    fn handle_route_poll(
5165        &self,
5166        ctx: &RouteCtx,
5167        frame: Frame,
5168        route_channel: u16,
5169        route_epoch: u32,
5170        kind: PollKind,
5171    ) -> Result<Vec<Frame>, RouterError> {
5172        let snapshot = self
5173            .forwarding
5174            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5175            .map_err(RouterError::Forwarding)?;
5176        let response = match (kind, snapshot) {
5177            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5178                ClientControlResponse::RoutePoll {
5179                    route_channel,
5180                    route_epoch,
5181                    status,
5182                    live: None,
5183                }
5184            }
5185            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5186                route_channel,
5187                route_epoch,
5188                status: None,
5189                live: None,
5190            },
5191            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5192                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
5193                // is what makes reporting `true` correct rather than a
5194                // confident guess. `process_live` returns None only when the
5195                // module id has no supervisor snapshot at all -- an
5196                // externally-started module the daemon did not spawn -- and
5197                // for those the supervisor has no opinion to offer, ever. It
5198                // is never None for a supervised module in an unknown state:
5199                // a supervised module always has a snapshot, and the answer
5200                // comes from `state == Running && process_alive`.
5201                //
5202                // The route is Bound, so the module completed a HELLO on a
5203                // live connection; "the process this route points at is
5204                // running" is therefore attested by the binding rather than
5205                // assumed. Reporting `false` for an unsupervised module would
5206                // be the actual lie -- it would tell a client its healthy
5207                // route is dead because the daemon does not manage the
5208                // process.
5209                //
5210                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
5211                // module whose liveness is genuinely unknown, e.g. a snapshot
5212                // that has not been populated yet -- THIS DEFAULT BECOMES
5213                // WRONG and must split: unsupervised stays true, unknown
5214                // becomes null so the client can tell the two apart. The
5215                // response field is already `Option<bool>`, so the wire can
5216                // carry that distinction today.
5217                let live = self
5218                    .process_liveness
5219                    .as_ref()
5220                    .and_then(|source| source.process_live(&module_id))
5221                    .unwrap_or(true);
5222                ClientControlResponse::RoutePoll {
5223                    route_channel,
5224                    route_epoch,
5225                    status: None,
5226                    live: Some(live),
5227                }
5228            }
5229            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5230                route_channel,
5231                route_epoch,
5232                status: None,
5233                live: Some(false),
5234            },
5235        };
5236
5237        Ok(vec![control_response_body_frame(
5238            &frame,
5239            &response,
5240            "ClientControlResponse::RoutePoll",
5241        )?])
5242    }
5243
5244    pub(crate) fn observe_module_control_completion(
5245        &self,
5246        completion: ModuleControlRpcCompletion,
5247    ) -> bool {
5248        match completion {
5249            ModuleControlRpcCompletion::Unknown => false,
5250            ModuleControlRpcCompletion::Settled => true,
5251            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5252                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5253                info!(
5254                    module_id = %module_id,
5255                    latency_ms,
5256                    "late health.check answer proves the module is alive"
5257                );
5258                match self
5259                    .supervisor
5260                    .record_late_health_answer(&module_id, latency_ms)
5261                {
5262                    Ok(true) => {}
5263                    Ok(false) => debug!(
5264                        module_id = %module_id,
5265                        latency_ms,
5266                        "late health.check answer has no active supervisor snapshot"
5267                    ),
5268                    Err(err) => warn!(
5269                        module_id = %module_id,
5270                        latency_ms,
5271                        error = %err,
5272                        "failed to record late health.check answer"
5273                    ),
5274                }
5275                true
5276            }
5277        }
5278    }
5279
5280    /// Decide whether a failure while settling a relayed `route.bind` belongs to
5281    /// the module connection whose frame is being handled, or to the client that
5282    /// relay was opened for.
5283    ///
5284    /// This runs on the MODULE connection's frame handler, where returning `Err`
5285    /// ends that connection -- and a module connection carries every client's
5286    /// routes to that module, so ending it costs the whole fleet its tools.
5287    /// `ConnectionClosing` carries the id of the connection that is closing, and
5288    /// when that id is a CLIENT's, the condition is entirely about that one
5289    /// client's route.open. A client-scoped condition has no authority over a
5290    /// shared module connection, so it is logged and the single relay is dropped:
5291    /// the client is going away, and `complete_pending_relay` already removed the
5292    /// relay before failing, so there is nothing left to settle. Anything that
5293    /// relay still reserved is released by that client's own connection teardown,
5294    /// which is already under way -- that is what "closing" means.
5295    ///
5296    /// Every other failure is a statement about THIS connection and stays fatal:
5297    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
5298    /// id in `ConnectionClosing` all mean this connection cannot keep serving
5299    /// frames correctly.
5300    fn refuse_to_end_module_connection_for_a_client(
5301        &self,
5302        module_connection_id: ConnectionId,
5303        corr: u64,
5304        err: ForwardingError,
5305    ) -> Result<(), RouterError> {
5306        if let ForwardingError::ConnectionClosing { connection_id } = err {
5307            if connection_id != module_connection_id {
5308                warn!(
5309                    module_connection_id = module_connection_id.get(),
5310                    client_connection_id = connection_id.get(),
5311                    corr,
5312                    "dropping a route.bind response for a closing client; the module connection keeps serving"
5313                );
5314                return Ok(());
5315            }
5316        }
5317        Err(RouterError::Forwarding(err))
5318    }
5319
5320    fn handle_module_relay_response(
5321        &self,
5322        connection_id: ConnectionId,
5323        frame: Frame,
5324    ) -> Result<Vec<Frame>, RouterError> {
5325        let mut secondary_error = None;
5326        let outcome = match frame.header.ty {
5327            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5328                Ok(probe) if probe.op == "route.bind" => {
5329                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5330                        Ok(ModuleControlResponse::RouteBindAck {}) => {
5331                            RouteBindRelayOutcome::Accepted
5332                        }
5333                        Ok(other) => {
5334                            let message =
5335                                format!("route.bind response carried unexpected body: {other:?}");
5336                            secondary_error = Some(control_error_frame(
5337                                &frame,
5338                                "invalid_control_body",
5339                                message.clone(),
5340                            )?);
5341                            RouteBindRelayOutcome::ModuleGone(message)
5342                        }
5343                        Err(err) => {
5344                            let message = format!("malformed route.bind response body: {err}");
5345                            secondary_error = Some(control_error_frame(
5346                                &frame,
5347                                "invalid_control_body",
5348                                message.clone(),
5349                            )?);
5350                            RouteBindRelayOutcome::ModuleGone(message)
5351                        }
5352                    }
5353                }
5354                Ok(probe) => {
5355                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5356                    {
5357                        Ok(response) => ModuleControlRpcOutcome::Response(response),
5358                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5359                            "malformed {} response body: {err}",
5360                            probe.op
5361                        )),
5362                    };
5363                    let completion = self
5364                        .forwarding
5365                        .complete_module_control_rpc(
5366                            connection_id,
5367                            frame.header.corr,
5368                            Some(&probe.op),
5369                            outcome,
5370                        )
5371                        .map_err(RouterError::Forwarding)?;
5372                    if !self.observe_module_control_completion(completion) {
5373                        debug!(
5374                            connection_id = connection_id.get(),
5375                            corr = frame.header.corr,
5376                            op = %probe.op,
5377                            "dropping late or unknown module-control RPC response"
5378                        );
5379                    }
5380                    return Ok(Vec::new());
5381                }
5382                Err(err) => {
5383                    if let Some(expected_op) = self
5384                        .forwarding
5385                        .pending_module_control_op(connection_id, frame.header.corr)
5386                        .map_err(RouterError::Forwarding)?
5387                    {
5388                        let completion = self
5389                            .forwarding
5390                            .complete_module_control_rpc(
5391                                connection_id,
5392                                frame.header.corr,
5393                                None,
5394                                ModuleControlRpcOutcome::MalformedResponse(format!(
5395                                    "malformed {expected_op} response body: {err}"
5396                                )),
5397                            )
5398                            .map_err(RouterError::Forwarding)?;
5399                        if !self.observe_module_control_completion(completion) {
5400                            debug!(
5401                                connection_id = connection_id.get(),
5402                                corr = frame.header.corr,
5403                                "dropping late malformed module-control RPC response"
5404                            );
5405                        }
5406                        return Ok(Vec::new());
5407                    }
5408                    let message = format!("malformed route.bind response body: {err}");
5409                    secondary_error = Some(control_error_frame(
5410                        &frame,
5411                        "invalid_control_body",
5412                        message.clone(),
5413                    )?);
5414                    RouteBindRelayOutcome::ModuleGone(message)
5415                }
5416            },
5417            FrameType::Error => {
5418                if self
5419                    .forwarding
5420                    .pending_module_control_op(connection_id, frame.header.corr)
5421                    .map_err(RouterError::Forwarding)?
5422                    .is_some()
5423                {
5424                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5425                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5426                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5427                            "malformed module-control ERROR body: {err}"
5428                        )),
5429                    };
5430                    let completion = self
5431                        .forwarding
5432                        .complete_module_control_rpc(
5433                            connection_id,
5434                            frame.header.corr,
5435                            None,
5436                            outcome,
5437                        )
5438                        .map_err(RouterError::Forwarding)?;
5439                    if !self.observe_module_control_completion(completion) {
5440                        debug!(
5441                            connection_id = connection_id.get(),
5442                            corr = frame.header.corr,
5443                            "dropping late or unknown module-control RPC error"
5444                        );
5445                    }
5446                    return Ok(Vec::new());
5447                }
5448                match serde_json::from_slice::<ErrorBody>(&frame.body) {
5449                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
5450                    Err(err) => {
5451                        let message = format!("malformed route.bind ERROR body: {err}");
5452                        secondary_error = Some(control_error_frame(
5453                            &frame,
5454                            "invalid_control_body",
5455                            message.clone(),
5456                        )?);
5457                        RouteBindRelayOutcome::ModuleGone(message)
5458                    }
5459                }
5460            }
5461            ty => {
5462                return Ok(vec![control_error_frame(
5463                    &frame,
5464                    "unsupported_control_frame",
5465                    format!("unsupported module channel-0 frame {ty:?}"),
5466                )?])
5467            }
5468        };
5469
5470        let settled =
5471            self.forwarding
5472                .complete_pending_relay(connection_id, frame.header.corr, outcome);
5473        let completion = match settled {
5474            Ok(completion) => completion,
5475            Err(err) => {
5476                self.refuse_to_end_module_connection_for_a_client(
5477                    connection_id,
5478                    frame.header.corr,
5479                    err,
5480                )?;
5481                return Ok(secondary_error.into_iter().collect());
5482            }
5483        };
5484        if let Some(target) = completion.abandoned.as_ref() {
5485            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5486        }
5487        if !completion.settled {
5488            debug!(
5489                connection_id = connection_id.get(),
5490                corr = frame.header.corr,
5491                frame_type = ?frame.header.ty,
5492                "dropping late or unknown route.bind relay response"
5493            );
5494        }
5495        Ok(secondary_error.into_iter().collect())
5496    }
5497
5498    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5499        debug!(connection_id = connection_id.get(), "handling GOODBYE");
5500        // GOODBYE ends the connection's logical session even when its socket
5501        // stays open. Use disconnect teardown so verdicts, client notices and
5502        // scope authority are released at the same lifecycle boundary.
5503        self.cleanup_connection(connection_id)
5504            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5505        Ok(Vec::new())
5506    }
5507}
5508
5509impl Default for ControlHandler {
5510    fn default() -> Self {
5511        Self::new(Arc::new(Registry::default()))
5512    }
5513}
5514
5515impl crate::supervise::SwapPromotionObserver for ControlHandler {
5516    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5517        self.apply_registration_capabilities(registration);
5518    }
5519}
5520
5521fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5522    CapabilityRequirementStatus {
5523        consumer: status.consumer,
5524        capability: status.capability,
5525        need: match status.need {
5526            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5527            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5528        },
5529        verdict: status.verdict.as_str().to_string(),
5530        episode_seq: status.episode_seq,
5531        config_satisfiable: status.config_satisfiable,
5532        runtime_available: status.runtime_available,
5533        detail: status.detail,
5534    }
5535}
5536
5537fn append_capability_problem_detail(
5538    detail: Option<String>,
5539    capability_detail: Option<String>,
5540) -> Option<String> {
5541    match (detail, capability_detail) {
5542        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5543        (Some(detail), None) => Some(detail),
5544        (None, Some(capability_detail)) => Some(capability_detail),
5545        (None, None) => None,
5546    }
5547}
5548
5549fn subc_ops() -> Vec<String> {
5550    SUBC_CONTROL_OPS
5551        .iter()
5552        .map(|op| (*op).to_string())
5553        .collect()
5554}
5555
5556fn module_subc_ops() -> Vec<String> {
5557    SUBC_CONTROL_OPS
5558        .iter()
5559        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5560        .map(|op| (*op).to_string())
5561        .collect()
5562}
5563
5564#[cfg(test)]
5565fn module_baseline_control_ops() -> Vec<String> {
5566    MODULE_BASELINE_CONTROL_OPS
5567        .iter()
5568        .map(|op| (*op).to_string())
5569        .collect()
5570}
5571
5572fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5573    let mut seen = HashSet::new();
5574    let mut effective = Vec::new();
5575    for op in MODULE_BASELINE_CONTROL_OPS {
5576        if seen.insert((*op).to_string()) {
5577            effective.push((*op).to_string());
5578        }
5579    }
5580    for op in declared.unwrap_or_default() {
5581        if seen.insert(op.clone()) {
5582            effective.push(op);
5583        }
5584    }
5585    effective
5586}
5587
5588fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5589    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5590}
5591
5592fn target_module_id(target: &RouteTarget) -> &str {
5593    match target {
5594        RouteTarget::ToolProvider { module_id }
5595        | RouteTarget::ManagementSurface { module_id }
5596        | RouteTarget::InternalService { module_id, .. } => module_id,
5597    }
5598}
5599
5600fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5601    roles.iter().any(|role| match (target, role) {
5602        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5603        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5604        (
5605            RouteTarget::InternalService { service_id, .. },
5606            ProviderRole::InternalService {
5607                service_id: provided,
5608                ..
5609            },
5610        ) => service_id == provided,
5611        _ => false,
5612    })
5613}
5614
5615fn is_routable_role(role: &ProviderRole) -> bool {
5616    matches!(
5617        role,
5618        ProviderRole::ToolProvider { .. }
5619            | ProviderRole::ManagementSurface { .. }
5620            | ProviderRole::InternalService { .. }
5621    )
5622}
5623
5624#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5625enum ControlRequestBodyError {
5626    UnknownOp,
5627    InvalidBody,
5628}
5629
5630#[derive(Debug, Deserialize)]
5631struct ControlOpProbe {
5632    op: String,
5633}
5634
5635/// Channel-0 push ops this subc version understands. A push whose `op` is not in
5636/// this set is treated as a forward-compat unknown and ignored rather than errored.
5637const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5638
5639fn is_known_module_push_op(body: &[u8]) -> bool {
5640    serde_json::from_slice::<ControlOpProbe>(body)
5641        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5642        .unwrap_or(false)
5643}
5644
5645fn is_known_module_request_op(body: &[u8]) -> bool {
5646    serde_json::from_slice::<ControlOpProbe>(body)
5647        .map(|probe| is_module_to_subc_op(&probe.op))
5648        .unwrap_or(false)
5649}
5650
5651fn is_module_to_subc_op(op: &str) -> bool {
5652    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5653}
5654
5655fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5656    debug!(
5657        op = %op,
5658        connection_id = connection_id.get(),
5659        corr,
5660        "control dispatch"
5661    );
5662}
5663
5664fn log_slow_control_dispatch(
5665    dispatch_started_at: Option<StdInstant>,
5666    op: &'static str,
5667    connection_id: ConnectionId,
5668    corr: u64,
5669) {
5670    let Some(dispatch_started_at) = dispatch_started_at else {
5671        return;
5672    };
5673    let elapsed = dispatch_started_at.elapsed();
5674    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5675        warn!(
5676            op = %op,
5677            connection_id = connection_id.get(),
5678            corr,
5679            elapsed_ms = elapsed.as_millis() as u64,
5680            "slow control dispatch"
5681        );
5682    }
5683}
5684
5685fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5686    match request {
5687        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5688        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5689        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5690        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5691        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5692        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5693        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5694        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5695        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5696        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5697        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5698        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5699        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5700        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5701        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5702        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5703        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5704        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5705        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5706    }
5707}
5708
5709fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5710    match request {
5711        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5712        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5713        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5714        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5715    }
5716}
5717
5718fn parse_client_control_request(
5719    body: &[u8],
5720) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5721    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5722        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5723            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5724                ControlRequestBodyError::InvalidBody
5725            }
5726            Ok(_) => ControlRequestBodyError::UnknownOp,
5727            Err(_) => ControlRequestBodyError::InvalidBody,
5728        };
5729        (err, classification)
5730    })
5731}
5732
5733fn parse_module_control_request_from_module(
5734    body: &[u8],
5735) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5736    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5737        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5738            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5739            Ok(_) => ControlRequestBodyError::UnknownOp,
5740            Err(_) => ControlRequestBodyError::InvalidBody,
5741        };
5742        (err, classification)
5743    })
5744}
5745
5746#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5747enum ProviderRoleKind {
5748    ToolProvider,
5749    PipelineStage,
5750    ManagementSurface,
5751    InternalService,
5752}
5753
5754fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5755    match role {
5756        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5757        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5758        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5759        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5760    }
5761}
5762
5763fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5764    roles.iter().map(provider_role_kind).collect()
5765}
5766
5767/// Most refused scope records named individually in the log per sync; the
5768/// `refused` count on the accepted line is always complete.
5769const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5770
5771/// Per-outcome counts of one accepted `scope.sync`, for its log line.
5772#[derive(Debug, Default, PartialEq, Eq)]
5773struct ScopeOutcomeCounts {
5774    created: usize,
5775    replaced: usize,
5776    updated: usize,
5777    unchanged: usize,
5778    refused: usize,
5779}
5780
5781impl ScopeOutcomeCounts {
5782    fn of(results: &[ScopeRecordResult]) -> Self {
5783        let mut counts = Self::default();
5784        for result in results {
5785            let slot = match result.outcome {
5786                ScopeRecordOutcome::Created => &mut counts.created,
5787                ScopeRecordOutcome::Replaced => &mut counts.replaced,
5788                ScopeRecordOutcome::Updated => &mut counts.updated,
5789                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5790                ScopeRecordOutcome::Refused => &mut counts.refused,
5791            };
5792            *slot += 1;
5793        }
5794        counts
5795    }
5796}
5797
5798#[cfg(test)]
5799mod scope_outcome_count_tests {
5800    use super::*;
5801
5802    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5803        ScopeRecordResult {
5804            scope_ref: "r".to_string(),
5805            scope_epoch: 1,
5806            outcome,
5807            code: None,
5808            message: None,
5809            version: None,
5810            parent_state: None,
5811        }
5812    }
5813
5814    /// Each outcome lands in its own count, so a refused record can never be
5815    /// hidden inside the total the log already printed.
5816    #[test]
5817    fn every_outcome_is_counted_in_its_own_field() {
5818        let results = [
5819            result(ScopeRecordOutcome::Created),
5820            result(ScopeRecordOutcome::Created),
5821            result(ScopeRecordOutcome::Replaced),
5822            result(ScopeRecordOutcome::Updated),
5823            result(ScopeRecordOutcome::Unchanged),
5824            result(ScopeRecordOutcome::Refused),
5825            result(ScopeRecordOutcome::Refused),
5826            result(ScopeRecordOutcome::Refused),
5827        ];
5828        assert_eq!(
5829            ScopeOutcomeCounts::of(&results),
5830            ScopeOutcomeCounts {
5831                created: 2,
5832                replaced: 1,
5833                updated: 1,
5834                unchanged: 1,
5835                refused: 3,
5836            }
5837        );
5838    }
5839}
5840
5841/// Return whether a catalog change can create a newly violating live route.
5842/// Removing an attested claim is intentionally excluded: it makes fewer routes
5843/// forbidden and therefore must leave the existing route census untouched.
5844fn capability_census_trigger(
5845    old: Option<&CapabilityDeclarations>,
5846    new: Option<&CapabilityDeclarations>,
5847) -> bool {
5848    let old_provides = old
5849        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5850        .unwrap_or_default();
5851    let old_denies = old
5852        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5853        .unwrap_or_default();
5854    let new = new.cloned().unwrap_or(CapabilityDeclarations {
5855        provides: Vec::new(),
5856        requires: Vec::new(),
5857        must_never_reach: Vec::new(),
5858    });
5859
5860    new.provides
5861        .iter()
5862        .any(|capability| !old_provides.contains(capability))
5863        || new
5864            .must_never_reach
5865            .iter()
5866            .any(|capability| !old_denies.contains(capability))
5867}
5868
5869/// Find the first capability an attested opener denies that an attested target
5870/// claims. Both manifests are live registry records, never cached or client data.
5871fn denied_capability<'a>(
5872    opening_manifest: &'a ModuleManifest,
5873    target_manifest: &ModuleManifest,
5874) -> Option<&'a str> {
5875    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5876    let target_capabilities = target_manifest.capabilities.as_ref()?;
5877    opening_capabilities
5878        .must_never_reach
5879        .iter()
5880        .find(|denied| {
5881            target_capabilities
5882                .provides
5883                .iter()
5884                .any(|provided| provided == *denied)
5885        })
5886        .map(String::as_str)
5887}
5888
5889fn catalog_update_frozen_field_message(
5890    registered: &ModuleManifest,
5891    provides: &[ProviderRole],
5892) -> Option<String> {
5893    let old_has_provides = !registered.provides.is_empty();
5894    let new_has_provides = !provides.is_empty();
5895    if old_has_provides != new_has_provides {
5896        return Some(format!(
5897            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5898            registered.module_id
5899        ));
5900    }
5901
5902    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
5903        return Some(format!(
5904            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5905            registered.module_id
5906        ));
5907    }
5908
5909    let registered_concurrency = manifest_concurrency(registered);
5910    let mut candidate = registered.clone();
5911    candidate.provides = provides.to_vec();
5912    let candidate_concurrency = manifest_concurrency(&candidate);
5913    if candidate_concurrency != registered_concurrency {
5914        return Some(format!(
5915            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5916            registered.module_id, registered_concurrency, candidate_concurrency
5917        ));
5918    }
5919
5920    // control_ops live beside the manifest in the HELLO body, not inside
5921    // ModuleManifest, so a provides-only catalog.update cannot change them.
5922    None
5923}
5924
5925fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5926    manifest.provides.iter().any(is_routable_role)
5927}
5928
5929/// Returns the routable-provider concurrency subc should enforce for this manifest.
5930///
5931/// ToolProvider and ManagementSurface store their delivery concurrency directly.
5932/// InternalService has no role-specific concurrency field, so it retains the
5933/// existing ModuleManaged default for backward compatibility.
5934fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5935    manifest
5936        .provides
5937        .iter()
5938        .find_map(|provider| match provider {
5939            ProviderRole::ToolProvider { concurrency, .. }
5940            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5941            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5942        })
5943        .unwrap_or(Concurrency::ModuleManaged)
5944}
5945
5946/// True when the manifest carries a ManagementSurface role whose concurrency
5947/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
5948/// bytes because the typed manifest deliberately erases that distinction: the
5949/// default exists for wire compatibility, and this probe exists so the default
5950/// stays observable. Any parse irregularity returns false -- the caller only
5951/// logs, and a malformed body already failed registration upstream.
5952fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5953    let has_management_surface = manifest
5954        .provides
5955        .iter()
5956        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5957    if !has_management_surface {
5958        return false;
5959    }
5960    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5961        return false;
5962    };
5963    let Some(provides) = raw
5964        .get("manifest")
5965        .and_then(|manifest| manifest.get("provides"))
5966        .and_then(serde_json::Value::as_array)
5967    else {
5968        return false;
5969    };
5970    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
5971    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
5972    // against the management_surface_manifest_without_concurrency golden, not
5973    // recalled (the externally-tagged guess was this function's first bug).
5974    provides.iter().any(|role| {
5975        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5976            && role.get("concurrency").is_none()
5977    })
5978}
5979
5980fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5981    if peer_version != PROTOCOL_VERSION {
5982        return Err(format!(
5983            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5984        ));
5985    }
5986    Ok(PROTOCOL_VERSION)
5987}
5988
5989fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5990    Frame::build_with_version(
5991        response_version(frame),
5992        FrameType::Pong,
5993        frame.header.flags,
5994        0,
5995        0,
5996        frame.header.corr,
5997        Vec::new(),
5998    )
5999    .map_err(RouterError::FrameBuild)
6000}
6001
6002fn control_error_frame(
6003    frame: &Frame,
6004    code: &'static str,
6005    message: impl Into<String>,
6006) -> Result<Frame, RouterError> {
6007    control_error_body_frame(
6008        frame,
6009        ErrorBody {
6010            code: code.to_string(),
6011            message: message.into(),
6012            detail: None,
6013        },
6014    )
6015}
6016
6017fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6018    let body = serde_json::to_vec(&error).map_err(|err| {
6019        RouterError::backend(
6020            0,
6021            frame.header.corr,
6022            format!("failed to encode control ERROR: {err}"),
6023        )
6024    })?;
6025
6026    Frame::build_with_version(
6027        response_version(frame),
6028        FrameType::Error,
6029        control_flags(),
6030        0,
6031        0,
6032        frame.header.corr,
6033        body,
6034    )
6035    .map_err(RouterError::FrameBuild)
6036}
6037
6038fn control_response_body_frame<T: Serialize>(
6039    frame: &Frame,
6040    reply: &T,
6041    label: &'static str,
6042) -> Result<Frame, RouterError> {
6043    let body = serde_json::to_vec(reply).map_err(|err| {
6044        RouterError::backend(
6045            0,
6046            frame.header.corr,
6047            format!("failed to encode {label}: {err}"),
6048        )
6049    })?;
6050
6051    Frame::build_with_version(
6052        response_version(frame),
6053        FrameType::Response,
6054        control_flags(),
6055        0,
6056        0,
6057        frame.header.corr,
6058        body,
6059    )
6060    .map_err(RouterError::FrameBuild)
6061}
6062
6063/// Map a forwarding failure to the wire code a client sees.
6064///
6065/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
6066/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
6067/// chosen here decides whether a caller retries or gives up.
6068///
6069/// That makes attribution the load-bearing property, not merely having a code. A
6070/// permanent fault published as a retryable one produces a fleet-wide retry storm
6071/// against something that can never recover; a transient fault published as
6072/// permanent gives up on work that would have succeeded. Both look correct in a
6073/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
6074/// the mapping per variant rather than merely asserting that some code exists.
6075///
6076/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
6077/// two codes on the same side of the boundary passes it. Measured rather than
6078/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
6079/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
6080/// a test named for something else that happens to assert the string.
6081///
6082/// That accidental coverage is deliberately left alone rather than promoted to a
6083/// named test, because it guards a property this function does not promise.
6084/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
6085/// specific code within a class, so identity is free to change and only the
6086/// partition is a contract. Splitting it out would assert a guarantee nothing
6087/// depends on — and a suite that promises more than the code does is the harder
6088/// thing to correct later, because the next reader cannot tell which assertions
6089/// are load-bearing.
6090///
6091/// Pin identity here the moment a consumer branches on a specific code.
6092fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6093    match err {
6094        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6095        ForwardingError::NoModuleConnection => "target_unavailable",
6096        ForwardingError::ModuleReloading { .. } => "module_reloading",
6097        ForwardingError::ClientRouteChannelExhausted { .. }
6098        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6099        ForwardingError::StaleModuleEndpoint
6100        | ForwardingError::UnknownReservation { .. }
6101        | ForwardingError::ConnectionClosing { .. }
6102        | ForwardingError::ClientEgressClosed { .. }
6103        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6104        // Only a swap candidate's registration can produce this, and it means
6105        // exactly what a second active HELLO for a live id means.
6106        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6107        ForwardingError::RelayCorrelationExhausted
6108        | ForwardingError::RouteOpenBuild(_)
6109        | ForwardingError::Poisoned => "forwarding_error",
6110    }
6111}
6112
6113fn response_version(frame: &Frame) -> u8 {
6114    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6115        frame.header.ver
6116    } else {
6117        PROTOCOL_VERSION
6118    }
6119}
6120
6121fn control_flags() -> Flags {
6122    Flags::new(false, Priority::Passive, false)
6123}
6124
6125/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
6126/// channel. The target is the module (a client never saw the route), so this
6127/// takes the module path: delivered late rather than dropped when the module's
6128/// queue is momentarily full, and never closing its connection.
6129fn send_goodbye_target_best_effort(
6130    counters: &DaemonCounters,
6131    target: &GoodbyeTarget,
6132    context: &'static str,
6133) {
6134    let Ok(frame) = Frame::build_with_version(
6135        target.negotiated_ver,
6136        FrameType::Goodbye,
6137        control_flags(),
6138        target.channel,
6139        target.epoch,
6140        0,
6141        Vec::new(),
6142    ) else {
6143        return;
6144    };
6145    crate::forwarding::send_module_route_goodbye(
6146        counters,
6147        &target.sink,
6148        frame,
6149        target.module_id.as_deref(),
6150        context,
6151    );
6152}
6153
6154pub(crate) fn send_route_control_pushes(
6155    forwarding: &ForwardingTable,
6156    routes: Vec<EndpointRoute>,
6157    push: ClientControlPush,
6158) {
6159    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6160    for route in routes {
6161        let target = route.goodbye_target;
6162        if let Some((existing, channels)) = targets
6163            .iter_mut()
6164            .find(|(existing, _)| existing.connection_id == target.connection_id)
6165        {
6166            debug_assert_eq!(
6167                existing.negotiated_ver, target.negotiated_ver,
6168                "one connection cannot negotiate multiple frame versions"
6169            );
6170            if !channels.contains(&target.channel) {
6171                channels.push(target.channel);
6172            }
6173            continue;
6174        }
6175        let channel = target.channel;
6176        targets.push((target, vec![channel]));
6177    }
6178    for (target, mut channels) in targets {
6179        channels.sort_unstable();
6180        let mut push = push.clone();
6181        match &mut push {
6182            ClientControlPush::RouteClosing {
6183                channels: covered, ..
6184            }
6185            | ClientControlPush::RouteClosed {
6186                channels: covered, ..
6187            } => *covered = channels,
6188        }
6189        let body = match serde_json::to_vec(&push) {
6190            Ok(body) => body,
6191            Err(err) => {
6192                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6193                continue;
6194            }
6195        };
6196        let frame = match Frame::build_with_version(
6197            target.negotiated_ver,
6198            FrameType::Push,
6199            control_flags(),
6200            0,
6201            0,
6202            0,
6203            body.clone(),
6204        ) {
6205            Ok(frame) => frame,
6206            Err(err) => {
6207                warn!(
6208                    route_channel = target.channel,
6209                    error = %err,
6210                    "failed to build route lifecycle control PUSH frame"
6211                );
6212                continue;
6213            }
6214        };
6215        if let Err(err) = target.sink.try_send(frame) {
6216            if target.close_on_delivery_failure() {
6217                warn!(
6218                    target_connection_id = target.connection_id.get(),
6219                    route_channel = target.channel,
6220                    error = %err,
6221                    "route lifecycle control PUSH was not delivered to client; closing target connection"
6222                );
6223                let _ = forwarding.escalate_client_delivery_failure(
6224                    target.connection_id,
6225                    target.channel,
6226                    target.epoch,
6227                    CloseReason::new(
6228                        "route_lifecycle_push_delivery_failed",
6229                        format!(
6230                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6231                            target.channel
6232                        ),
6233                    ),
6234                    crate::forwarding::UndeliveredFrame {
6235                        module_id: target.module_id.as_deref(),
6236                        sink: &target.sink,
6237                    },
6238                );
6239            }
6240        }
6241    }
6242}
6243
6244#[cfg(test)]
6245mod tests {
6246    #[cfg(unix)]
6247    #[tokio::test]
6248    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6249        let dir = subc_test_support::TestTempDir::new("rescan-live-health");
6250        let path = dir.join("subc.jsonc");
6251        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6252            "program":"/bin/sleep","args":["60"],"protocol":"none",
6253            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6254        }}}).to_string()).unwrap();
6255        let mut configured = crate::daemon_config::load(&path)
6256            .unwrap()
6257            .unwrap()
6258            .modules
6259            .pop()
6260            .unwrap();
6261        let registry = std::sync::Arc::new(crate::Registry::default());
6262        let handle = crate::SupervisorHandle::new();
6263        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6264            .with_handle(handle.clone());
6265        let module = supervisor
6266            .supervise_configured_with_health(
6267                configured.module_spec(),
6268                true,
6269                configured.health.clone(),
6270                None,
6271                configured.restart,
6272            )
6273            .unwrap();
6274        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6275        let before = module.status().unwrap().pid;
6276        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6277        configured.health.cadence = std::time::Duration::from_secs(3600);
6278        let health_only = handler
6279            .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6280            .await
6281            .unwrap();
6282        assert!(
6283            health_only.changed_pending_reload.is_empty(),
6284            "health policy is already applied live"
6285        );
6286        assert_eq!(module.status().unwrap().pid, before);
6287        assert_eq!(
6288            module.configuration().unwrap().1.http,
6289            configured.health.http
6290        );
6291        configured.args = vec!["61".into()];
6292        let launch = handler
6293            .reconcile_supervised_modules(&supervisor, vec![configured], false)
6294            .await
6295            .unwrap();
6296        assert_eq!(launch.changed_pending_reload, ["stock"]);
6297        assert_eq!(
6298            module.status().unwrap().pid,
6299            before,
6300            "a launch edit is stored until reload"
6301        );
6302        module.drain().await.unwrap();
6303    }
6304    use std::{
6305        collections::BTreeMap,
6306        fmt,
6307        path::PathBuf,
6308        sync::{Arc, Mutex},
6309        time::Duration,
6310    };
6311    use subc_test_support::TestTempDir;
6312
6313    use serde_json::{json, Value};
6314    use subc_protocol::{
6315        manifest::{
6316            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6317            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6318        },
6319        session::HealthStatus,
6320        FrameType,
6321    };
6322
6323    use super::*;
6324    use crate::{
6325        forwarding::{DataRoute, DataRouteState},
6326        registry::ChannelState,
6327        router::FrameSink,
6328        stderr_tail::DEFAULT_MAX_LINE_BYTES,
6329        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6330        RouteCtx, Router,
6331    };
6332    use tokio::{
6333        sync::mpsc,
6334        time::{sleep, Instant},
6335    };
6336    use tracing::{
6337        field::{Field, Visit},
6338        Event, Subscriber,
6339    };
6340    use tracing_subscriber::{layer::Context, prelude::*, Layer};
6341
6342    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
6343    ///
6344    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
6345    /// is only populated for `tests/*.rs` integration test binaries -- this file
6346    /// compiles as part of the library target, which gets neither. This test's
6347    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
6348    /// and the sibling binary lives two directories up at
6349    /// `<target-dir>/<profile>/fake-aft-stub`.
6350    ///
6351    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
6352    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
6353    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
6354    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
6355    /// `NotFound`, which reads as a broken test rather than an unbuilt
6356    /// dependency -- so state the cause and the remedy instead. Deliberately a
6357    /// panic and not a silent skip: a test that quietly passes when it could not
6358    /// run is worse than one that fails, because it reports health it never
6359    /// verified.
6360    fn fake_aft_stub_path() -> PathBuf {
6361        let mut path = std::env::current_exe().expect("current_exe available in tests");
6362        path.pop(); // .../deps/
6363        path.pop(); // .../<profile>/
6364        path.push(if cfg!(windows) {
6365            "fake-aft-stub.exe"
6366        } else {
6367            "fake-aft-stub"
6368        });
6369        assert!(
6370            path.exists(),
6371            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6372             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6373            path.display()
6374        );
6375        path
6376    }
6377
6378    /// Whether clients retry `code` in place: the predicate itself, never a copy
6379    /// of its set. A copied list breaks silently when a code is added to or
6380    /// removed from the real one, and a stale copy here would let exactly the
6381    /// failure this test exists to catch pass.
6382    fn client_retries(code: &str) -> bool {
6383        subc_protocol::error_codes::is_retryable_route_open(code)
6384    }
6385
6386    /// A code is not a label — clients branch on it, so publishing the wrong KIND
6387    /// of failure is worse than publishing none. A permanent fault dressed as
6388    /// retryable makes every client in the fleet retry forever against something
6389    /// that cannot recover; a transient fault dressed as permanent abandons work
6390    /// that would have succeeded.
6391    ///
6392    /// Asserting "a code exists" cannot catch either, because the string is free
6393    /// to say anything. This enumerates every variant and pins which side of the
6394    /// retry boundary it lands on, so a new variant must be classified here
6395    /// deliberately rather than inheriting whichever arm it was appended to.
6396    #[test]
6397    fn retryability_of_forwarding_codes_matches_the_failure() {
6398        // Transient by nature: the target is booting, reloading, or its endpoint
6399        // was swapped mid-flight. Retrying is how these resolve.
6400        let transient = [
6401            ForwardingError::NoModuleConnection,
6402            ForwardingError::ModuleReloading {
6403                module_id: "m".into(),
6404            },
6405            ForwardingError::StaleModuleEndpoint,
6406            ForwardingError::UnknownReservation {
6407                client_channel: 1,
6408                module_channel: 1,
6409            },
6410            ForwardingError::ConnectionClosing {
6411                connection_id: ConnectionId::new(1),
6412            },
6413            ForwardingError::ClientEgressClosed {
6414                connection_id: ConnectionId::new(1),
6415            },
6416            ForwardingError::ModuleEgressUnavailable {
6417                connection_id: ConnectionId::new(1),
6418            },
6419        ];
6420        for err in transient {
6421            let code = forwarding_error_code(&err);
6422            assert!(
6423                client_retries(code),
6424                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6425            );
6426        }
6427
6428        // Not fixed by retrying. Channel and correlation exhaustion need the
6429        // caller to close routes, and a poisoned lock is a daemon that cannot
6430        // recover at all — the worst thing to advertise as retryable, since every
6431        // client would storm a daemon that will never answer.
6432        let permanent = [
6433            ForwardingError::ConnectionRoleConflict {
6434                connection_id: ConnectionId::new(1),
6435            },
6436            ForwardingError::ClientRouteChannelExhausted {
6437                connection_id: ConnectionId::new(1),
6438            },
6439            ForwardingError::ModuleRouteChannelExhausted {
6440                endpoint: ModuleEndpointId {
6441                    connection_id: ConnectionId::new(1),
6442                    generation: 1,
6443                },
6444            },
6445            ForwardingError::RelayCorrelationExhausted,
6446            ForwardingError::RouteOpenBuild("x".into()),
6447            ForwardingError::Poisoned,
6448        ];
6449        for err in permanent {
6450            let code = forwarding_error_code(&err);
6451            assert!(
6452                !client_retries(code),
6453                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6454            );
6455        }
6456    }
6457
6458    /// The principal is the daemon's answer to "who is calling", and modules
6459    /// branch on it: aft gates bash on it, cerebellum gates browser control,
6460    /// plexus gates connector invocation. So a stamp is an authorization input in
6461    /// another process, not a label — and both possible answers SUCCEED, which is
6462    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
6463    /// first-party capability to something that never proved it; a supervised one
6464    /// stamped `Direct` silently strips a module of capability it is entitled to.
6465    ///
6466    /// Neither shows up in a test that only checks the bind succeeded. Before this
6467    /// test the only coverage was accidental —
6468    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
6469    /// stamped principal on its way past, so narrowing that wire-shape test to its
6470    /// stated subject would have deleted the last assertion on this value. It
6471    /// still asserts the stamp, which is now redundancy rather than the only
6472    /// guard: both fail under the same mutation, and this one names the reason.
6473    /// SCOPE: this handler's supervisor has spawned nothing, so
6474    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
6475    /// is unreachable here. Both assertions below are refusals, and a mutant that
6476    /// refuses everything would satisfy them.
6477    ///
6478    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
6479    /// spawns a supervised consumer, reads its live nonce, and asserts the module
6480    /// observed `principal.kind == "reserved"` carrying that module_id — verified
6481    /// at source rather than assumed, since a citation is a claim about another
6482    /// file and ages like one. Recorded because a harness that structurally
6483    /// cannot reach an arm reports "none" for that arm identically to one that
6484    /// covers it and found nothing.
6485    #[tokio::test]
6486    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6487        let handler = ControlHandler::default();
6488        let frame =
6489            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6490
6491        // Absent consumer_identity is the ordinary case: a human at a terminal, or
6492        // any process holding the connection file. Nothing was proved, so nothing
6493        // may be granted beyond the unattested floor.
6494        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6495        assert_eq!(
6496            stamped,
6497            Principal::Direct,
6498            "a caller that proved nothing must not be stamped as a supervised module"
6499        );
6500
6501        // A claimed module_id with a nonce no supervised child was given is a
6502        // forgery attempt, not a weaker caller: it must be REFUSED rather than
6503        // quietly demoted to Direct, or an impersonation attempt looks identical
6504        // to an ordinary unattested connection.
6505        let forged = handler
6506            .route_open_principal(
6507                &frame,
6508                Some(ConsumerIdentity {
6509                    module_id: "aft".to_string(),
6510                    launch_nonce: "not-a-real-nonce".to_string(),
6511                }),
6512            )
6513            .unwrap();
6514        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6515        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6516    }
6517
6518    /// The test above hands `route_open_principal` an identity it built itself,
6519    /// which proves the stamping rule and nothing about where the identity comes
6520    /// from. The real producer is a wire body, and the two are joined by a serde
6521    /// field name that nothing else asserts.
6522    ///
6523    /// That join fails quietly in one specific way: an unrecognised key is simply
6524    /// absent after parsing, so a renamed or misspelled `consumer_identity`
6525    /// yields `None` and every supervised module silently drops to `Direct`.
6526    /// Capability-wise that is the safe direction, but it surfaces far from its
6527    /// cause — as a module mysteriously refused bash — and it would pass every
6528    /// test that builds its own input.
6529    ///
6530    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
6531    /// would break every client the moment the daemon gains a field, trading a
6532    /// quiet demotion for a hard refusal on additive change. Asserting the join
6533    /// instead means a rename breaks a test here rather than the fleet.
6534    #[test]
6535    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6536        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6537        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6538        let ClientControlRequest::RouteOpen {
6539            consumer_identity, ..
6540        } = parsed
6541        else {
6542            panic!("route.open body must parse as RouteOpen");
6543        };
6544        assert_eq!(
6545            consumer_identity,
6546            Some(ConsumerIdentity {
6547                module_id: "aft".to_string(),
6548                launch_nonce: "n".to_string(),
6549            }),
6550            "the wire field name must reach the value route_open_principal reads"
6551        );
6552    }
6553
6554    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6555        ModuleManifest::builder(module_id, "0.1.0")
6556            .protocol_ver(protocol_ver)
6557            .provides(vec![ProviderRole::ToolProvider {
6558                tools: vec![Tool {
6559                    name: "read".to_string(),
6560                    description: None,
6561                    execution_mode: ExecutionMode::Pure,
6562                    schema: json!({"type": "object"}),
6563                }],
6564                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6565                concurrency: Concurrency::ModuleManaged,
6566                emits_push: true,
6567                sub_supervises: true,
6568            }])
6569            .build()
6570    }
6571
6572    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6573        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6574    }
6575
6576    fn hello_frame_with_control_ops(
6577        module_id: &str,
6578        protocol_ver: u8,
6579        corr: u64,
6580        control_ops: Option<Vec<String>>,
6581    ) -> Frame {
6582        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6583    }
6584
6585    fn hello_frame_with_nonce(
6586        module_id: &str,
6587        protocol_ver: u8,
6588        corr: u64,
6589        launch_nonce: Option<&str>,
6590    ) -> Frame {
6591        hello_frame_full(
6592            module_id,
6593            protocol_ver,
6594            corr,
6595            None,
6596            launch_nonce.map(ToOwned::to_owned),
6597        )
6598    }
6599
6600    fn hello_frame_full(
6601        module_id: &str,
6602        protocol_ver: u8,
6603        corr: u64,
6604        control_ops: Option<Vec<String>>,
6605        launch_nonce: Option<String>,
6606    ) -> Frame {
6607        let body = serde_json::to_vec(&ModuleHelloBody {
6608            manifest: manifest(module_id, protocol_ver),
6609            protocol_ver,
6610            control_ops,
6611            launch_nonce,
6612        })
6613        .unwrap();
6614        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6615    }
6616
6617    fn non_routable_hello_frame_with_control_ops(
6618        module_id: &str,
6619        corr: u64,
6620        control_ops: Option<Vec<String>>,
6621    ) -> Frame {
6622        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6623        manifest.provides.clear();
6624        let body = serde_json::to_vec(&ModuleHelloBody {
6625            manifest,
6626            protocol_ver: PROTOCOL_VERSION,
6627            control_ops,
6628            launch_nonce: None,
6629        })
6630        .unwrap();
6631        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6632    }
6633
6634    fn capability_grammar_hello_frame(
6635        capabilities: Value,
6636        runtime_computed: Option<Value>,
6637        corr: u64,
6638    ) -> Frame {
6639        let mut body = serde_json::to_value(ModuleHelloBody {
6640            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6641            protocol_ver: PROTOCOL_VERSION,
6642            control_ops: None,
6643            launch_nonce: None,
6644        })
6645        .expect("HELLO body serializes");
6646        body["manifest"]["capabilities"] = capabilities;
6647        if let Some(runtime_computed) = runtime_computed {
6648            body["runtime_computed"] = runtime_computed;
6649        }
6650        Frame::build(
6651            FrameType::Hello,
6652            control_flags(),
6653            0,
6654            0,
6655            corr,
6656            serde_json::to_vec(&body).expect("HELLO body reserializes"),
6657        )
6658        .expect("HELLO frame builds")
6659    }
6660
6661    fn channel_request(channel: u16, corr: u64) -> Frame {
6662        Frame::build(
6663            FrameType::Request,
6664            Flags::new(true, Priority::Interactive, false),
6665            channel,
6666            0,
6667            corr,
6668            b"opaque".to_vec(),
6669        )
6670        .unwrap()
6671    }
6672
6673    fn route_ctx(
6674        connection_id: ConnectionId,
6675    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6676        let (tx, rx) = mpsc::channel(8);
6677        (
6678            RouteCtx {
6679                connection_id,
6680                egress: FrameSink::new(tx),
6681            },
6682            rx,
6683        )
6684    }
6685
6686    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6687        serde_json::from_slice(&frame.body).unwrap()
6688    }
6689
6690    /// Register a module over a connection that has a sink and return the
6691    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
6692    /// module's own sink rather than returning it as a reply, so the ack is
6693    /// taken off `rx` here and whatever the test reads next is what followed it.
6694    async fn hello_via_sink(
6695        handler: &ControlHandler,
6696        ctx: &RouteCtx,
6697        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6698        hello: Frame,
6699    ) -> Frame {
6700        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6701        assert!(
6702            replies.is_empty(),
6703            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6704        );
6705        let ack = rx
6706            .try_recv()
6707            .expect("HELLO_ACK is queued on the module sink")
6708            .frame;
6709        assert_eq!(ack.header.ty, FrameType::HelloAck);
6710        ack
6711    }
6712
6713    fn parse_error(frame: &Frame) -> Value {
6714        serde_json::from_slice(&frame.body).unwrap()
6715    }
6716
6717    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6718        serde_json::from_slice(&frame.body).unwrap()
6719    }
6720
6721    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6722        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6723            route_channel,
6724            route_epoch: 0,
6725            kind,
6726        })
6727        .unwrap();
6728        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6729    }
6730
6731    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6732        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6733            module_id: module_id.to_string(),
6734        })
6735        .unwrap();
6736        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6737    }
6738
6739    fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6740        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6741    }
6742
6743    fn route_open_frame_with_consumer_capabilities(
6744        corr: u64,
6745        module_id: &str,
6746        project_root: TestTempDir,
6747        consumer_capabilities: Option<Vec<String>>,
6748    ) -> Frame {
6749        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6750            target: RouteTarget::ToolProvider {
6751                module_id: module_id.to_string(),
6752            },
6753            identity: BindIdentity::new(
6754                project_root.path().to_path_buf(),
6755                "unit".to_string(),
6756                "session".to_string(),
6757            ),
6758            consumer_identity: None,
6759            consumer_capabilities,
6760            role_versions: None,
6761            admission_facts: None,
6762            scope: None,
6763        })
6764        .unwrap();
6765        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6766    }
6767
6768    fn route_open_frame_with_role_versions(
6769        corr: u64,
6770        module_id: &str,
6771        project_root: TestTempDir,
6772        role_versions: Option<BTreeMap<String, String>>,
6773    ) -> Frame {
6774        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6775            target: RouteTarget::ToolProvider {
6776                module_id: module_id.to_string(),
6777            },
6778            identity: BindIdentity::new(
6779                project_root.path().to_path_buf(),
6780                "unit".to_string(),
6781                format!("session-{corr}"),
6782            ),
6783            consumer_identity: None,
6784            consumer_capabilities: None,
6785            role_versions,
6786            admission_facts: None,
6787            scope: None,
6788        })
6789        .unwrap();
6790        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6791    }
6792
6793    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6794        entries
6795            .iter()
6796            .map(|(role, version)| (role.to_string(), version.to_string()))
6797            .collect()
6798    }
6799
6800    fn route_open_frame_with_admission_facts(
6801        corr: u64,
6802        module_id: &str,
6803        project_root: TestTempDir,
6804        consumer_identity: Option<subc_control::ConsumerIdentity>,
6805        facts: Option<Value>,
6806    ) -> Frame {
6807        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6808            target: RouteTarget::ToolProvider {
6809                module_id: module_id.to_string(),
6810            },
6811            identity: BindIdentity::new(
6812                project_root.path().to_path_buf(),
6813                "unit".to_string(),
6814                format!("session-{corr}"),
6815            ),
6816            consumer_identity,
6817            consumer_capabilities: None,
6818            role_versions: None,
6819            admission_facts: facts,
6820            scope: None,
6821        })
6822        .unwrap();
6823        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6824    }
6825
6826    #[derive(Clone, Default)]
6827    struct EventCapture {
6828        events: Arc<Mutex<Vec<CapturedEvent>>>,
6829    }
6830
6831    #[derive(Clone, Debug)]
6832    struct CapturedEvent {
6833        target: String,
6834        level: tracing::Level,
6835        fields: BTreeMap<String, String>,
6836    }
6837
6838    impl EventCapture {
6839        fn events(&self) -> Vec<CapturedEvent> {
6840            self.events.lock().unwrap().clone()
6841        }
6842    }
6843
6844    impl<S> Layer<S> for EventCapture
6845    where
6846        S: Subscriber,
6847    {
6848        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6849            let mut visitor = EventFieldVisitor::default();
6850            event.record(&mut visitor);
6851            self.events.lock().unwrap().push(CapturedEvent {
6852                target: event.metadata().target().to_string(),
6853                level: *event.metadata().level(),
6854                fields: visitor.fields,
6855            });
6856        }
6857    }
6858
6859    #[derive(Default)]
6860    struct EventFieldVisitor {
6861        fields: BTreeMap<String, String>,
6862    }
6863
6864    impl Visit for EventFieldVisitor {
6865        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6866            self.fields
6867                .insert(field.name().to_string(), format!("{value:?}"));
6868        }
6869    }
6870
6871    fn health_response(corr: u64, status: HealthStatus) -> Frame {
6872        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6873            status,
6874            detail: Some("warming".to_string()),
6875            metrics: Some(json!({"queue_depth": 3})),
6876        })
6877        .unwrap();
6878        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6879    }
6880
6881    fn route_bind_ack(corr: u64) -> Frame {
6882        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6883        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6884    }
6885
6886    fn unique_project_root(label: &str) -> TestTempDir {
6887        TestTempDir::new(label)
6888    }
6889
6890    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6891        match parse_route_poll(frame) {
6892            ClientControlResponse::RoutePoll {
6893                status: None,
6894                live: Some(live),
6895                ..
6896            } => assert_eq!(live, expected_live),
6897            other => panic!("unexpected route.poll response: {other:?}"),
6898        }
6899    }
6900
6901    fn bind_liveness_route(
6902        registry: &Registry,
6903        forwarding: &ForwardingTable,
6904        module_id: &str,
6905    ) -> (RouteCtx, u16, u32) {
6906        let module_connection = ConnectionId::new(101);
6907        let client_connection = ConnectionId::new(202);
6908        let registration = registry
6909            .register_with_control_ops(
6910                manifest(module_id, PROTOCOL_VERSION),
6911                PROTOCOL_VERSION,
6912                module_connection,
6913                module_baseline_control_ops(),
6914            )
6915            .unwrap();
6916        let (module_tx, _module_rx) = mpsc::channel(8);
6917        let endpoint = forwarding
6918            .register_module_connection(
6919                module_connection,
6920                module_id.to_string(),
6921                PROTOCOL_VERSION,
6922                manifest_concurrency(&registration.manifest),
6923                FrameSink::new(module_tx),
6924            )
6925            .unwrap();
6926        let (client_ctx, _client_rx) = route_ctx(client_connection);
6927        let pending = forwarding
6928            .begin_route_bind_relay_for_test(
6929                client_connection,
6930                client_ctx.egress.clone(),
6931                1,
6932                module_id,
6933            )
6934            .unwrap();
6935        assert_eq!(pending.endpoint, endpoint);
6936        let route_channel = pending.client_channel;
6937        let route_epoch = pending.client_epoch;
6938        forwarding
6939            .complete_pending_relay(
6940                module_connection,
6941                pending.corr,
6942                RouteBindRelayOutcome::Accepted,
6943            )
6944            .unwrap();
6945        (client_ctx, route_channel, route_epoch)
6946    }
6947
6948    struct FakeProcessLiveness {
6949        live: Option<bool>,
6950    }
6951
6952    impl ModuleProcessLiveness for FakeProcessLiveness {
6953        fn process_live(&self, _module_id: &str) -> Option<bool> {
6954            self.live
6955        }
6956    }
6957
6958    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6959    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6960    {
6961        let registry = Arc::new(Registry::default());
6962        let supervisor_handle = SupervisorHandle::new();
6963        let supervisor = Supervisor::new(
6964            Arc::clone(&registry),
6965            RestartPolicy::new(1, Duration::from_millis(10)),
6966        )
6967        .with_handle(supervisor_handle.clone());
6968        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6969        let module = supervisor
6970            .spawn(ModuleSpec {
6971                module_id: "stderr-tail-wire".to_string(),
6972                program: fake_aft_stub_path(),
6973                args: Vec::new(),
6974                env: vec![
6975                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6976                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6977                ],
6978                reserved: false,
6979                reserved_prefixes: Vec::new(),
6980                protocol: ModuleProtocol::Subc,
6981                overlap: Default::default(),
6982            })
6983            .unwrap();
6984
6985        let deadline = Instant::now() + Duration::from_secs(5);
6986        loop {
6987            let tail = module.stderr_tail(None, None);
6988            if tail
6989                .entries
6990                .iter()
6991                .any(|entry| matches!(entry, TailEntry::ProcessStart))
6992                && tail.entries.iter().any(|entry| {
6993                    matches!(
6994                        entry,
6995                        TailEntry::Line {
6996                            truncated: true,
6997                            ..
6998                        }
6999                    )
7000                })
7001            {
7002                break;
7003            }
7004            assert!(
7005                Instant::now() < deadline,
7006                "module did not produce a truncated line and restart boundary: {tail:?}"
7007            );
7008            sleep(Duration::from_millis(10)).await;
7009        }
7010
7011        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
7012        let request = ClientControlRequest::SupervisorStderrTail {
7013            module_id: "stderr-tail-wire".to_string(),
7014            max_lines: None,
7015            max_bytes: None,
7016        };
7017        let frame = Frame::build(
7018            FrameType::Request,
7019            control_flags(),
7020            0,
7021            0,
7022            1,
7023            serde_json::to_vec(&request).unwrap(),
7024        )
7025        .unwrap();
7026        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7027        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7028        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7029            serde_json::from_slice(&responses[0].body).unwrap()
7030        else {
7031            panic!("expected supervisor.stderr_tail response");
7032        };
7033
7034        assert!(
7035            tail.entries
7036                .iter()
7037                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7038            "the control response lost the restart boundary"
7039        );
7040        let Some(StderrTailEntry::Line {
7041            text,
7042            truncated,
7043            at_ms,
7044        }) = tail.entries.iter().find(|entry| {
7045            matches!(
7046                entry,
7047                StderrTailEntry::Line {
7048                    truncated: true,
7049                    ..
7050                }
7051            )
7052        })
7053        else {
7054            panic!("the control response lost the truncated line");
7055        };
7056        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7057        assert!(*truncated);
7058        assert!(
7059            at_ms.is_some(),
7060            "the control response lost the line's capture time"
7061        );
7062    }
7063
7064    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
7065    /// read done on the worker thread would stall every other task until it
7066    /// finished; the read must run off the worker so this test's own task keeps
7067    /// running while the read is paused.
7068    #[tokio::test(flavor = "current_thread")]
7069    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7070        let dir = TestTempDir::new("terminals-off-worker");
7071        let journal_path = dir.join("terminals.jsonl");
7072        let registry = Arc::new(Registry::default());
7073        let supervisor_handle = SupervisorHandle::new();
7074        let supervisor =
7075            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
7076                .with_handle(supervisor_handle.clone())
7077                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7078        let module = supervisor
7079            .spawn(ModuleSpec {
7080                module_id: "terminal-off-worker".to_string(),
7081                program: fake_aft_stub_path(),
7082                args: Vec::new(),
7083                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7084                reserved: false,
7085                reserved_prefixes: Vec::new(),
7086                protocol: ModuleProtocol::Subc,
7087                overlap: Default::default(),
7088            })
7089            .unwrap();
7090        let deadline = Instant::now() + Duration::from_secs(5);
7091        while module.terminal_history().entries.len() != 2 {
7092            assert!(Instant::now() < deadline, "module did not record two exits");
7093            sleep(Duration::from_millis(10)).await;
7094        }
7095
7096        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7097        let handler =
7098            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
7099        let frame = Frame::build(
7100            FrameType::Request,
7101            control_flags(),
7102            0,
7103            0,
7104            1,
7105            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7106                module_id: "terminal-off-worker".to_string(),
7107            })
7108            .unwrap(),
7109        )
7110        .unwrap();
7111        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7112        let spawned_at = std::time::Instant::now();
7113        let read = tokio::spawn({
7114            let handler = Arc::clone(&handler);
7115            async move { handler.handle_control_frame(&ctx, frame).await }
7116        });
7117        // Waiting for the pause from a blocking thread keeps this task pending,
7118        // so the runtime's single worker is free to run the read task.
7119        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7120            .await
7121            .unwrap()
7122            .expect("the history read reached its pause");
7123        let elapsed = spawned_at.elapsed();
7124        assert!(
7125            elapsed < Duration::from_secs(2) && !read.is_finished(),
7126            "this task could not run while the history read was paused \
7127             (resumed after {elapsed:?}, read finished: {})",
7128            read.is_finished()
7129        );
7130
7131        drop(release);
7132        let responses = read.await.unwrap().unwrap();
7133        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7134        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7135            panic!("expected supervisor.terminals response");
7136        };
7137        assert_eq!(terminals.entries.len(), 2);
7138        assert_eq!(terminals.journal_skipped_lines, 0);
7139        assert_eq!(terminals.journal_read_errors, 0);
7140    }
7141
7142    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7143    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7144        let registry = Arc::new(Registry::default());
7145        let supervisor_handle = SupervisorHandle::new();
7146        let supervisor =
7147            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
7148                .with_handle(supervisor_handle.clone());
7149        let module = supervisor
7150            .spawn(ModuleSpec {
7151                module_id: "terminal-golden".to_string(),
7152                program: fake_aft_stub_path(),
7153                args: Vec::new(),
7154                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7155                reserved: false,
7156                reserved_prefixes: Vec::new(),
7157                protocol: ModuleProtocol::Subc,
7158                overlap: Default::default(),
7159            })
7160            .unwrap();
7161
7162        let deadline = Instant::now() + Duration::from_secs(5);
7163        while module.terminal_history().entries.len() != 2 {
7164            assert!(
7165                Instant::now() < deadline,
7166                "module did not retain two terminal exits: {:?}",
7167                module.terminal_history()
7168            );
7169            sleep(Duration::from_millis(10)).await;
7170        }
7171
7172        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
7173        let request = ClientControlRequest::SupervisorTerminals {
7174            module_id: "terminal-golden".to_string(),
7175        };
7176        let frame = Frame::build(
7177            FrameType::Request,
7178            control_flags(),
7179            0,
7180            0,
7181            1,
7182            serde_json::to_vec(&request).unwrap(),
7183        )
7184        .unwrap();
7185        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7186        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7187        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7188        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7189            panic!("expected supervisor.terminals response");
7190        };
7191        assert_eq!(terminals.entries.len(), 2);
7192        assert_eq!(terminals.dropped, 0);
7193
7194        let mut rendered = serde_json::to_value(response).unwrap();
7195        // Wall-clock fields are the observation contract, but not stable fixture
7196        // bytes; normalize only them after the real handler has shaped the response.
7197        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7198        for (index, entry) in rendered["entries"]
7199            .as_array_mut()
7200            .expect("terminal response entries array")
7201            .iter_mut()
7202            .enumerate()
7203        {
7204            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7205        }
7206
7207        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7208            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7209        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7210        if std::env::var_os("UPDATE_GOLDEN").is_some() {
7211            std::fs::write(&golden_path, &serialized).unwrap();
7212        }
7213        let expected: Value =
7214            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7215        assert_eq!(rendered, expected);
7216    }
7217
7218    #[test]
7219    fn hello_registers_manifest_and_returns_ack() {
7220        let registry = Arc::new(Registry::default());
7221        let handler = ControlHandler::new(Arc::clone(&registry));
7222        let conn = ConnectionId::new(1);
7223
7224        let responses = handler
7225            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7226            .unwrap();
7227
7228        assert_eq!(responses.len(), 1);
7229        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7230        assert_eq!(responses[0].header.channel, 0);
7231        assert_eq!(responses[0].header.corr, 7);
7232        let ack = parse_ack(&responses[0]);
7233        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7234        assert!(ack
7235            .subc_capabilities
7236            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7237        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7238        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7239        assert!(ack
7240            .subc_ops
7241            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7242        assert!(ack
7243            .subc_ops
7244            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7245
7246        let registration = registry.get_module("aft").unwrap().unwrap();
7247        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7248        assert_eq!(registration.state, ChannelState::Active);
7249        assert_eq!(registration.connection_id, conn);
7250        assert_eq!(registration.control_ops, module_baseline_control_ops());
7251    }
7252
7253    #[test]
7254    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7255        let invalid_identifiers = [
7256            ("case_change", "credentials-Provider/v1"),
7257            ("leading_zero", "credentials-provider/v01"),
7258            ("trailing_hyphen", "credentials-provider-/v1"),
7259            ("consecutive_hyphens", "credentials--provider/v1"),
7260            ("uppercase", "Credentials-provider/v1"),
7261            ("missing_v", "credentials-provider/1"),
7262            ("whitespace", "credentials provider/v1"),
7263            ("zero_version", "credentials-provider/v0"),
7264            ("out_of_range_version", "credentials-provider/v4294967296"),
7265            (
7266                "overlength_name",
7267                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7268            ),
7269        ];
7270        let mut cases = invalid_identifiers
7271            .into_iter()
7272            .map(|(name, identifier)| {
7273                (
7274                    format!("identifier_{name}"),
7275                    "capabilities.provides[0]".to_string(),
7276                    identifier.to_string(),
7277                    json!({ "provides": [identifier] }),
7278                    None,
7279                )
7280            })
7281            .collect::<Vec<_>>();
7282        cases.extend([
7283            (
7284                "unknown_need".to_string(),
7285                "capabilities.requires[0].need".to_string(),
7286                "deferred".to_string(),
7287                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7288                None,
7289            ),
7290            (
7291                "duplicate_provides".to_string(),
7292                "capabilities.provides[1]".to_string(),
7293                "credentials-provider/v1".to_string(),
7294                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7295                None,
7296            ),
7297            (
7298                "duplicate_must_never_reach".to_string(),
7299                "capabilities.must_never_reach[1]".to_string(),
7300                "credentials-provider/v1".to_string(),
7301                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7302                None,
7303            ),
7304            (
7305                "duplicate_requires_same_need".to_string(),
7306                "capabilities.requires[1]".to_string(),
7307                "credentials-provider/v1".to_string(),
7308                json!({ "requires": [
7309                    { "capability": "credentials-provider/v1", "need": "required" },
7310                    { "capability": "credentials-provider/v1", "need": "required" }
7311                ] }),
7312                None,
7313            ),
7314            (
7315                "duplicate_requires_conflicting_need".to_string(),
7316                "capabilities.requires[1]".to_string(),
7317                "credentials-provider/v1".to_string(),
7318                json!({ "requires": [
7319                    { "capability": "credentials-provider/v1", "need": "required" },
7320                    { "capability": "credentials-provider/v1", "need": "optional" }
7321                ] }),
7322                None,
7323            ),
7324            (
7325                "capabilities_root_pointer".to_string(),
7326                "runtime_computed[0]".to_string(),
7327                "/capabilities".to_string(),
7328                json!({}),
7329                Some(json!(["/capabilities"])),
7330            ),
7331            (
7332                "capabilities_descendant_pointer".to_string(),
7333                "runtime_computed[0]".to_string(),
7334                "/capabilities/provides".to_string(),
7335                json!({}),
7336                Some(json!(["/capabilities/provides"])),
7337            ),
7338            (
7339                "malformed_pointer_without_leading_slash".to_string(),
7340                "runtime_computed[0]".to_string(),
7341                "capabilities".to_string(),
7342                json!({}),
7343                Some(json!(["capabilities"])),
7344            ),
7345            (
7346                "malformed_pointer_escape".to_string(),
7347                "runtime_computed[0]".to_string(),
7348                "/roles/~2/tools".to_string(),
7349                json!({}),
7350                Some(json!(["/roles/~2/tools"])),
7351            ),
7352            (
7353                "unknown_capabilities_field".to_string(),
7354                "capabilities.future".to_string(),
7355                "<array>".to_string(),
7356                json!({ "future": [] }),
7357                None,
7358            ),
7359        ]);
7360
7361        for (index, (name, field, value, capabilities, runtime_computed)) in
7362            cases.into_iter().enumerate()
7363        {
7364            let registry = Arc::new(Registry::default());
7365            let handler = ControlHandler::new(Arc::clone(&registry));
7366            let response = handler
7367                .handle_control(
7368                    ConnectionId::new((index + 1) as u64),
7369                    capability_grammar_hello_frame(
7370                        capabilities,
7371                        runtime_computed,
7372                        index as u64 + 1,
7373                    ),
7374                )
7375                .expect("invalid HELLO returns a refusal");
7376
7377            assert_eq!(response.len(), 1, "{name} must emit one refusal");
7378            let error = parse_error(&response[0]);
7379            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7380            let message = error["message"]
7381                .as_str()
7382                .expect("error message is a string");
7383            assert!(
7384                message.contains(&field),
7385                "{name}: field missing from {message}"
7386            );
7387            assert!(
7388                message.contains(&value),
7389                "{name}: value missing from {message}"
7390            );
7391            assert_eq!(
7392                registry
7393                    .active_registration_count()
7394                    .expect("registry reads"),
7395                0,
7396                "{name}: refused HELLO must not create a catalog entry"
7397            );
7398        }
7399    }
7400
7401    #[test]
7402    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7403        let registry = Arc::new(Registry::default());
7404        let handler = ControlHandler::new(Arc::clone(&registry));
7405        let capabilities = json!({
7406            "provides": ["credentials-provider/v1"],
7407            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7408            "must_never_reach": ["federation-transport/v1"]
7409        });
7410        let response = handler
7411            .handle_control(
7412                ConnectionId::new(99),
7413                capability_grammar_hello_frame(
7414                    capabilities.clone(),
7415                    Some(json!(["/roles/0/tools"])),
7416                    99,
7417                ),
7418            )
7419            .expect("valid HELLO registers");
7420        assert_eq!(response[0].header.ty, FrameType::HelloAck);
7421
7422        let request = Frame::build(
7423            FrameType::Request,
7424            control_flags(),
7425            0,
7426            0,
7427            100,
7428            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7429                .expect("catalog request serializes"),
7430        )
7431        .expect("catalog request frame builds");
7432        let response = handler
7433            .handle_catalog_list(request, None)
7434            .expect("catalog list succeeds");
7435        let ClientControlResponse::CatalogList { modules, .. } =
7436            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7437        else {
7438            panic!("catalog request must return catalog.list");
7439        };
7440        assert_eq!(modules.len(), 1);
7441        assert_eq!(
7442            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7443            capabilities
7444        );
7445    }
7446
7447    #[test]
7448    fn catalog_list_mirrors_management_operation_description() {
7449        let registry = Arc::new(Registry::default());
7450        let handler = ControlHandler::new(Arc::clone(&registry));
7451        let description = "List managed records and return their identifiers and metadata.";
7452        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7453        manifest.provides = vec![ProviderRole::ManagementSurface {
7454            operations: vec![ManagementOperation {
7455                name: "records.list".to_string(),
7456                kind: ManagementOperationKind::Query,
7457                description: Some(description.to_string()),
7458            }],
7459            config_schema: json!({"type": "object"}),
7460            observability: vec![ObservabilitySurface {
7461                name: "records.stats".to_string(),
7462                kind: ObservabilityKind::Snapshot,
7463            }],
7464            identity_scope: vec![IdentityScope::Project],
7465            concurrency: Concurrency::ModuleManaged,
7466        }];
7467        registry
7468            .register_with_control_ops(
7469                manifest,
7470                PROTOCOL_VERSION,
7471                ConnectionId::new(99),
7472                Vec::new(),
7473            )
7474            .expect("described management manifest registers");
7475
7476        let request = Frame::build(
7477            FrameType::Request,
7478            control_flags(),
7479            0,
7480            0,
7481            100,
7482            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7483                .expect("catalog request serializes"),
7484        )
7485        .expect("catalog request frame builds");
7486        let response = handler
7487            .handle_catalog_list(request, None)
7488            .expect("catalog list succeeds");
7489        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7490        assert_eq!(
7491            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7492            "catalog.list must preserve the declared operation description verbatim"
7493        );
7494    }
7495
7496    #[test]
7497    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7498        let registry = Arc::new(Registry::default());
7499        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7500            [("vault".to_string(), true), ("squatter".to_string(), true)],
7501            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7502        );
7503        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7504        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7505            provides: vec!["credentials-provider/v1".to_string()],
7506            requires: Vec::new(),
7507            must_never_reach: Vec::new(),
7508        });
7509        let frame = Frame::build(
7510            FrameType::Hello,
7511            control_flags(),
7512            0,
7513            0,
7514            77,
7515            serde_json::to_vec(&ModuleHelloBody {
7516                manifest: squatter,
7517                protocol_ver: PROTOCOL_VERSION,
7518                control_ops: None,
7519                launch_nonce: None,
7520            })
7521            .expect("HELLO serializes"),
7522        )
7523        .expect("HELLO frame builds");
7524        let response = handler
7525            .handle_control(ConnectionId::new(77), frame)
7526            .expect("reserved claim receives a typed refusal");
7527        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7528        assert_eq!(
7529            registry
7530                .active_registration_count()
7531                .expect("registry reads"),
7532            0,
7533            "a reserved capability refusal must not leave a catalog entry"
7534        );
7535    }
7536
7537    #[test]
7538    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7539        for settlement in ["timeout", "inconclusive", "drop"] {
7540            let breakers = RouteBindBreakers::default();
7541            let RouteBindAdmission::Admitted {
7542                guard: mut old,
7543                probe: false,
7544            } = breakers.admit("prov")
7545            else {
7546                panic!("ordinary relay admitted")
7547            };
7548            let RouteBindAdmission::Admitted {
7549                guard: mut opener, ..
7550            } = breakers.admit("prov")
7551            else {
7552                panic!("second relay admitted")
7553            };
7554            assert!(
7555                !opener
7556                    .record_timeout(1, Duration::ZERO)
7557                    .unwrap()
7558                    .reopened_after_probe
7559            );
7560            let RouteBindAdmission::Admitted {
7561                guard: mut probe,
7562                probe: true,
7563            } = breakers.admit("prov")
7564            else {
7565                panic!("one cooldown probe admitted")
7566            };
7567            match settlement {
7568                "timeout" => assert!(
7569                    !old.record_timeout(1, Duration::ZERO)
7570                        .unwrap()
7571                        .reopened_after_probe
7572                ),
7573                "inconclusive" => old.record_inconclusive(),
7574                "drop" => drop(old),
7575                _ => unreachable!(),
7576            }
7577            assert!(
7578                matches!(
7579                    breakers.admit("prov"),
7580                    RouteBindAdmission::Refused {
7581                        probe_in_flight: true,
7582                        ..
7583                    }
7584                ),
7585                "{settlement} of a pre-open relay cannot release the real probe"
7586            );
7587            assert!(
7588                probe
7589                    .record_timeout(1, Duration::ZERO)
7590                    .unwrap()
7591                    .reopened_after_probe
7592            );
7593            assert!(matches!(
7594                breakers.admit("prov"),
7595                RouteBindAdmission::Admitted { probe: true, .. }
7596            ));
7597        }
7598        let breakers = RouteBindBreakers::default();
7599        let admit = || match breakers.admit("prov") {
7600            RouteBindAdmission::Admitted { guard, .. } => guard,
7601            _ => panic!("relay admitted"),
7602        };
7603        admit().record_timeout(1, Duration::ZERO);
7604        let mut old_probe = admit();
7605        breakers.reset_for_new_module_connection("prov");
7606        admit().record_timeout(1, Duration::ZERO);
7607        let _new_probe = admit();
7608        old_probe.record_inconclusive();
7609        assert!(matches!(
7610            breakers.admit("prov"),
7611            RouteBindAdmission::Refused {
7612                probe_in_flight: true,
7613                ..
7614            }
7615        ));
7616    }
7617
7618    #[tokio::test]
7619    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7620        for candidate in [false, true] {
7621            let registry = Arc::new(Registry::default());
7622            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7623                [("vault".to_string(), true), ("squatter".to_string(), true)],
7624                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7625            );
7626            let conn = ConnectionId::new(77);
7627            let (ctx, mut rx) = route_ctx(conn);
7628            let initial = capability_manifest("squatter", &[], &[]);
7629            if candidate {
7630                registry
7631                    .register_candidate_with_control_ops(
7632                        initial.clone(),
7633                        PROTOCOL_VERSION,
7634                        conn,
7635                        module_baseline_control_ops(),
7636                    )
7637                    .unwrap();
7638                handler
7639                    .forwarding
7640                    .register_candidate_module_connection(
7641                        conn,
7642                        "squatter".to_string(),
7643                        PROTOCOL_VERSION,
7644                        manifest_concurrency(&initial),
7645                        ctx.egress.clone(),
7646                    )
7647                    .unwrap();
7648            } else {
7649                hello_via_sink(
7650                    &handler,
7651                    &ctx,
7652                    &mut rx,
7653                    hello_frame_with_manifest(initial.clone(), 1),
7654                )
7655                .await;
7656            }
7657            let response = handler
7658                .handle_control_frame(
7659                    &ctx,
7660                    catalog_update_with_capabilities_frame(
7661                        2,
7662                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
7663                            .capabilities
7664                            .unwrap(),
7665                    ),
7666                )
7667                .await
7668                .unwrap();
7669            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7670            assert_eq!(
7671                registry
7672                    .get_module_by_connection(conn)
7673                    .unwrap()
7674                    .unwrap()
7675                    .manifest,
7676                initial
7677            );
7678        }
7679    }
7680
7681    #[test]
7682    fn server_describe_surfaces_required_capability_verdict_fields() {
7683        let registry = Arc::new(Registry::default());
7684        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7685            [
7686                ("consumer".to_string(), true),
7687                ("provider".to_string(), false),
7688            ],
7689            BTreeMap::new(),
7690        );
7691        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7692        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7693            provides: Vec::new(),
7694            requires: vec![subc_protocol::manifest::CapabilityRequirement {
7695                capability: "credentials-provider/v1".to_string(),
7696                need: subc_protocol::manifest::CapabilityNeed::Required,
7697            }],
7698            must_never_reach: Vec::new(),
7699        });
7700        let hello = Frame::build(
7701            FrameType::Hello,
7702            control_flags(),
7703            0,
7704            0,
7705            78,
7706            serde_json::to_vec(&ModuleHelloBody {
7707                manifest: consumer,
7708                protocol_ver: PROTOCOL_VERSION,
7709                control_ops: None,
7710                launch_nonce: None,
7711            })
7712            .expect("HELLO serializes"),
7713        )
7714        .expect("HELLO frame builds");
7715        handler
7716            .handle_control(ConnectionId::new(78), hello)
7717            .expect("consumer registers");
7718        let describe = Frame::build(
7719            FrameType::Request,
7720            control_flags(),
7721            0,
7722            0,
7723            79,
7724            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7725                .expect("request serializes"),
7726        )
7727        .expect("describe frame builds");
7728        let response = handler
7729            .handle_server_describe(describe)
7730            .expect("server.describe succeeds");
7731        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7732        let requirement = &rendered["capability_requirements"][0];
7733        assert_eq!(requirement["consumer"], "consumer");
7734        assert_eq!(requirement["verdict"], "never_provided");
7735        assert_eq!(requirement["episode_seq"], 1);
7736        assert_eq!(requirement["config_satisfiable"], false);
7737        assert_eq!(requirement["runtime_available"], false);
7738        assert!(requirement["detail"]
7739            .as_str()
7740            .expect("detail string")
7741            .contains("credentials-provider/v1"));
7742    }
7743
7744    #[test]
7745    fn catalog_list_omits_capabilities_for_legacy_manifest() {
7746        let registry = Arc::new(Registry::default());
7747        let handler = ControlHandler::new(Arc::clone(&registry));
7748        let hello = handler
7749            .handle_control(
7750                ConnectionId::new(101),
7751                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7752            )
7753            .expect("legacy HELLO registers");
7754        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7755
7756        let request = Frame::build(
7757            FrameType::Request,
7758            control_flags(),
7759            0,
7760            0,
7761            102,
7762            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7763                .expect("catalog request serializes"),
7764        )
7765        .expect("catalog request frame builds");
7766        let response = handler
7767            .handle_catalog_list(request, None)
7768            .expect("catalog list succeeds");
7769        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7770        assert!(
7771            body["modules"][0].get("capabilities").is_none(),
7772            "legacy manifest must retain an absent capabilities field on catalog.list"
7773        );
7774    }
7775
7776    #[test]
7777    fn hello_ack_omits_storage_when_no_storage_config() {
7778        let registry = Arc::new(Registry::default());
7779        let handler = ControlHandler::new(Arc::clone(&registry));
7780        let responses = handler
7781            .handle_control(
7782                ConnectionId::new(1),
7783                hello_frame("aft", PROTOCOL_VERSION, 7),
7784            )
7785            .unwrap();
7786        let ack = parse_ack(&responses[0]);
7787        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7788        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7789    }
7790
7791    #[tokio::test]
7792    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7793        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7794        let registry = Arc::new(Registry::default());
7795        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
7796        let responses = handler
7797            .handle_control(
7798                ConnectionId::new(1),
7799                hello_frame("aft", PROTOCOL_VERSION, 7),
7800            )
7801            .unwrap();
7802        let ack = parse_ack(&responses[0]);
7803        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7804
7805        let described = handler
7806            .handle_control_frame(
7807                &route_ctx(ConnectionId::new(2)).0,
7808                Frame::build(
7809                    FrameType::Request,
7810                    control_flags(),
7811                    0,
7812                    0,
7813                    9,
7814                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7815                )
7816                .unwrap(),
7817            )
7818            .await
7819            .unwrap();
7820        let ClientControlResponse::ServerDescribe { machine_id, .. } =
7821            serde_json::from_slice(&described[0].body).unwrap()
7822        else {
7823            panic!("server.describe answered with another shape");
7824        };
7825        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7826    }
7827
7828    #[test]
7829    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7830        // With a central sqlite storage policy, each registering module gets its
7831        // own resolved descriptor in HELLO_ACK, keyed by its module id.
7832        let registry = Arc::new(Registry::default());
7833        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
7834            crate::daemon_config::StorageConfig::Sqlite {
7835                data_home: std::path::PathBuf::from("/data"),
7836            },
7837        ));
7838
7839        let responses = handler
7840            .handle_control(
7841                ConnectionId::new(1),
7842                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7843            )
7844            .unwrap();
7845        let ack = parse_ack(&responses[0]);
7846        assert_eq!(
7847            ack.storage,
7848            Some(serde_json::json!({
7849                "module_id": "alfonso-routing",
7850                "storage_namespace": "default",
7851                "isolation": { "kind": "module" },
7852                "backend": {
7853                    "backend": "sqlite",
7854                    "path": "/data/cortexkit/alfonso-routing/store.db"
7855                }
7856            })),
7857            "the delivered descriptor is the module's own sqlite store path"
7858        );
7859    }
7860
7861    #[test]
7862    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7863        let registry = Arc::new(Registry::default());
7864        let handler = ControlHandler::new(Arc::clone(&registry));
7865        let conn = ConnectionId::new(1);
7866        let responses = handler
7867            .handle_control(
7868                conn,
7869                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7870            )
7871            .unwrap();
7872        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7873        let registration = registry.get_module("aft").unwrap().unwrap();
7874        assert_eq!(registration.control_ops, module_baseline_control_ops());
7875
7876        let frame =
7877            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7878        assert!(handler
7879            .guard_module_control_op(&frame, "aft", "route.bind")
7880            .unwrap()
7881            .is_none());
7882        let error = handler
7883            .guard_module_control_op(&frame, "aft", "test.synthetic")
7884            .unwrap()
7885            .expect("synthetic ungranted op should be rejected");
7886        assert_eq!(error.header.ty, FrameType::Error);
7887        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7888    }
7889
7890    #[test]
7891    fn hello_control_ops_some_adds_optional_grants() {
7892        let registry = Arc::new(Registry::default());
7893        let handler = ControlHandler::new(Arc::clone(&registry));
7894        handler
7895            .handle_control(
7896                ConnectionId::new(1),
7897                hello_frame_with_control_ops(
7898                    "aft",
7899                    PROTOCOL_VERSION,
7900                    7,
7901                    Some(vec![
7902                        "future.synthetic".to_string(),
7903                        "route.bind".to_string(),
7904                    ]),
7905                ),
7906            )
7907            .unwrap();
7908        let registration = registry.get_module("aft").unwrap().unwrap();
7909        assert_eq!(
7910            registration.control_ops,
7911            vec![
7912                "route.bind".to_string(),
7913                "route.status".to_string(),
7914                "future.synthetic".to_string(),
7915            ]
7916        );
7917        let frame =
7918            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7919        assert!(handler
7920            .guard_module_control_op(&frame, "aft", "future.synthetic")
7921            .unwrap()
7922            .is_none());
7923    }
7924
7925    #[tokio::test]
7926    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7927        let registry = Arc::new(Registry::default());
7928        let forwarding = Arc::new(ForwardingTable::default());
7929        let handler =
7930            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7931        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7932        hello_via_sink(
7933            &handler,
7934            &module_ctx,
7935            &mut module_rx,
7936            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7937        )
7938        .await;
7939
7940        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7941        let responses = handler
7942            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7943            .await
7944            .unwrap();
7945        assert_eq!(responses.len(), 1);
7946        assert_eq!(responses[0].header.ty, FrameType::Error);
7947        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7948        assert!(module_rx.try_recv().is_err());
7949    }
7950
7951    #[tokio::test]
7952    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7953        let registry = Arc::new(Registry::default());
7954        let forwarding = Arc::new(ForwardingTable::default());
7955        let handler =
7956            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7957        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7958        hello_via_sink(
7959            &handler,
7960            &module_ctx,
7961            &mut module_rx,
7962            hello_frame_with_control_ops(
7963                "aft",
7964                PROTOCOL_VERSION,
7965                7,
7966                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7967            ),
7968        )
7969        .await;
7970
7971        let project_root = unique_project_root("demux");
7972        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7973        let route_handler = handler.clone();
7974        let route_task = tokio::spawn(async move {
7975            route_handler
7976                .handle_control_frame(
7977                    &route_client_ctx,
7978                    route_open_frame(100, "aft", project_root),
7979                )
7980                .await
7981                .unwrap()
7982        });
7983        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7984            .await
7985            .unwrap()
7986            .unwrap();
7987        assert!(matches!(
7988            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7989            ModuleControlRequest::RouteBind { .. }
7990        ));
7991
7992        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7993        let health_handler = handler.clone();
7994        let health_task = tokio::spawn(async move {
7995            health_handler
7996                .handle_control_frame(
7997                    &health_client_ctx,
7998                    supervisor_health_probe_frame(101, "aft"),
7999                )
8000                .await
8001                .unwrap()
8002        });
8003        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8004            .await
8005            .unwrap()
8006            .unwrap();
8007        assert_eq!(
8008            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8009            ModuleControlRequest::HealthCheck {}
8010        );
8011
8012        handler
8013            .handle_control_frame(
8014                &module_ctx,
8015                health_response(health_frame.header.corr, HealthStatus::Degraded),
8016            )
8017            .await
8018            .unwrap();
8019        let health_response = health_task.await.unwrap();
8020        assert_eq!(health_response.len(), 1);
8021        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8022            ClientControlResponse::SupervisorHealthProbe {
8023                module_id,
8024                status,
8025                detail,
8026                metrics,
8027            } => {
8028                assert_eq!(module_id, "aft");
8029                assert_eq!(status, HealthStatus::Degraded);
8030                assert_eq!(detail.as_deref(), Some("warming"));
8031                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8032            }
8033            other => panic!("unexpected health response: {other:?}"),
8034        }
8035
8036        handler
8037            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8038            .await
8039            .unwrap();
8040        let route_response = route_task.await.unwrap();
8041        assert!(route_response.is_empty());
8042        let published = route_client_rx.recv().await.unwrap();
8043        assert!(matches!(
8044            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8045            ClientControlResponse::RouteOpen { .. }
8046        ));
8047    }
8048
8049    /// Start one `route.open` on `client_connection` and return its still-running
8050    /// handler task together with the `route.bind` the module received for it.
8051    /// The handler blocks until the module answers, so it has to run as a task
8052    /// while the test drives the module side.
8053    async fn relay_route_open(
8054        handler: &ControlHandler,
8055        client_connection: ConnectionId,
8056        client_egress: &FrameSink,
8057        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8058        corr: u64,
8059        module_id: &str,
8060        project_root_label: &str,
8061    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8062        let ctx = RouteCtx {
8063            connection_id: client_connection,
8064            egress: client_egress.clone(),
8065        };
8066        let handler = handler.clone();
8067        let project_root = unique_project_root(project_root_label);
8068        let module_id = module_id.to_string();
8069        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8070        let task = tokio::spawn(async move {
8071            let _guard = tracing::dispatcher::set_default(&dispatch);
8072            handler
8073                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8074                .await
8075                .unwrap()
8076        });
8077        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8078            .await
8079            .expect("module receives the relayed route.bind")
8080            .expect("module egress is open");
8081        (task, bind.frame)
8082    }
8083
8084    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8085        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8086            ModuleControlRequest::RouteBind {
8087                route_channel,
8088                epoch,
8089                ..
8090            } => (route_channel, epoch),
8091            other => panic!("expected a route.bind request, got {other:?}"),
8092        }
8093    }
8094
8095    fn published_route(frame: &Frame) -> (u16, u32) {
8096        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8097            ClientControlResponse::RouteOpen {
8098                route_channel,
8099                route_epoch,
8100            } => (route_channel, route_epoch),
8101            other => panic!("expected a route.open response, got {other:?}"),
8102        }
8103    }
8104
8105    /// Reproduction of a production outage. A client had `route.open`s in
8106    /// flight to a module and was already marked closing -- its egress had refused a
8107    /// module frame, so the daemon asked its connection to end -- while its sink
8108    /// was still open. When the module acked those binds, the daemon refused to
8109    /// commit a route for a closing client, and that refusal was returned from
8110    /// the MODULE connection's frame handler, where a router error that has no
8111    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
8112    /// the supervisor correctly did not respawn a clean exit, and every seat lost
8113    /// its tools for hours -- one client's teardown took down a connection
8114    /// carrying ~170 other routes.
8115    ///
8116    /// The window is opened here by calling the production path that opens it
8117    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
8118    /// state that matters is "in `closing_connections`, sink still open, relay
8119    /// still pending", and it lasts only from the close request until the
8120    /// connection loop reacts to it; a socket-level test can flood a client into
8121    /// that escalation but cannot pin the module's ack inside the window. Closing
8122    /// the socket instead takes the other path entirely -- connection teardown
8123    /// removes the pending relay under the same lock, so the ack finds nothing.
8124    #[tokio::test]
8125    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8126        let registry = Arc::new(Registry::default());
8127        let forwarding = Arc::new(ForwardingTable::default());
8128        let handler =
8129            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8130
8131        let module_connection = ConnectionId::new(30);
8132        let (module_ctx, mut module_rx) = route_ctx(module_connection);
8133        hello_via_sink(
8134            &handler,
8135            &module_ctx,
8136            &mut module_rx,
8137            hello_frame("aft", PROTOCOL_VERSION, 7),
8138        )
8139        .await;
8140
8141        let dying_client = ConnectionId::new(31);
8142        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8143
8144        // A published route on the dying client. The escalation below only marks
8145        // a connection closing for a route it has already published.
8146        let (first_task, first_bind) = relay_route_open(
8147            &handler,
8148            dying_client,
8149            &dying_ctx.egress,
8150            &mut module_rx,
8151            100,
8152            "aft",
8153            "closing-first",
8154        )
8155        .await;
8156        handler
8157            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8158            .await
8159            .unwrap();
8160        assert!(first_task.await.unwrap().is_empty());
8161        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8162
8163        // A second route.open from the same client, relayed and awaiting its ack.
8164        let (second_task, second_bind) = relay_route_open(
8165            &handler,
8166            dying_client,
8167            &dying_ctx.egress,
8168            &mut module_rx,
8169            101,
8170            "aft",
8171            "closing-second",
8172        )
8173        .await;
8174        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8175
8176        // The window: the client is closing, its sink is still open, and its
8177        // second bind is still pending.
8178        assert!(forwarding
8179            .escalate_client_delivery_failure(
8180                dying_client,
8181                first_channel,
8182                first_epoch,
8183                CloseReason::new(
8184                    "module_to_client_delivery_failed",
8185                    "client egress refused a module frame",
8186                ),
8187                crate::forwarding::UndeliveredFrame {
8188                    module_id: None,
8189                    sink: &dying_ctx.egress,
8190                },
8191            )
8192            .unwrap());
8193        assert!(!dying_ctx.egress.is_closed());
8194
8195        // The frame that used to end the module connection.
8196        let ack = handler
8197            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8198            .await;
8199        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8200        if module_loop_error.is_some() {
8201            // What the server's connection loop does with a router error that has
8202            // no ERROR-frame translation: end the connection, which releases the
8203            // module's registration and every route on it.
8204            handler.cleanup_connection(module_connection).unwrap();
8205        }
8206        // Read the module's next frame before opening the co-tenant's route, so
8207        // the GOODBYE assertion below is about THIS ack and not about later
8208        // traffic. `None` means the module was told nothing.
8209        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8210            .await
8211            .ok()
8212            .flatten();
8213
8214        // 1. The module connection is still registered.
8215        assert!(
8216            registry
8217                .get_module_by_connection(module_connection)
8218                .unwrap()
8219                .is_some(),
8220            "one client's closing connection ended the shared module connection: \
8221             {module_loop_error:?}"
8222        );
8223        // ...and still serving: another client can open and use a route on it.
8224        let cotenant = ConnectionId::new(32);
8225        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8226        let (cotenant_task, cotenant_bind) = relay_route_open(
8227            &handler,
8228            cotenant,
8229            &cotenant_ctx.egress,
8230            &mut module_rx,
8231            102,
8232            "aft",
8233            "closing-cotenant",
8234        )
8235        .await;
8236        handler
8237            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8238            .await
8239            .unwrap();
8240        assert!(cotenant_task.await.unwrap().is_empty());
8241        let (cotenant_channel, cotenant_epoch) =
8242            published_route(&cotenant_rx.recv().await.unwrap());
8243        assert!(matches!(
8244            forwarding
8245                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8246                .unwrap(),
8247            DataRoute::Client(DataRouteState::Bound(_))
8248        ));
8249
8250        // 2. The module was told to drop the binding it created for the route
8251        //    that will never be published.
8252        let goodbye = post_ack_module_frame
8253            .expect("module receives a GOODBYE for the abandoned route channel");
8254        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8255        assert_eq!(goodbye.header.channel, abandoned_channel);
8256        assert_eq!(goodbye.header.epoch, abandoned_epoch);
8257
8258        // 3. The dying client received nothing: no route was ever published to
8259        //    it. Its route.open is answered as unavailable, which the connection
8260        //    loop would write to a socket that is already going away.
8261        assert!(dying_rx.try_recv().is_err());
8262        let second_response = second_task.await.unwrap();
8263        assert_eq!(second_response.len(), 1);
8264        assert_eq!(
8265            parse_error(&second_response[0])["code"],
8266            "target_unavailable"
8267        );
8268    }
8269
8270    /// A handler whose `route.open` parks after queueing the route.bind and
8271    /// before marking it enqueued, with module "aft" registered on connection
8272    /// 40. Returns the two-party barrier that parks and releases it.
8273    async fn route_open_held_after_bind_send() -> (
8274        ControlHandler,
8275        Arc<tokio::sync::Barrier>,
8276        RouteCtx,
8277        mpsc::Receiver<crate::router::OutboundFrame>,
8278    ) {
8279        let hold = Arc::new(tokio::sync::Barrier::new(2));
8280        let handler = ControlHandler::with_forwarding(
8281            Arc::new(Registry::default()),
8282            Arc::new(ForwardingTable::default()),
8283        )
8284        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
8285        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8286        hello_via_sink(
8287            &handler,
8288            &module_ctx,
8289            &mut module_rx,
8290            hello_frame("aft", PROTOCOL_VERSION, 7),
8291        )
8292        .await;
8293        (handler, hold, module_ctx, module_rx)
8294    }
8295
8296    /// A fast module acks the route.bind, and the reader commits the route,
8297    /// before `route.open` has marked the relay enqueued. The daemon used to
8298    /// read the missing relay as an abandoned bind and send the module a
8299    /// GOODBYE for the route it had just committed: the client was told its
8300    /// route was open, the module unbound the channel, and the client's first
8301    /// request was dropped on the module side as unbound.
8302    #[tokio::test]
8303    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
8304        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8305        let client = ConnectionId::new(41);
8306        let (client_ctx, mut client_rx) = route_ctx(client);
8307
8308        let (open_task, bind) = relay_route_open(
8309            &handler,
8310            client,
8311            &client_ctx.egress,
8312            &mut module_rx,
8313            200,
8314            "aft",
8315            "acked-before-mark",
8316        )
8317        .await;
8318        let (module_channel, module_epoch) = route_bind_channel(&bind);
8319        // route.open is now parked between queueing the bind and marking it.
8320        hold.wait().await;
8321
8322        // The module answers inside that window, and the route commits.
8323        assert!(handler
8324            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8325            .await
8326            .unwrap()
8327            .is_empty());
8328        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
8329
8330        hold.wait().await;
8331        assert!(
8332            open_task.await.unwrap().is_empty(),
8333            "route.open answers accepted"
8334        );
8335
8336        // Every frame route.open could have sent is already queued, because it
8337        // has returned; the module must have been sent nothing.
8338        match module_rx.try_recv() {
8339            Err(mpsc::error::TryRecvError::Empty) => {}
8340            Ok(outbound) => panic!(
8341                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
8342                outbound.frame.header.ty,
8343                outbound.frame.header.channel,
8344                outbound.frame.header.epoch
8345            ),
8346            Err(err) => panic!("module egress closed: {err}"),
8347        }
8348
8349        // The client's first request on the route is the next thing the module
8350        // reads, on the binding it just created.
8351        let router = Router::with_control_handler(Arc::new(handler.clone()));
8352        let request = Frame::build(
8353            FrameType::Request,
8354            Flags::new(true, Priority::Interactive, false),
8355            client_channel,
8356            client_epoch,
8357            201,
8358            b"first request".to_vec(),
8359        )
8360        .unwrap();
8361        router
8362            .route_for_connection(&client_ctx, request)
8363            .await
8364            .unwrap();
8365        let delivered = module_rx.try_recv().unwrap().frame;
8366        assert_eq!(delivered.header.ty, FrameType::Request);
8367        assert_eq!(
8368            (delivered.header.channel, delivered.header.epoch),
8369            (module_channel, module_epoch)
8370        );
8371        assert_eq!(delivered.body, b"first request");
8372    }
8373
8374    /// The client goes away while `route.open` is still queueing the bind, so
8375    /// the relay is dropped with no module reply. The teardown cannot send the
8376    /// GOODBYE itself (it could reach the module ahead of the bind), so
8377    /// `route.open` must send it once the bind is queued; otherwise the module
8378    /// keeps a binding the daemon will never commit.
8379    #[tokio::test]
8380    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
8381        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8382        let client = ConnectionId::new(42);
8383        let (client_ctx, _client_rx) = route_ctx(client);
8384
8385        let (open_task, bind) = relay_route_open(
8386            &handler,
8387            client,
8388            &client_ctx.egress,
8389            &mut module_rx,
8390            300,
8391            "aft",
8392            "released-before-mark",
8393        )
8394        .await;
8395        let (module_channel, module_epoch) = route_bind_channel(&bind);
8396        hold.wait().await;
8397
8398        handler.cleanup_connection(client).unwrap();
8399        assert!(
8400            module_rx.try_recv().is_err(),
8401            "client teardown must not GOODBYE a bind route.open is still queueing"
8402        );
8403
8404        hold.wait().await;
8405        let response = open_task.await.unwrap();
8406        assert_eq!(response.len(), 1);
8407        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8408
8409        let goodbye = module_rx
8410            .try_recv()
8411            .expect("module is told to drop the binding the daemon abandoned")
8412            .frame;
8413        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8414        assert_eq!(
8415            (goodbye.header.channel, goodbye.header.epoch),
8416            (module_channel, module_epoch)
8417        );
8418        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8419    }
8420
8421    /// The module acks inside the same window, but the client's egress closed
8422    /// first, so the reader refuses to commit. The ack proves the module holds
8423    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
8424    /// send a second one.
8425    #[tokio::test]
8426    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
8427        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8428        let client = ConnectionId::new(43);
8429        let (client_ctx, client_rx) = route_ctx(client);
8430
8431        let (open_task, bind) = relay_route_open(
8432            &handler,
8433            client,
8434            &client_ctx.egress,
8435            &mut module_rx,
8436            400,
8437            "aft",
8438            "gone-client-before-mark",
8439        )
8440        .await;
8441        let (module_channel, module_epoch) = route_bind_channel(&bind);
8442        hold.wait().await;
8443
8444        drop(client_rx);
8445        assert!(client_ctx.egress.is_closed());
8446        handler
8447            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8448            .await
8449            .unwrap();
8450
8451        hold.wait().await;
8452        let response = open_task.await.unwrap();
8453        assert_eq!(response.len(), 1);
8454        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8455
8456        let goodbye = module_rx
8457            .try_recv()
8458            .expect("module is told to drop the binding it acked")
8459            .frame;
8460        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8461        assert_eq!(
8462            (goodbye.header.channel, goodbye.header.epoch),
8463            (module_channel, module_epoch)
8464        );
8465        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8466        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
8467    }
8468
8469    /// The fence at the module-loop boundary, stated as its own contract: which
8470    /// forwarding failures are allowed to end the module connection that is being
8471    /// served. A `ConnectionClosing` naming some client is about that client, and
8472    /// a module connection is shared; the same error naming the module's own
8473    /// connection is about this connection and must stay fatal, as must failures
8474    /// that are about the forwarding table itself.
8475    #[test]
8476    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8477        let handler = ControlHandler::default();
8478        let module_connection = ConnectionId::new(30);
8479        let client_connection = ConnectionId::new(31);
8480
8481        handler
8482            .refuse_to_end_module_connection_for_a_client(
8483                module_connection,
8484                77,
8485                ForwardingError::ConnectionClosing {
8486                    connection_id: client_connection,
8487                },
8488            )
8489            .expect("a closing client must never end the module connection");
8490
8491        assert!(matches!(
8492            handler.refuse_to_end_module_connection_for_a_client(
8493                module_connection,
8494                78,
8495                ForwardingError::ConnectionClosing {
8496                    connection_id: module_connection,
8497                },
8498            ),
8499            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8500                connection_id
8501            })) if connection_id == module_connection
8502        ));
8503        assert!(matches!(
8504            handler.refuse_to_end_module_connection_for_a_client(
8505                module_connection,
8506                79,
8507                ForwardingError::Poisoned,
8508            ),
8509            Err(RouterError::Forwarding(ForwardingError::Poisoned))
8510        ));
8511        assert!(matches!(
8512            handler.refuse_to_end_module_connection_for_a_client(
8513                module_connection,
8514                80,
8515                ForwardingError::StaleModuleEndpoint,
8516            ),
8517            Err(RouterError::Forwarding(
8518                ForwardingError::StaleModuleEndpoint
8519            ))
8520        ));
8521    }
8522
8523    /// The spawn-attestation guard is what stops a connected module from claiming
8524    /// another module's identity and being stamped `Reserved` for it. Every other
8525    /// test that supplies a consumer_identity supplies a CORRECT one, because a
8526    /// correct one is what the rest of the flow needs -- so the guard's rejection
8527    /// branch was never the subject of an assertion, only its acceptance branch.
8528    ///
8529    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
8530    /// whole subc-core library suite green; only the forwarding integration tests
8531    /// notice, and they notice for unrelated reasons. This test exists so the
8532    /// refusal itself is asserted where the guard lives: it fails if the identity
8533    /// check stops refusing, which is the direction that matters, since a guard
8534    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
8535    #[tokio::test]
8536    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8537        let registry = Arc::new(Registry::default());
8538        let forwarding = Arc::new(ForwardingTable::default());
8539        let supervisor = SupervisorHandle::new();
8540        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8541        let handler =
8542            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8543                .with_supervisor(supervisor);
8544
8545        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8546        hello_via_sink(
8547            &handler,
8548            &target_ctx,
8549            &mut target_rx,
8550            hello_frame("target", PROTOCOL_VERSION, 1),
8551        )
8552        .await;
8553
8554        // A real supervised module id presenting the wrong nonce. This is the
8555        // impersonation case: the attacker knows a privileged module_id, which is
8556        // public, and guesses at the nonce, which is not.
8557        let wrong_nonce = handler
8558            .handle_control_frame(
8559                &route_ctx(ConnectionId::new(91)).0,
8560                route_open_frame_with_admission_facts(
8561                    20,
8562                    "target",
8563                    unique_project_root("admission-facts"),
8564                    Some(subc_control::ConsumerIdentity {
8565                        module_id: "fed".to_string(),
8566                        launch_nonce: "not-the-real-nonce".to_string(),
8567                    }),
8568                    None,
8569                ),
8570            )
8571            .await
8572            .unwrap();
8573        assert_eq!(
8574            parse_error(&wrong_nonce[0])["code"],
8575            "bad_consumer_identity",
8576            "a mismatched launch nonce must be refused, not stamped Reserved"
8577        );
8578
8579        // A module id the supervisor never spawned at all, so no nonce exists to
8580        // compare against. An implementation that treats "no record" as "nothing
8581        // to check" fails open here while passing the case above.
8582        let never_spawned = handler
8583            .handle_control_frame(
8584                &route_ctx(ConnectionId::new(92)).0,
8585                route_open_frame_with_admission_facts(
8586                    21,
8587                    "target",
8588                    unique_project_root("admission-facts"),
8589                    Some(subc_control::ConsumerIdentity {
8590                        module_id: "never-spawned".to_string(),
8591                        launch_nonce: "any-nonce".to_string(),
8592                    }),
8593                    None,
8594                ),
8595            )
8596            .await
8597            .unwrap();
8598        assert_eq!(
8599            parse_error(&never_spawned[0])["code"],
8600            "bad_consumer_identity",
8601            "an unspawned module_id must be refused rather than accepted for lack of a record"
8602        );
8603    }
8604
8605    /// The refusal test above proves the guard says NO. Nothing proved it can say
8606    /// YES, and the difference is not academic: replacing the whole authorization
8607    /// with `false` -- admitting no consumer identity at all, revoking Reserved
8608    /// standing for every supervised module in the fleet -- leaves 110 of the 111
8609    /// library tests GREEN. The one that notices does so by HANGING, because it
8610    /// waits for a bind that can no longer happen.
8611    ///
8612    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
8613    /// or flaky test, invites a RETRY rather than an investigation, and the retry
8614    /// hangs too and gets blamed on the runner. So a total revocation of the
8615    /// daemon's trust grant would have shipped behind a symptom nobody attributes
8616    /// to code.
8617    ///
8618    /// The bias is structural rather than accidental. A REFUSAL looks like a
8619    /// failure someone writes a test for; a GRANT looks like the happy path. Every
8620    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
8621    /// suite for that reason, and this one is the purest case in the daemon.
8622    ///
8623    /// This test asserts the EFFECT rather than the absence of an error: the module
8624    /// receives a RouteBind and it carries `Reserved` naming the attested module.
8625    /// A guard that admitted nobody would produce no bind at all; one that admitted
8626    /// everybody would stamp the wrong principal, which the refusal test catches.
8627    #[tokio::test]
8628    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8629        let registry = Arc::new(Registry::default());
8630        let forwarding = Arc::new(ForwardingTable::default());
8631        let supervisor = SupervisorHandle::new();
8632        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8633        let handler =
8634            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8635                .with_supervisor(supervisor);
8636
8637        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8638        hello_via_sink(
8639            &handler,
8640            &target_ctx,
8641            &mut target_rx,
8642            hello_frame("target", PROTOCOL_VERSION, 1),
8643        )
8644        .await;
8645
8646        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8647        let route_handler = handler.clone();
8648        let route_task = tokio::spawn(async move {
8649            route_handler
8650                .handle_control_frame(
8651                    &client_ctx,
8652                    route_open_frame_with_admission_facts(
8653                        30,
8654                        "target",
8655                        unique_project_root("admission-facts"),
8656                        Some(subc_control::ConsumerIdentity {
8657                            module_id: "fed".to_string(),
8658                            launch_nonce: "fed-nonce".to_string(),
8659                        }),
8660                        None,
8661                    ),
8662                )
8663                .await
8664                .unwrap()
8665        });
8666
8667        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
8668        // the very mutation it exists to catch -- a guard that admits nobody -- no
8669        // bind is ever sent, so it HUNG rather than failing. That reproduces the
8670        // exact defect being fixed: a total revocation detected only as a stalled
8671        // suite, which reads as flakiness and invites a retry. An acceptance test
8672        // that waits for an effect must bound the wait, or a red becomes a hang.
8673        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8674            .await
8675            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8676            .expect("module control channel closed before route.bind");
8677        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8678        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8679            panic!("expected route.bind")
8680        };
8681        assert_eq!(
8682            principal,
8683            Some(Principal::Reserved {
8684                module_id: "fed".to_string()
8685            }),
8686            "a correctly attested consumer must be stamped Reserved for its own id"
8687        );
8688
8689        handler
8690            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8691            .await
8692            .unwrap();
8693        assert!(route_task.await.unwrap().is_empty());
8694        assert!(
8695            matches!(
8696                serde_json::from_slice::<ClientControlResponse>(
8697                    &client_rx.recv().await.unwrap().body
8698                )
8699                .unwrap(),
8700                ClientControlResponse::RouteOpen { .. }
8701            ),
8702            "the route must actually open, not merely avoid an error"
8703        );
8704    }
8705
8706    #[tokio::test(start_paused = true)]
8707    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8708        let registry = Arc::new(Registry::default());
8709        let forwarding = Arc::new(ForwardingTable::default());
8710        let supervisor = SupervisorHandle::new();
8711        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8712        let handler =
8713            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8714                .with_supervisor(supervisor);
8715
8716        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8717        hello_via_sink(
8718            &handler,
8719            &target_ctx,
8720            &mut target_rx,
8721            hello_frame("target", PROTOCOL_VERSION, 1),
8722        )
8723        .await;
8724
8725        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8726        let direct_handler = handler.clone();
8727        let direct_open = tokio::spawn(async move {
8728            direct_handler
8729                .handle_control_frame(
8730                    &direct_ctx,
8731                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
8732                )
8733                .await
8734                .unwrap()
8735        });
8736        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8737            .await
8738            .expect("no direct route.bind within 5s")
8739            .expect("target control channel closed before direct route.bind");
8740        handler
8741            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8742            .await
8743            .unwrap();
8744        assert!(direct_open.await.unwrap().is_empty());
8745        let _ = direct_rx.recv().await.unwrap();
8746
8747        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8748        let reserved_handler = handler.clone();
8749        let reserved_open = tokio::spawn(async move {
8750            reserved_handler
8751                .handle_control_frame(
8752                    &reserved_ctx,
8753                    route_open_frame_with_admission_facts(
8754                        3,
8755                        "target",
8756                        unique_project_root("admission-facts"),
8757                        Some(ConsumerIdentity {
8758                            module_id: "fed".to_string(),
8759                            launch_nonce: "fed-nonce".to_string(),
8760                        }),
8761                        None,
8762                    ),
8763                )
8764                .await
8765                .unwrap()
8766        });
8767        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8768            .await
8769            .expect("no reserved route.bind within 5s")
8770            .expect("target control channel closed before reserved route.bind");
8771        handler
8772            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8773            .await
8774            .unwrap();
8775        assert!(reserved_open.await.unwrap().is_empty());
8776        let _ = reserved_rx.recv().await.unwrap();
8777
8778        forwarding
8779            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8780            .unwrap();
8781        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8782        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8783            module_id: Some("target".to_string()),
8784        })
8785        .unwrap();
8786        let census_frame =
8787            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8788        let response = handler
8789            .handle_control_frame(&census_ctx, census_frame)
8790            .await
8791            .unwrap()
8792            .pop()
8793            .unwrap();
8794        let actual: Value = serde_json::from_slice(&response.body).unwrap();
8795        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8796        assert!(matches!(
8797            decoded,
8798            ClientControlResponse::SupervisorRoutes { .. }
8799        ));
8800        let routes = actual["modules"][0]["routes"].as_array().unwrap();
8801        assert_eq!(routes.len(), 2);
8802        assert!(routes.iter().all(|route| route["draining"] == true));
8803        // The census carries WHY: the reason the drain was begun with, in the
8804        // route.closing vocabulary, on every draining route this drain marked.
8805        assert!(
8806            routes.iter().all(|route| route["drain_reason"] == "reload"),
8807            "draining routes must name the drain's reason: {routes:?}"
8808        );
8809        assert!(routes.iter().any(|route| {
8810            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8811        }));
8812        assert!(routes.iter().any(|route| {
8813            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8814        }));
8815
8816        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8817            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8818        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8819            std::fs::write(
8820                &golden_path,
8821                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8822            )
8823            .unwrap();
8824        }
8825        let expected: Value =
8826            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8827        assert_eq!(actual, expected);
8828    }
8829
8830    async fn query_live_roots(
8831        handler: &ControlHandler,
8832        module_ctx: &RouteCtx,
8833    ) -> ModuleControlResponseToModule {
8834        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8835        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8836        let response = handler
8837            .handle_control_frame(module_ctx, frame)
8838            .await
8839            .unwrap()
8840            .pop()
8841            .unwrap();
8842        serde_json::from_slice(&response.body).unwrap()
8843    }
8844
8845    #[tokio::test(start_paused = true)]
8846    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8847        let registry = Arc::new(Registry::default());
8848        let forwarding = Arc::new(ForwardingTable::default());
8849        let handler = ControlHandler::with_forwarding(registry, forwarding);
8850        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8851        hello_via_sink(
8852            &handler,
8853            &target_ctx,
8854            &mut target_rx,
8855            hello_frame("target", PROTOCOL_VERSION, 1),
8856        )
8857        .await;
8858        let root = unique_project_root("live-roots-known");
8859        let path = ProjectRootId::from_path_allowing_missing(root.path())
8860            .unwrap()
8861            .as_path()
8862            .to_path_buf();
8863        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8864        let open_handler = handler.clone();
8865        let opened = tokio::spawn(async move {
8866            open_handler
8867                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8868                .await
8869                .unwrap()
8870        });
8871        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8872            .await
8873            .unwrap()
8874            .unwrap();
8875        handler
8876            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8877            .await
8878            .unwrap();
8879        assert!(opened.await.unwrap().is_empty());
8880        let _ = client_rx.recv().await.unwrap();
8881
8882        let root = unique_project_root("live-roots-pending");
8883        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8884            .unwrap()
8885            .as_path()
8886            .to_path_buf();
8887        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8888        let open_handler = handler.clone();
8889        let pending = tokio::spawn(async move {
8890            open_handler
8891                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8892                .await
8893                .unwrap()
8894        });
8895        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8896            .await
8897            .unwrap()
8898            .unwrap();
8899        let actual = query_live_roots(&handler, &target_ctx).await;
8900        let ModuleControlResponseToModule::LiveRoots {
8901            roots,
8902            unknown_root_bindings,
8903            total_bindings,
8904        } = actual
8905        else {
8906            panic!("expected live roots")
8907        };
8908        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8909        assert_eq!(unknown_root_bindings, 0);
8910        assert_eq!(
8911            roots.len(),
8912            2,
8913            "root-known arm must retain each canonical root"
8914        );
8915        assert_eq!(
8916            total_bindings,
8917            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8918        );
8919        let counts = roots
8920            .iter()
8921            .map(|root| (root.project_root.clone(), root.bound, root.pending))
8922            .collect::<Vec<_>>();
8923        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8924        expected.sort_by(|a, b| a.0.cmp(&b.0));
8925        assert_eq!(
8926            counts, expected,
8927            "roots must sort by path and count pending separately"
8928        );
8929        handler
8930            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8931            .await
8932            .unwrap();
8933        assert!(pending.await.unwrap().is_empty());
8934    }
8935
8936    #[tokio::test(start_paused = true)]
8937    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8938        let forwarding = Arc::new(ForwardingTable::default());
8939        let handler =
8940            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8941        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8942        hello_via_sink(
8943            &handler,
8944            &target_ctx,
8945            &mut target_rx,
8946            hello_frame("target", PROTOCOL_VERSION, 1),
8947        )
8948        .await;
8949        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8950        let pending = forwarding
8951            .begin_route_bind_relay_for_test(
8952                client_ctx.connection_id,
8953                client_ctx.egress.clone(),
8954                2,
8955                "target",
8956            )
8957            .unwrap();
8958        forwarding
8959            .complete_pending_relay(
8960                target_ctx.connection_id,
8961                pending.corr,
8962                RouteBindRelayOutcome::Accepted,
8963            )
8964            .unwrap();
8965        let actual = query_live_roots(&handler, &target_ctx).await;
8966        let ModuleControlResponseToModule::LiveRoots {
8967            roots,
8968            unknown_root_bindings,
8969            total_bindings,
8970        } = actual
8971        else {
8972            panic!("expected live roots")
8973        };
8974        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8975        assert_eq!(
8976            unknown_root_bindings, 1,
8977            "unknown-root arm must not read as no bindings"
8978        );
8979        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8980        assert_eq!(
8981            total_bindings,
8982            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8983        );
8984    }
8985
8986    /// A module reads HELLO_ACK as its first frame and exits on anything else,
8987    /// so the ack has to be on its outbound queue before the module is
8988    /// routable. The connection loop writes a handler's replies only after the
8989    /// handler returns; this test stops in exactly that gap, runs a real
8990    /// route.open from another connection, and only then writes whatever the
8991    /// HELLO handler returned, the way the loop would. If the ack were still a
8992    /// reply, the route.bind request would reach the module first.
8993    #[tokio::test(start_paused = true)]
8994    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8995        let forwarding = Arc::new(ForwardingTable::default());
8996        let handler =
8997            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8998        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8999        let replies = handler
9000            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9001            .await
9002            .unwrap();
9003        let queued_by_hello = module_rx.len();
9004
9005        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9006        let open_handler = handler.clone();
9007        let open = tokio::spawn(async move {
9008            open_handler
9009                .handle_control_frame(
9010                    &client_ctx,
9011                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9012                )
9013                .await
9014                .unwrap()
9015        });
9016        // Let the route.open run until its route.bind is on the module's queue.
9017        let mut spins = 0;
9018        while module_rx.len() == queued_by_hello {
9019            spins += 1;
9020            assert!(spins < 10_000, "route.open never queued a route.bind");
9021            tokio::task::yield_now().await;
9022        }
9023
9024        // Now the connection loop's half: write the HELLO handler's replies.
9025        for reply in replies {
9026            module_ctx.egress.send(reply).await.unwrap();
9027        }
9028
9029        let first = module_rx.recv().await.unwrap().frame;
9030        assert_eq!(
9031            first.header.ty,
9032            FrameType::HelloAck,
9033            "the first frame a registering module reads must be its HELLO_ACK"
9034        );
9035        assert_eq!(first.header.corr, 7);
9036        let second = module_rx.recv().await.unwrap().frame;
9037        assert_eq!(second.header.ty, FrameType::Request);
9038        assert!(
9039            matches!(
9040                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9041                ModuleControlRequest::RouteBind { .. }
9042            ),
9043            "the route.bind follows the ack"
9044        );
9045        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9046
9047        handler
9048            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9049            .await
9050            .unwrap();
9051        assert!(open.await.unwrap().is_empty());
9052        let _ = client_rx.recv().await.unwrap();
9053    }
9054
9055    #[tokio::test(start_paused = true)]
9056    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9057        let handler = ControlHandler::with_forwarding(
9058            Arc::new(Registry::default()),
9059            Arc::new(ForwardingTable::default()),
9060        );
9061        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9062        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9063        hello_via_sink(
9064            &handler,
9065            &first_ctx,
9066            &mut first_rx,
9067            hello_frame("first", PROTOCOL_VERSION, 1),
9068        )
9069        .await;
9070        hello_via_sink(
9071            &handler,
9072            &second_ctx,
9073            &mut second_rx,
9074            hello_frame("second", PROTOCOL_VERSION, 2),
9075        )
9076        .await;
9077        let root = unique_project_root("second-only");
9078        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9079        let cloned = handler.clone();
9080        let open = tokio::spawn(async move {
9081            cloned
9082                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9083                .await
9084                .unwrap()
9085        });
9086        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9087            .await
9088            .unwrap()
9089            .unwrap();
9090        let first = query_live_roots(&handler, &first_ctx).await;
9091        let second = query_live_roots(&handler, &second_ctx).await;
9092        assert!(
9093            matches!(
9094                first,
9095                ModuleControlResponseToModule::LiveRoots {
9096                    total_bindings: 0,
9097                    ..
9098                }
9099            ),
9100            "cross-module scope must not expose another module's roots"
9101        );
9102        assert!(
9103            matches!(
9104                second,
9105                ModuleControlResponseToModule::LiveRoots {
9106                    total_bindings: 1,
9107                    ..
9108                }
9109            ),
9110            "second module must see its pending route"
9111        );
9112        handler
9113            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9114            .await
9115            .unwrap();
9116        assert!(open.await.unwrap().is_empty());
9117    }
9118
9119    #[tokio::test(start_paused = true)]
9120    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9121        let handler = ControlHandler::with_forwarding(
9122            Arc::new(Registry::default()),
9123            Arc::new(ForwardingTable::default()),
9124        );
9125        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9126        hello_via_sink(
9127            &handler,
9128            &target_ctx,
9129            &mut target_rx,
9130            hello_frame("target", PROTOCOL_VERSION, 1),
9131        )
9132        .await;
9133        let actual = query_live_roots(&handler, &target_ctx).await;
9134        let ModuleControlResponseToModule::LiveRoots {
9135            roots,
9136            unknown_root_bindings,
9137            total_bindings,
9138        } = actual
9139        else {
9140            panic!("expected live roots")
9141        };
9142        assert!(roots.is_empty());
9143        assert_eq!(unknown_root_bindings, 0);
9144        assert_eq!(total_bindings, 0);
9145        assert_eq!(
9146            total_bindings,
9147            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9148        );
9149    }
9150
9151    /// Read the vendored fed corpus rather than hand-building a package.
9152    ///
9153    /// A hand-built object encodes what the test author believed the carrier
9154    /// emits. These vectors are what it actually emits, and one of them exists
9155    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
9156    /// ignores additive unknown fields at the traversal emit terminus."
9157    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9158        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9159            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9160        let text = std::fs::read_to_string(&path)
9161            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9162        let vectors: Vec<(String, Value)> = text
9163            .lines()
9164            .filter(|line| !line.trim().is_empty())
9165            .map(|line| {
9166                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9167                let id = entry["corpus_id"]
9168                    .as_str()
9169                    .expect("every vector carries a corpus_id")
9170                    .to_string();
9171                (id, entry["package"].clone())
9172            })
9173            .collect();
9174        // Pin the count: a corpus that silently shrinks would take its coverage
9175        // with it, and a suite reading N-1 vectors reports the same clean pass
9176        // as one reading N.
9177        assert_eq!(
9178            vectors.len(),
9179            3,
9180            "vendored fed corpus changed size; re-sync from subc-federation"
9181        );
9182
9183        // Pin what makes the corpus DISCRIMINATING, not just present.
9184        //
9185        // The relay test below takes its expected value from the corpus, so the
9186        // corpus supplies the test's power to detect a lossy relay rather than
9187        // its correctness. A relay that dropped unrecognised fields would still
9188        // be caught -- but only by a package carrying fields it does not know.
9189        // Shrink every package to the handful of keys any implementation would
9190        // recognise and the test keeps passing over an input that can no longer
9191        // fail, which is the same clean green as a corpus that shrank away.
9192        //
9193        // So assert the precondition rather than duplicating the packages here:
9194        // at least one vector must carry a field beyond the small common set.
9195        // That is one claim to maintain instead of nine, and it fails loudly if
9196        // a re-sync ever flattens the corpus.
9197        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9198        let richest = vectors
9199            .iter()
9200            .filter_map(|(_, package)| package.as_object())
9201            .map(|object| {
9202                object
9203                    .keys()
9204                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9205                    .count()
9206            })
9207            .max()
9208            .unwrap_or(0);
9209        assert!(
9210            richest >= 2,
9211            "vendored corpus no longer carries a package with unmodelled fields, \
9212             so the relay test can no longer distinguish a verbatim relay from a lossy one"
9213        );
9214
9215        vectors
9216    }
9217
9218    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
9219    ///
9220    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
9221    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
9222    /// three-key object, so a relay that quietly dropped fields it did not
9223    /// recognise would satisfy it. These vectors carry nine keys including ones
9224    /// this crate has no type for, so a typed relay fails here and only here.
9225    #[tokio::test]
9226    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9227        for (corpus_id, package) in fed_admission_facts_vectors() {
9228            let registry = Arc::new(Registry::default());
9229            let forwarding = Arc::new(ForwardingTable::default());
9230            let supervisor = SupervisorHandle::new();
9231            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9232            let handler =
9233                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9234                    .with_supervisor(supervisor)
9235                    .with_admission_facts_config(
9236                        Some("fed".to_string()),
9237                        Some(vec!["target".to_string()]),
9238                    );
9239
9240            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9241            hello_via_sink(
9242                &handler,
9243                &target_ctx,
9244                &mut target_rx,
9245                hello_frame("target", PROTOCOL_VERSION, 1),
9246            )
9247            .await;
9248
9249            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9250            let route_handler = handler.clone();
9251            let expected = package.clone();
9252            let route_task = tokio::spawn(async move {
9253                route_handler
9254                    .handle_control_frame(
9255                        &client_ctx,
9256                        route_open_frame_with_admission_facts(
9257                            20,
9258                            "target",
9259                            unique_project_root("admission-facts"),
9260                            Some(subc_control::ConsumerIdentity {
9261                                module_id: "fed".to_string(),
9262                                launch_nonce: "fed-nonce".to_string(),
9263                            }),
9264                            Some(package),
9265                        ),
9266                    )
9267                    .await
9268                    .unwrap()
9269            });
9270
9271            let bind_frame = target_rx.recv().await.unwrap();
9272            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9273            let ModuleControlRequest::RouteBind {
9274                admission_facts, ..
9275            } = bind
9276            else {
9277                panic!("{corpus_id}: expected route.bind")
9278            };
9279            assert_eq!(
9280                admission_facts,
9281                Some(expected),
9282                "{corpus_id}: relay must not add, drop or reshape any field"
9283            );
9284
9285            handler
9286                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9287                .await
9288                .unwrap();
9289            route_task.await.unwrap();
9290        }
9291    }
9292
9293    #[tokio::test]
9294    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9295        let registry = Arc::new(Registry::default());
9296        let forwarding = Arc::new(ForwardingTable::default());
9297        let supervisor = SupervisorHandle::new();
9298        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9299        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9300        let handler =
9301            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9302                .with_supervisor(supervisor)
9303                .with_admission_facts_config(
9304                    Some("fed".to_string()),
9305                    Some(vec!["target".to_string()]),
9306                );
9307
9308        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9309        hello_via_sink(
9310            &handler,
9311            &target_ctx,
9312            &mut target_rx,
9313            hello_frame("target", PROTOCOL_VERSION, 1),
9314        )
9315        .await;
9316        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9317        hello_via_sink(
9318            &handler,
9319            &other_ctx,
9320            &mut other_rx,
9321            hello_frame("other", PROTOCOL_VERSION, 2),
9322        )
9323        .await;
9324
9325        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9326        let expected_facts = facts.clone();
9327        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9328        let route_handler = handler.clone();
9329        let route_task = tokio::spawn(async move {
9330            route_handler
9331                .handle_control_frame(
9332                    &client_ctx,
9333                    route_open_frame_with_admission_facts(
9334                        10,
9335                        "target",
9336                        unique_project_root("admission-facts"),
9337                        Some(subc_control::ConsumerIdentity {
9338                            module_id: "fed".to_string(),
9339                            launch_nonce: "fed-nonce".to_string(),
9340                        }),
9341                        Some(facts.clone()),
9342                    ),
9343                )
9344                .await
9345                .unwrap()
9346        });
9347        let bind_frame = target_rx.recv().await.unwrap();
9348        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9349        let ModuleControlRequest::RouteBind {
9350            admission_facts, ..
9351        } = bind
9352        else {
9353            panic!("expected route.bind")
9354        };
9355        assert_eq!(admission_facts, Some(expected_facts));
9356        handler
9357            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9358            .await
9359            .unwrap();
9360        assert!(route_task.await.unwrap().is_empty());
9361        assert!(matches!(
9362            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9363                .unwrap(),
9364            ClientControlResponse::RouteOpen { .. }
9365        ));
9366
9367        let direct = handler
9368            .handle_control_frame(
9369                &route_ctx(ConnectionId::new(73)).0,
9370                route_open_frame_with_admission_facts(
9371                    11,
9372                    "target",
9373                    unique_project_root("admission-facts"),
9374                    None,
9375                    Some(json!({"x": 1})),
9376                ),
9377            )
9378            .await
9379            .unwrap();
9380        assert_eq!(
9381            parse_error(&direct[0])["code"],
9382            "admission_facts_not_permitted"
9383        );
9384
9385        let different_reserved = handler
9386            .handle_control_frame(
9387                &route_ctx(ConnectionId::new(77)).0,
9388                route_open_frame_with_admission_facts(
9389                    15,
9390                    "target",
9391                    unique_project_root("admission-facts"),
9392                    Some(subc_control::ConsumerIdentity {
9393                        module_id: "other".to_string(),
9394                        launch_nonce: "other-nonce".to_string(),
9395                    }),
9396                    Some(json!({"x": 1})),
9397                ),
9398            )
9399            .await
9400            .unwrap();
9401        assert_eq!(
9402            parse_error(&different_reserved[0])["code"],
9403            "admission_facts_not_permitted"
9404        );
9405
9406        let other_target = handler
9407            .handle_control_frame(
9408                &route_ctx(ConnectionId::new(74)).0,
9409                route_open_frame_with_admission_facts(
9410                    12,
9411                    "other",
9412                    unique_project_root("admission-facts"),
9413                    Some(subc_control::ConsumerIdentity {
9414                        module_id: "fed".to_string(),
9415                        launch_nonce: "fed-nonce".to_string(),
9416                    }),
9417                    Some(json!({"x": 1})),
9418                ),
9419            )
9420            .await
9421            .unwrap();
9422        assert_eq!(
9423            parse_error(&other_target[0])["code"],
9424            "admission_facts_target_not_allowed"
9425        );
9426
9427        let nonexistent = handler
9428            .handle_control_frame(
9429                &route_ctx(ConnectionId::new(75)).0,
9430                route_open_frame_with_admission_facts(
9431                    13,
9432                    "missing",
9433                    unique_project_root("admission-facts"),
9434                    None,
9435                    Some(json!({"x": 1})),
9436                ),
9437            )
9438            .await
9439            .unwrap();
9440        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9441
9442        let described = handler
9443            .handle_control_frame(
9444                &route_ctx(ConnectionId::new(76)).0,
9445                Frame::build(
9446                    FrameType::Request,
9447                    control_flags(),
9448                    0,
9449                    0,
9450                    14,
9451                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9452                )
9453                .unwrap(),
9454            )
9455            .await
9456            .unwrap();
9457        let ClientControlResponse::ServerDescribe { capabilities, .. } =
9458            serde_json::from_slice(&described[0].body).unwrap()
9459        else {
9460            panic!("expected server.describe response")
9461        };
9462        assert!(capabilities
9463            .iter()
9464            .any(|cap| cap == "admission_facts_relay_v1"));
9465    }
9466
9467    #[tokio::test]
9468    async fn admission_facts_without_configured_carrier_are_rejected() {
9469        let registry = Arc::new(Registry::default());
9470        let forwarding = Arc::new(ForwardingTable::default());
9471        let handler = ControlHandler::with_forwarding(registry, forwarding);
9472        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9473        hello_via_sink(
9474            &handler,
9475            &target_ctx,
9476            &mut target_rx,
9477            hello_frame("target", PROTOCOL_VERSION, 1),
9478        )
9479        .await;
9480
9481        let responses = handler
9482            .handle_control_frame(
9483                &route_ctx(ConnectionId::new(79)).0,
9484                route_open_frame_with_admission_facts(
9485                    16,
9486                    "target",
9487                    unique_project_root("admission-facts"),
9488                    None,
9489                    Some(json!({"x": 1})),
9490                ),
9491            )
9492            .await
9493            .unwrap();
9494        assert_eq!(
9495            parse_error(&responses[0])["code"],
9496            "admission_facts_not_permitted"
9497        );
9498    }
9499
9500    #[tokio::test]
9501    async fn route_open_relays_consumer_capabilities_verbatim() {
9502        let registry = Arc::new(Registry::default());
9503        let forwarding = Arc::new(ForwardingTable::default());
9504        let handler =
9505            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9506        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9507        hello_via_sink(
9508            &handler,
9509            &module_ctx,
9510            &mut module_rx,
9511            hello_frame("aft", PROTOCOL_VERSION, 7),
9512        )
9513        .await;
9514
9515        let expected = vec!["elicitation".to_string(), "roots".to_string()];
9516        let expected_for_request = expected.clone();
9517        let project_root = unique_project_root("consumer-capabilities-present");
9518        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9519        let route_handler = handler.clone();
9520        let route_task = tokio::spawn(async move {
9521            route_handler
9522                .handle_control_frame(
9523                    &client_ctx,
9524                    route_open_frame_with_consumer_capabilities(
9525                        401,
9526                        "aft",
9527                        project_root,
9528                        Some(expected_for_request),
9529                    ),
9530                )
9531                .await
9532                .unwrap()
9533        });
9534        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9535            .await
9536            .unwrap()
9537            .unwrap();
9538        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9539        let ModuleControlRequest::RouteBind {
9540            consumer_capabilities,
9541            ..
9542        } = bind
9543        else {
9544            panic!("expected route.bind request, got {bind:?}");
9545        };
9546        assert_eq!(consumer_capabilities, Some(expected.clone()));
9547
9548        handler
9549            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9550            .await
9551            .unwrap();
9552        let route_response = route_task.await.unwrap();
9553        assert!(route_response.is_empty());
9554        let published = client_rx.recv().await.unwrap();
9555        assert!(matches!(
9556            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9557            ClientControlResponse::RouteOpen { .. }
9558        ));
9559    }
9560
9561    #[tokio::test]
9562    async fn route_open_without_consumer_capabilities_relays_none() {
9563        let registry = Arc::new(Registry::default());
9564        let forwarding = Arc::new(ForwardingTable::default());
9565        let handler =
9566            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9567        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9568        hello_via_sink(
9569            &handler,
9570            &module_ctx,
9571            &mut module_rx,
9572            hello_frame("aft", PROTOCOL_VERSION, 7),
9573        )
9574        .await;
9575
9576        let project_root = unique_project_root("consumer-capabilities-absent");
9577        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9578        let route_handler = handler.clone();
9579        let route_task = tokio::spawn(async move {
9580            route_handler
9581                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9582                .await
9583                .unwrap()
9584        });
9585        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9586            .await
9587            .unwrap()
9588            .unwrap();
9589        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9590        let ModuleControlRequest::RouteBind {
9591            consumer_capabilities,
9592            ..
9593        } = bind
9594        else {
9595            panic!("expected route.bind request, got {bind:?}");
9596        };
9597        assert_eq!(consumer_capabilities, None);
9598
9599        handler
9600            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9601            .await
9602            .unwrap();
9603        let route_response = route_task.await.unwrap();
9604        assert!(route_response.is_empty());
9605        let published = client_rx.recv().await.unwrap();
9606        assert!(matches!(
9607            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9608            ClientControlResponse::RouteOpen { .. }
9609        ));
9610    }
9611
9612    /// Opens a route to a freshly registered `aft` with `sent` as the
9613    /// route.open's role_versions, acks the bind, and returns the role_versions
9614    /// the module's bind carried.
9615    async fn bind_role_versions_for(
9616        sent: Option<BTreeMap<String, String>>,
9617        connection: u64,
9618    ) -> Option<BTreeMap<String, String>> {
9619        let registry = Arc::new(Registry::default());
9620        let forwarding = Arc::new(ForwardingTable::default());
9621        let handler =
9622            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9623        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9624        hello_via_sink(
9625            &handler,
9626            &module_ctx,
9627            &mut module_rx,
9628            hello_frame("aft", PROTOCOL_VERSION, 7),
9629        )
9630        .await;
9631        let project_root = unique_project_root("role-versions");
9632        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9633        let route_handler = handler.clone();
9634        let route_task = tokio::spawn(async move {
9635            route_handler
9636                .handle_control_frame(
9637                    &client_ctx,
9638                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
9639                )
9640                .await
9641                .unwrap()
9642        });
9643        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9644            .await
9645            .expect("a well-formed route.open reaches the module as a bind")
9646            .unwrap();
9647        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9648        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9649            panic!("expected route.bind request, got {bind:?}");
9650        };
9651        handler
9652            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9653            .await
9654            .unwrap();
9655        assert!(route_task.await.unwrap().is_empty());
9656        let published = client_rx.recv().await.unwrap();
9657        assert!(matches!(
9658            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9659            ClientControlResponse::RouteOpen { .. }
9660        ));
9661        role_versions
9662    }
9663
9664    #[tokio::test]
9665    async fn route_open_relays_role_versions_verbatim() {
9666        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9667        assert_eq!(
9668            bind_role_versions_for(Some(sent.clone()), 141).await,
9669            Some(sent)
9670        );
9671    }
9672
9673    /// An empty map declares nothing, so the provider sees no field rather
9674    /// than an empty object it would have to treat as a second "none".
9675    #[tokio::test]
9676    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9677        assert_eq!(bind_role_versions_for(None, 143).await, None);
9678        assert_eq!(
9679            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9680            None
9681        );
9682    }
9683
9684    #[tokio::test]
9685    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9686        let registry = Arc::new(Registry::default());
9687        let forwarding = Arc::new(ForwardingTable::default());
9688        let handler =
9689            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9690        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9691        hello_via_sink(
9692            &handler,
9693            &module_ctx,
9694            &mut module_rx,
9695            hello_frame("aft", PROTOCOL_VERSION, 7),
9696        )
9697        .await;
9698
9699        let nine: BTreeMap<String, String> = (0..9)
9700            .map(|index| (format!("role-{index}"), "v1".to_string()))
9701            .collect();
9702        for (label, malformed) in [
9703            (
9704                "invalid role name",
9705                role_versions(&[("Tool_Provider", "v1")]),
9706            ),
9707            ("invalid version", role_versions(&[("tool-provider", "v0")])),
9708            ("nine entries", nine),
9709        ] {
9710            // A refused open answers at once; one that reached the module
9711            // would wait for its bind ack and trip this timeout.
9712            let responses = tokio::time::timeout(
9713                Duration::from_secs(1),
9714                handler.handle_control_frame(
9715                    &route_ctx(ConnectionId::new(148)).0,
9716                    route_open_frame_with_role_versions(
9717                        404,
9718                        "aft",
9719                        unique_project_root("role-versions-malformed"),
9720                        Some(malformed),
9721                    ),
9722                ),
9723            )
9724            .await
9725            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9726            .unwrap();
9727            assert_eq!(responses.len(), 1, "{label}");
9728            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9729            let error = parse_error(&responses[0]);
9730            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9731            assert_eq!(
9732                error["detail"]["field"], "role_versions",
9733                "{label}: {error}"
9734            );
9735            assert!(
9736                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9737                "{label}: a malformed declaration is terminal"
9738            );
9739            assert!(
9740                module_rx.try_recv().is_err(),
9741                "{label}: the module must never see a bind"
9742            );
9743        }
9744    }
9745
9746    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
9747    /// consumer can tell this daemon forwards the field from one that would
9748    /// drop it.
9749    #[tokio::test]
9750    async fn route_role_versions_capability_is_advertised() {
9751        let handler = ControlHandler::new(Arc::new(Registry::default()));
9752        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9753        let ack = hello_via_sink(
9754            &handler,
9755            &ctx,
9756            &mut rx,
9757            hello_frame("m", PROTOCOL_VERSION, 1),
9758        )
9759        .await;
9760        let ack = parse_ack(&ack);
9761        assert!(
9762            ack.subc_capabilities
9763                .iter()
9764                .any(|c| c == "route-role-versions/v1"),
9765            "{:?}",
9766            ack.subc_capabilities
9767        );
9768
9769        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9770        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9771        let reply = handler
9772            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9773            .await
9774            .unwrap()
9775            .pop()
9776            .unwrap();
9777        let ClientControlResponse::ServerDescribe { capabilities, .. } =
9778            serde_json::from_slice(&reply.body).unwrap()
9779        else {
9780            panic!("not a server.describe reply");
9781        };
9782        assert!(
9783            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9784            "{capabilities:?}"
9785        );
9786    }
9787
9788    #[tokio::test]
9789    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9790        let registry = Arc::new(Registry::default());
9791        let forwarding = Arc::new(ForwardingTable::default());
9792        let handler =
9793            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9794                .with_health_probe_timeout(Duration::from_secs(5));
9795        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9796        hello_via_sink(
9797            &handler,
9798            &module_ctx,
9799            &mut module_rx,
9800            non_routable_hello_frame_with_control_ops(
9801                "mcp",
9802                300,
9803                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9804            ),
9805        )
9806        .await;
9807        assert!(registry
9808            .get_module("mcp")
9809            .unwrap()
9810            .unwrap()
9811            .manifest
9812            .provides
9813            .is_empty());
9814
9815        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9816        let route_response = handler
9817            .handle_control_frame(
9818                &route_client_ctx,
9819                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9820            )
9821            .await
9822            .unwrap();
9823        assert_eq!(route_response[0].header.ty, FrameType::Error);
9824        assert_eq!(
9825            parse_error(&route_response[0])["code"],
9826            "target_unavailable"
9827        );
9828        assert!(parse_error(&route_response[0])["message"]
9829            .as_str()
9830            .unwrap()
9831            .contains("does not provide the requested target"));
9832        assert!(module_rx.try_recv().is_err());
9833
9834        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9835        let health_handler = handler.clone();
9836        let health_task = tokio::spawn(async move {
9837            health_handler
9838                .handle_control_frame(
9839                    &health_client_ctx,
9840                    supervisor_health_probe_frame(302, "mcp"),
9841                )
9842                .await
9843                .unwrap()
9844        });
9845        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9846            .await
9847            .unwrap()
9848            .unwrap();
9849        assert_eq!(
9850            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9851            ModuleControlRequest::HealthCheck {}
9852        );
9853        handler
9854            .handle_control_frame(
9855                &module_ctx,
9856                health_response(health_frame.header.corr, HealthStatus::Ok),
9857            )
9858            .await
9859            .unwrap();
9860        let health_response = health_task.await.unwrap();
9861        assert_eq!(health_response[0].header.ty, FrameType::Response);
9862        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9863            ClientControlResponse::SupervisorHealthProbe {
9864                module_id, status, ..
9865            } => {
9866                assert_eq!(module_id, "mcp");
9867                assert_eq!(status, HealthStatus::Ok);
9868            }
9869            other => panic!("unexpected health response: {other:?}"),
9870        }
9871
9872        // Exercise the forwarding cleanup path directly while leaving the registry
9873        // advertisement in place. If cleanup leaves a stale control sink behind,
9874        // the next probe will enqueue onto it and wait for the long probe timeout
9875        // instead of returning an immediate no-connection error.
9876        forwarding
9877            .cleanup_connection(module_ctx.connection_id)
9878            .unwrap();
9879        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9880        let cleanup_response = tokio::time::timeout(
9881            Duration::from_millis(200),
9882            handler.handle_control_frame(
9883                &cleanup_probe_ctx,
9884                supervisor_health_probe_frame(303, "mcp"),
9885            ),
9886        )
9887        .await
9888        .expect("probe should fail immediately when the control lane is gone")
9889        .unwrap();
9890        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9891        assert_eq!(
9892            parse_error(&cleanup_response[0])["code"],
9893            "target_unavailable"
9894        );
9895        assert!(parse_error(&cleanup_response[0])["message"]
9896            .as_str()
9897            .unwrap()
9898            .contains("no module connection"));
9899
9900        handler
9901            .cleanup_connection(module_ctx.connection_id)
9902            .unwrap();
9903    }
9904
9905    #[tokio::test]
9906    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9907        let registry = Arc::new(Registry::default());
9908        let supervisor_handle = SupervisorHandle::new();
9909        let supervisor =
9910            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9911                .with_handle(supervisor_handle.clone())
9912                .with_connection_file_path(
9913                    std::env::temp_dir()
9914                        .join(format!("subc-route-open-warming-{}", std::process::id())),
9915                );
9916        let module = supervisor
9917            .supervise_configured(
9918                ModuleSpec {
9919                    module_id: "warming".to_string(),
9920                    program: fake_aft_stub_path(),
9921                    args: Vec::new(),
9922                    env: Vec::new(),
9923                    reserved: false,
9924                    reserved_prefixes: Vec::new(),
9925                    protocol: ModuleProtocol::Subc,
9926                    overlap: Default::default(),
9927                },
9928                true,
9929            )
9930            .unwrap();
9931        assert_eq!(module.state().unwrap(), ModuleState::Running);
9932
9933        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9934        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9935        let response = handler
9936            .handle_control_frame(
9937                &ctx,
9938                route_open_frame(304, "warming", unique_project_root("warming")),
9939            )
9940            .await
9941            .unwrap();
9942        module.stop().await.unwrap();
9943
9944        assert_eq!(response[0].header.ty, FrameType::Error);
9945        let error = parse_error(&response[0]);
9946        assert_eq!(error["code"], "module_warming");
9947        assert!(error["message"]
9948            .as_str()
9949            .unwrap()
9950            .contains("state=running, enabled=true, live=false"));
9951    }
9952
9953    #[test]
9954    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9955        let handler = ControlHandler::new(Arc::new(Registry::default()));
9956        let capture = EventCapture::default();
9957        let _subscriber =
9958            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9959        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9960        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9961        let pending = (0..limit).collect::<Vec<_>>();
9962        let response = handler
9963            .route_open_capacity_refusal(
9964                &ctx,
9965                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9966                "busy",
9967                pending.len(),
9968                limit,
9969            )
9970            .unwrap();
9971        assert_eq!(parse_error(&response)["code"], "target_unavailable");
9972        let event = capture
9973            .events()
9974            .into_iter()
9975            .find(|event| {
9976                event.target == "control"
9977                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9978            })
9979            .expect("connection admission refusal event");
9980        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9981        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9982    }
9983
9984    #[test]
9985    fn route_open_target_cap_logs_admission_reason_and_capacity() {
9986        let handler = ControlHandler::new(Arc::new(Registry::default()));
9987        let capture = EventCapture::default();
9988        let _subscriber =
9989            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9990        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9991        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9992        let guards = (0..limit)
9993            .map(|_| {
9994                handler
9995                    .route_bind_concurrency
9996                    .try_admit("busy", limit)
9997                    .unwrap()
9998            })
9999            .collect::<Vec<_>>();
10000        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10001            Err(in_flight) => in_flight,
10002            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10003        };
10004        let response = handler
10005            .route_open_target_capacity_refusal(
10006                &ctx,
10007                &route_open_frame(397, "busy", unique_project_root("target-cap")),
10008                "busy",
10009                in_flight,
10010            )
10011            .unwrap();
10012        assert_eq!(parse_error(&response)["code"], "target_unavailable");
10013        let event = capture
10014            .events()
10015            .into_iter()
10016            .find(|event| {
10017                event.target == "control"
10018                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10019            })
10020            .expect("target admission refusal event");
10021        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10022        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10023        drop(guards);
10024    }
10025
10026    /// One wire code has several senders, so the refusal line names the check
10027    /// that refused. This drives the shared refusal path for ordinary refusals
10028    /// with an unregistered
10029    /// target and requires the branch label on the event.
10030    #[tokio::test]
10031    async fn route_open_refusal_names_the_check_that_refused() {
10032        let handler = ControlHandler::new(Arc::new(Registry::default()));
10033        let capture = EventCapture::default();
10034        let _subscriber =
10035            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10036        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10037        let response = handler
10038            .handle_control_frame(
10039                &ctx,
10040                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10041            )
10042            .await
10043            .unwrap();
10044
10045        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10046        let event = capture
10047            .events()
10048            .into_iter()
10049            .find(|event| {
10050                event.target == "control"
10051                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10052            })
10053            .expect("route.open refusal event");
10054        assert_eq!(
10055            event.fields.get("reason"),
10056            Some(&"\"not_registered\"".to_string())
10057        );
10058    }
10059
10060    #[tokio::test]
10061    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10062        let registry = Arc::new(Registry::default());
10063        let supervisor_handle = SupervisorHandle::new();
10064        let supervisor =
10065            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
10066                .with_handle(supervisor_handle.clone())
10067                .with_connection_file_path(std::env::temp_dir().join(format!(
10068                    "subc-route-open-refusal-info-{}",
10069                    std::process::id()
10070                )));
10071        let module = supervisor
10072            .supervise_configured(
10073                ModuleSpec {
10074                    module_id: "warming".to_string(),
10075                    program: fake_aft_stub_path(),
10076                    args: Vec::new(),
10077                    env: Vec::new(),
10078                    reserved: false,
10079                    reserved_prefixes: Vec::new(),
10080                    protocol: ModuleProtocol::Subc,
10081                    overlap: Default::default(),
10082                },
10083                true,
10084            )
10085            .unwrap();
10086        assert_eq!(module.state().unwrap(), ModuleState::Running);
10087
10088        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10089        assert!(handler
10090            .counters()
10091            .snapshot()
10092            .get("route_open_refused_by_code")
10093            .is_none());
10094        let capture = EventCapture::default();
10095        let _subscriber =
10096            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10097        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
10098        let response = handler
10099            .handle_control_frame(
10100                &ctx,
10101                route_open_frame(394, "warming", unique_project_root("refusal-info")),
10102            )
10103            .await
10104            .unwrap();
10105        module.stop().await.unwrap();
10106
10107        assert_eq!(parse_error(&response[0])["code"], "module_warming");
10108        let event = capture
10109            .events()
10110            .into_iter()
10111            .find(|event| {
10112                event.target == "control"
10113                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
10114            })
10115            .expect("route.open refusal event");
10116        assert_eq!(
10117            event.fields.get("module_id"),
10118            Some(&"\"warming\"".to_string())
10119        );
10120        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10121        assert_eq!(
10122            event.fields.get("reason"),
10123            Some(&"\"supervised_not_registered\"".to_string())
10124        );
10125        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10126        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10127        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10128        assert_eq!(
10129            handler.counters().snapshot()["route_open_refused_by_code"],
10130            json!({ "module_warming": 1 })
10131        );
10132    }
10133
10134    const OUTAGE_START: &str = "route.open refusing module: not serving";
10135    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10136
10137    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10138        capture
10139            .events()
10140            .into_iter()
10141            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10142            .collect()
10143    }
10144
10145    fn supervise_stub(
10146        registry: &Arc<Registry>,
10147        module_id: &str,
10148        enabled: bool,
10149    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10150        let supervisor_handle = SupervisorHandle::new();
10151        let supervisor =
10152            Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10153                .with_handle(supervisor_handle.clone())
10154                .with_connection_file_path(std::env::temp_dir().join(format!(
10155                    "subc-route-outage-{module_id}-{}",
10156                    std::process::id()
10157                )));
10158        let module = supervisor
10159            .supervise_configured(
10160                ModuleSpec {
10161                    module_id: module_id.to_string(),
10162                    program: fake_aft_stub_path(),
10163                    args: Vec::new(),
10164                    env: Vec::new(),
10165                    reserved: false,
10166                    reserved_prefixes: Vec::new(),
10167                    protocol: ModuleProtocol::Subc,
10168                    overlap: Default::default(),
10169                },
10170                enabled,
10171            )
10172            .unwrap();
10173        (supervisor_handle, module)
10174    }
10175
10176    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10177        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10178            module_id: module_id.to_string(),
10179            drain_timeout_ms: Some(50),
10180        })
10181        .unwrap();
10182        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10183    }
10184
10185    /// Two handlers built over one forwarding table must share one outage
10186    /// tracker; separate trackers would each log their own opening line for
10187    /// the same outage.
10188    #[test]
10189    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10190        let registry = Arc::new(Registry::default());
10191        let forwarding = Arc::new(ForwardingTable::default());
10192        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10193        let second = ControlHandler::with_forwarding(registry, forwarding);
10194        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10195    }
10196
10197    /// A client can name any module id it likes. Refusing an unknown one,
10198    /// however often, must not create outage state or outage lines, or the
10199    /// tracker would be a memory sink any client could fill.
10200    #[tokio::test(flavor = "current_thread")]
10201    async fn route_open_unknown_module_refusals_add_no_outage_state() {
10202        let handler = ControlHandler::new(Arc::new(Registry::default()));
10203        let capture = EventCapture::default();
10204        let _subscriber =
10205            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10206        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10207        for corr in 0..8 {
10208            let response = handler
10209                .handle_control_frame(
10210                    &ctx,
10211                    route_open_frame(
10212                        380 + corr,
10213                        &format!("nobody-{corr}"),
10214                        unique_project_root("outage-unknown"),
10215                    ),
10216                )
10217                .await
10218                .unwrap();
10219            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10220        }
10221
10222        assert_eq!(handler.route_outages.tracked_module_count(), 0);
10223        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10224        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10225    }
10226
10227    /// Drives the refusal path end to end: a supervised module that served
10228    /// before and stopped being registered with no instruction to stop is a
10229    /// WARN, and the same module refused after an operator `supervisor.restart`
10230    /// is an INFO.
10231    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10232    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10233        let registry = Arc::new(Registry::default());
10234        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
10235        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10236        let capture = EventCapture::default();
10237        let _subscriber =
10238            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10239        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10240        // The stub never registers, so pretend it served once: otherwise every
10241        // refusal would fall in its startup window.
10242        handler.route_outages.record_accepted("outage-restart");
10243
10244        let response = handler
10245            .handle_control_frame(
10246                &ctx,
10247                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10248            )
10249            .await
10250            .unwrap();
10251        assert_eq!(response[0].header.ty, FrameType::Error);
10252        let starts = outage_lines(&capture, OUTAGE_START);
10253        assert_eq!(starts.len(), 1, "{starts:?}");
10254        assert_eq!(starts[0].level, tracing::Level::WARN);
10255        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10256        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10257        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10258        handler.route_outages.record_accepted("outage-restart");
10259        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10260
10261        let restart = handler
10262            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10263            .await
10264            .unwrap();
10265        assert_eq!(
10266            restart[0].header.ty,
10267            FrameType::Response,
10268            "{:?}",
10269            parse_error(&restart[0])
10270        );
10271        handler
10272            .handle_control_frame(
10273                &ctx,
10274                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10275            )
10276            .await
10277            .unwrap();
10278        module.stop().await.unwrap();
10279
10280        let starts = outage_lines(&capture, OUTAGE_START);
10281        assert_eq!(starts.len(), 2, "{starts:?}");
10282        assert_eq!(starts[1].level, tracing::Level::INFO);
10283        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10284    }
10285
10286    /// A restart refused before it touched the module (here: the module is
10287    /// disabled) must clear its operator mark, so the next real outage is
10288    /// still reported as a warning.
10289    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10290    async fn failed_operator_restart_leaves_no_operator_mark() {
10291        let registry = Arc::new(Registry::default());
10292        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
10293        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10294        let capture = EventCapture::default();
10295        let _subscriber =
10296            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10297        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10298        handler.route_outages.record_accepted("outage-disabled");
10299
10300        let restart = handler
10301            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10302            .await
10303            .unwrap();
10304        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10305        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10306
10307        handler
10308            .handle_control_frame(
10309                &ctx,
10310                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10311            )
10312            .await
10313            .unwrap();
10314        let starts = outage_lines(&capture, OUTAGE_START);
10315        assert_eq!(starts.len(), 1, "{starts:?}");
10316        assert_eq!(starts[0].level, tracing::Level::WARN);
10317    }
10318
10319    #[tokio::test(flavor = "current_thread")]
10320    async fn route_open_unknown_module_escapes_target_module_id() {
10321        let handler = ControlHandler::new(Arc::new(Registry::default()));
10322        let capture = EventCapture::default();
10323        let _subscriber =
10324            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10325        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10326        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10327        let response = handler
10328            .handle_control_frame(
10329                &ctx,
10330                route_open_frame(
10331                    395,
10332                    hostile_module_id,
10333                    unique_project_root("hostile-target-module-id"),
10334                ),
10335            )
10336            .await
10337            .unwrap();
10338
10339        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10340        let event = capture
10341            .events()
10342            .into_iter()
10343            .find(|event| {
10344                event.target == "control"
10345                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10346            })
10347            .expect("route.open unknown-module refusal event");
10348        let logged = event.fields.get("module_id").expect("module_id field");
10349        assert!(!logged.bytes().any(|byte| byte < 0x20));
10350        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10351    }
10352
10353    #[tokio::test(flavor = "current_thread")]
10354    async fn route_open_module_rejection_uses_daemon_counter_key() {
10355        let registry = Arc::new(Registry::default());
10356        let forwarding = Arc::new(ForwardingTable::default());
10357        let handler =
10358            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10359        let module_connection = ConnectionId::new(95);
10360        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10361        hello_via_sink(
10362            &handler,
10363            &module_ctx,
10364            &mut module_rx,
10365            hello_frame("aft", PROTOCOL_VERSION, 395),
10366        )
10367        .await;
10368
10369        let client_connection = ConnectionId::new(96);
10370        let (client_ctx, _client_rx) = route_ctx(client_connection);
10371        let capture = EventCapture::default();
10372        let _subscriber =
10373            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10374        let (route_task, bind) = relay_route_open(
10375            &handler,
10376            client_connection,
10377            &client_ctx.egress,
10378            &mut module_rx,
10379            396,
10380            "aft",
10381            "hostile-module-code",
10382        )
10383        .await;
10384        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10385        let rejection = Frame::build(
10386            FrameType::Error,
10387            control_flags(),
10388            0,
10389            0,
10390            bind.header.corr,
10391            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10392        )
10393        .unwrap();
10394        handler
10395            .handle_control_frame(&module_ctx, rejection)
10396            .await
10397            .unwrap();
10398
10399        let response = route_task.await.unwrap();
10400        assert_eq!(parse_error(&response[0])["code"], hostile_code);
10401        let counters = handler.counters().snapshot();
10402        assert_eq!(
10403            counters["route_open_refused_by_code"],
10404            json!({ "module_rejected": 1 })
10405        );
10406        assert!(counters["route_open_refused_by_code"]
10407            .get(hostile_code)
10408            .is_none());
10409
10410        let event = capture
10411            .events()
10412            .into_iter()
10413            .find(|event| {
10414                event.target == "control"
10415                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10416            })
10417            .expect("route.open module-rejection refusal event");
10418        let logged = event.fields.get("module_code").expect("module_code field");
10419        assert!(!logged.bytes().any(|byte| byte < 0x20));
10420        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10421    }
10422
10423    #[tokio::test]
10424    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10425        let registry = Arc::new(Registry::default());
10426        let supervisor_handle = SupervisorHandle::new();
10427        let missing_program = std::env::temp_dir().join(format!(
10428            "subc-route-open-missing-program-{}",
10429            std::process::id()
10430        ));
10431        let supervisor =
10432            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
10433                .with_handle(supervisor_handle.clone());
10434        let module = supervisor
10435            .supervise_configured(
10436                ModuleSpec {
10437                    module_id: "failed".to_string(),
10438                    program: missing_program,
10439                    args: Vec::new(),
10440                    env: Vec::new(),
10441                    reserved: false,
10442                    reserved_prefixes: Vec::new(),
10443                    protocol: ModuleProtocol::Subc,
10444                    overlap: Default::default(),
10445                },
10446                true,
10447            )
10448            .unwrap();
10449        assert_eq!(module.state().unwrap(), ModuleState::Failed);
10450
10451        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10452        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10453        let response = handler
10454            .handle_control_frame(
10455                &ctx,
10456                route_open_frame(305, "failed", unique_project_root("failed")),
10457            )
10458            .await
10459            .unwrap();
10460
10461        assert_eq!(response[0].header.ty, FrameType::Error);
10462        let error = parse_error(&response[0]);
10463        assert_eq!(error["code"], "target_unavailable");
10464        assert!(error["message"]
10465            .as_str()
10466            .unwrap()
10467            .contains("state=failed, enabled=true, live=false"));
10468    }
10469
10470    #[tokio::test]
10471    async fn route_open_role_mismatch_remains_target_unavailable() {
10472        let registry = Arc::new(Registry::default());
10473        let handler = ControlHandler::new(Arc::clone(&registry));
10474        handler
10475            .handle_control(
10476                ConnectionId::new(41),
10477                non_routable_hello_frame_with_control_ops("health-only", 306, None),
10478            )
10479            .unwrap();
10480
10481        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10482        let response = handler
10483            .handle_control_frame(
10484                &ctx,
10485                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10486            )
10487            .await
10488            .unwrap();
10489
10490        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10491        assert!(parse_error(&response[0])["message"]
10492            .as_str()
10493            .unwrap()
10494            .contains("does not provide the requested target"));
10495    }
10496
10497    #[tokio::test]
10498    async fn route_open_inactive_registration_remains_target_unavailable() {
10499        let registry = Arc::new(Registry::default());
10500        let handler = ControlHandler::new(Arc::clone(&registry));
10501        handler
10502            .handle_control(
10503                ConnectionId::new(43),
10504                hello_frame("inactive", PROTOCOL_VERSION, 308),
10505            )
10506            .unwrap();
10507        assert!(registry
10508            .set_module_state_for_test("inactive", ChannelState::Closed)
10509            .unwrap());
10510
10511        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10512        let response = handler
10513            .handle_control_frame(
10514                &ctx,
10515                route_open_frame(309, "inactive", unique_project_root("inactive")),
10516            )
10517            .await
10518            .unwrap();
10519
10520        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10521        assert!(parse_error(&response[0])["message"]
10522            .as_str()
10523            .unwrap()
10524            .contains("is not active"));
10525    }
10526
10527    #[tokio::test]
10528    async fn late_health_reply_is_recorded_through_the_module_response_path() {
10529        let registry = Arc::new(Registry::default());
10530        let forwarding = Arc::new(ForwardingTable::default());
10531        let supervisor_handle = SupervisorHandle::new();
10532        let supervisor = Supervisor::new(Arc::clone(&registry), crate::RestartPolicy::default())
10533            .with_forwarding(Arc::clone(&forwarding))
10534            .with_handle(supervisor_handle.clone());
10535        let module = supervisor
10536            .supervise_configured(
10537                crate::ModuleSpec {
10538                    module_id: "late-health-response".to_string(),
10539                    program: PathBuf::from("disabled-module"),
10540                    args: Vec::new(),
10541                    env: Vec::new(),
10542                    reserved: false,
10543                    reserved_prefixes: Vec::new(),
10544                    protocol: ModuleProtocol::Subc,
10545                    overlap: Default::default(),
10546                },
10547                false,
10548            )
10549            .unwrap();
10550        let handler =
10551            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10552                .with_supervisor(supervisor_handle);
10553        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
10554        handler
10555            .handle_control_frame(
10556                &module_ctx,
10557                hello_frame_with_control_ops(
10558                    "late-health-response",
10559                    PROTOCOL_VERSION,
10560                    7,
10561                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10562                ),
10563            )
10564            .await
10565            .unwrap();
10566        let probe_started_at = Instant::now() - Duration::from_millis(80);
10567        let pending = forwarding
10568            .begin_health_probe_rpc_for(
10569                "late-health-response",
10570                MODULE_CONTROL_OP_HEALTH_CHECK,
10571                probe_started_at,
10572                Instant::now() - Duration::from_millis(1),
10573            )
10574            .unwrap();
10575        assert!(forwarding
10576            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
10577            .unwrap());
10578
10579        let responses = handler
10580            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
10581            .await
10582            .unwrap();
10583
10584        assert!(responses.is_empty());
10585        let health = module.status().unwrap().health;
10586        assert_eq!(health.late_answer_count, 1);
10587        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10588    }
10589
10590    #[tokio::test]
10591    async fn health_probe_timeout_and_module_death_are_typed() {
10592        let registry = Arc::new(Registry::default());
10593        let forwarding = Arc::new(ForwardingTable::default());
10594        let handler =
10595            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10596                .with_health_probe_timeout(Duration::from_millis(50));
10597        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10598        hello_via_sink(
10599            &handler,
10600            &module_ctx,
10601            &mut module_rx,
10602            hello_frame_with_control_ops(
10603                "aft",
10604                PROTOCOL_VERSION,
10605                7,
10606                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10607            ),
10608        )
10609        .await;
10610
10611        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10612        let responses = handler
10613            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10614            .await
10615            .unwrap();
10616        assert_eq!(responses[0].header.ty, FrameType::Error);
10617        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10618        let _ = module_rx.try_recv();
10619
10620        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10621        let health_handler = handler.clone();
10622        let death_task = tokio::spawn(async move {
10623            health_handler
10624                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10625                .await
10626                .unwrap()
10627        });
10628        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10629            .await
10630            .unwrap()
10631            .unwrap();
10632        handler
10633            .cleanup_connection(module_ctx.connection_id)
10634            .unwrap();
10635        let responses = death_task.await.unwrap();
10636        assert_eq!(responses[0].header.ty, FrameType::Error);
10637        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10638    }
10639
10640    #[test]
10641    fn hello_requires_exact_protocol_version() {
10642        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10643            let registry = Arc::new(Registry::default());
10644            let handler = ControlHandler::new(Arc::clone(&registry));
10645            let responses = handler
10646                .handle_control(
10647                    ConnectionId::new(connection),
10648                    hello_frame("aft", offered, 9),
10649                )
10650                .unwrap();
10651
10652            assert_eq!(responses.len(), 1);
10653            assert_eq!(responses[0].header.ty, FrameType::Error);
10654            let error = parse_error(&responses[0]);
10655            assert_eq!(error["code"], "version_unsupported");
10656            assert!(registry.get_module("aft").unwrap().is_none());
10657            assert_eq!(registry.active_registration_count().unwrap(), 0);
10658        }
10659    }
10660
10661    #[test]
10662    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10663        let registry = Arc::new(Registry::default());
10664        let forwarding = Arc::new(ForwardingTable::default());
10665        let handler =
10666            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10667        let module_connection = ConnectionId::new(301);
10668        let registration = registry
10669            .register_with_control_ops(
10670                manifest("aft-push", PROTOCOL_VERSION),
10671                PROTOCOL_VERSION,
10672                module_connection,
10673                module_baseline_control_ops(),
10674            )
10675            .unwrap();
10676        let (module_tx, _module_rx) = mpsc::channel(8);
10677        let endpoint = forwarding
10678            .register_module_connection(
10679                module_connection,
10680                "aft-push".to_string(),
10681                PROTOCOL_VERSION,
10682                manifest_concurrency(&registration.manifest),
10683                FrameSink::new(module_tx),
10684            )
10685            .unwrap();
10686
10687        // A push op this version does not know is ignored (forward-compat), not errored.
10688        let unknown = Frame::build(
10689            FrameType::Push,
10690            control_flags(),
10691            0,
10692            0,
10693            5,
10694            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10695        )
10696        .unwrap();
10697        let out = handler.handle_status_update(endpoint, unknown).unwrap();
10698        assert!(
10699            out.is_empty(),
10700            "unknown push op must be ignored, got {out:?}"
10701        );
10702
10703        // A malformed body for a KNOWN op is a real error worth surfacing.
10704        let malformed = Frame::build(
10705            FrameType::Push,
10706            control_flags(),
10707            0,
10708            0,
10709            6,
10710            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10711        )
10712        .unwrap();
10713        let out = handler.handle_status_update(endpoint, malformed).unwrap();
10714        assert_eq!(out.len(), 1);
10715        assert_eq!(out[0].header.ty, FrameType::Error);
10716        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10717    }
10718
10719    #[test]
10720    fn hello_rejected_when_connection_already_owns_client_routes() {
10721        let registry = Arc::new(Registry::default());
10722        let forwarding = Arc::new(ForwardingTable::default());
10723        let handler =
10724            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10725        // Commits a client route on connection 202 (bound to a module on conn 101).
10726        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
10727        let client_connection = ConnectionId::new(202);
10728
10729        // That same connection now tries to register as a module: rejected, so one
10730        // connection never holds both client-route and module-endpoint state.
10731        let responses = handler
10732            .handle_control(
10733                client_connection,
10734                hello_frame("aft-second", PROTOCOL_VERSION, 9),
10735            )
10736            .unwrap();
10737        assert_eq!(responses[0].header.ty, FrameType::Error);
10738        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10739        assert!(registry.get_module("aft-second").unwrap().is_none());
10740    }
10741
10742    #[tokio::test]
10743    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
10744        let registry = Arc::new(Registry::default());
10745        let forwarding = Arc::new(ForwardingTable::default());
10746        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
10747        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
10748        hello_via_sink(
10749            &handler,
10750            &module_ctx,
10751            &mut module_rx,
10752            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
10753        )
10754        .await;
10755        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
10756        let pending = forwarding
10757            .begin_route_bind_relay_for_test(
10758                client_ctx.connection_id,
10759                client_ctx.egress.clone(),
10760                2,
10761                "alpha",
10762            )
10763            .unwrap();
10764        forwarding
10765            .complete_pending_relay(
10766                module_ctx.connection_id,
10767                pending.corr,
10768                RouteBindRelayOutcome::Accepted,
10769            )
10770            .unwrap();
10771        client_rx.try_recv().unwrap();
10772        for module_id in ["beta", "alpha"] {
10773            let replies = handler
10774                .handle_control_frame(
10775                    &module_ctx,
10776                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
10777                )
10778                .await
10779                .unwrap();
10780            assert_eq!(replies.len(), 1, "second HELLO must be refused");
10781            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
10782        }
10783        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
10784        assert!(registry.get_module("beta").unwrap().is_none());
10785        assert!(matches!(
10786            forwarding
10787                .lookup_data_route(
10788                    client_ctx.connection_id,
10789                    pending.client_channel,
10790                    pending.client_epoch,
10791                )
10792                .unwrap(),
10793            DataRoute::Client(DataRouteState::Bound(_))
10794        ));
10795        assert!(handler
10796            .hello_launch_nonces
10797            .lock()
10798            .unwrap()
10799            .presented(module_ctx.connection_id, Some("alpha-nonce")));
10800        assert!(module_rx.try_recv().is_err());
10801    }
10802
10803    #[test]
10804    fn reserved_module_hello_requires_matching_launch_nonce() {
10805        let registry = Arc::new(Registry::default());
10806        let supervisor = SupervisorHandle::new();
10807        // The supervisor recorded the nonce it injected when it spawned the reserved
10808        // module; the HELLO verifier checks against the same shared handle.
10809        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10810        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10811
10812        // A HELLO with NO nonce is rejected.
10813        let no_nonce = handler
10814            .handle_control(
10815                ConnectionId::new(1),
10816                hello_frame("vault", PROTOCOL_VERSION, 1),
10817            )
10818            .unwrap();
10819        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10820        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10821        assert!(registry.get_module("vault").unwrap().is_none());
10822
10823        // A HELLO with the WRONG nonce is rejected.
10824        let wrong = handler
10825            .handle_control(
10826                ConnectionId::new(2),
10827                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10828            )
10829            .unwrap();
10830        assert_eq!(wrong[0].header.ty, FrameType::Error);
10831        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10832        assert!(registry.get_module("vault").unwrap().is_none());
10833
10834        // A HELLO with the CORRECT nonce registers.
10835        let ok = handler
10836            .handle_control(
10837                ConnectionId::new(3),
10838                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10839            )
10840            .unwrap();
10841        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10842        assert!(registry.get_module("vault").unwrap().is_some());
10843    }
10844
10845    #[test]
10846    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10847        let registry = Arc::new(Registry::default());
10848        let supervisor = SupervisorHandle::new();
10849        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10850        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10851        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10852
10853        let squat = handler
10854            .handle_control(
10855                ConnectionId::new(1),
10856                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10857            )
10858            .unwrap();
10859        assert_eq!(squat[0].header.ty, FrameType::Error);
10860        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10861        assert!(parse_error(&squat[0])["message"]
10862            .as_str()
10863            .unwrap()
10864            .contains("fed:"));
10865
10866        let accepted_peer = handler
10867            .handle_control(
10868                ConnectionId::new(2),
10869                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10870            )
10871            .unwrap();
10872        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10873
10874        let accepted_short = handler
10875            .handle_control(
10876                ConnectionId::new(3),
10877                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10878            )
10879            .unwrap();
10880        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10881
10882        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10883            let response = handler
10884                .handle_control(
10885                    ConnectionId::new(conn),
10886                    hello_frame(module_id, PROTOCOL_VERSION, conn),
10887                )
10888                .unwrap();
10889            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10890        }
10891    }
10892
10893    #[test]
10894    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10895        let registry = Arc::new(Registry::default());
10896        let supervisor = SupervisorHandle::new();
10897        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10898        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10899        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10900        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10901
10902        let owner_nonce = handler
10903            .handle_control(
10904                ConnectionId::new(1),
10905                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10906            )
10907            .unwrap();
10908        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10909        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10910        assert!(registry.get_module("fed:special").unwrap().is_none());
10911
10912        let exact_nonce = handler
10913            .handle_control(
10914                ConnectionId::new(2),
10915                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10916            )
10917            .unwrap();
10918        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10919        assert!(registry.get_module("fed:special").unwrap().is_some());
10920    }
10921
10922    #[test]
10923    fn non_reserved_module_ignores_launch_nonce() {
10924        let registry = Arc::new(Registry::default());
10925        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
10926        // registration succeeds whether a spawned process echoes a nonce or not.
10927        let handler = ControlHandler::new(Arc::clone(&registry));
10928        let no_nonce = handler
10929            .handle_control(
10930                ConnectionId::new(1),
10931                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10932            )
10933            .unwrap();
10934        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10935        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10936
10937        let echoed_nonce = handler
10938            .handle_control(
10939                ConnectionId::new(2),
10940                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10941            )
10942            .unwrap();
10943        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10944        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10945    }
10946
10947    #[test]
10948    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10949        let handler = ControlHandler::default();
10950        let conn = ConnectionId::new(1);
10951        let malformed = Frame::build(
10952            FrameType::Hello,
10953            control_flags(),
10954            0,
10955            0,
10956            3,
10957            b"{not json".to_vec(),
10958        )
10959        .unwrap();
10960
10961        let error = handler.handle_control(conn, malformed).unwrap();
10962        assert_eq!(error[0].header.ty, FrameType::Error);
10963        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10964
10965        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10966        let pong = handler.handle_control(conn, ping).unwrap();
10967        assert_eq!(pong[0].header.ty, FrameType::Pong);
10968        assert_eq!(pong[0].header.corr, 4);
10969    }
10970
10971    #[test]
10972    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10973        let registry = Arc::new(Registry::default());
10974        let handler = ControlHandler::new(Arc::clone(&registry));
10975
10976        handler
10977            .handle_control(
10978                ConnectionId::new(1),
10979                hello_frame("aft", PROTOCOL_VERSION, 1),
10980            )
10981            .unwrap();
10982        let duplicate = handler
10983            .handle_control(
10984                ConnectionId::new(2),
10985                hello_frame("aft", PROTOCOL_VERSION, 2),
10986            )
10987            .unwrap();
10988
10989        assert_eq!(duplicate[0].header.ty, FrameType::Error);
10990        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10991        let registration = registry.get_module("aft").unwrap().unwrap();
10992        assert_eq!(registration.connection_id, ConnectionId::new(1));
10993    }
10994
10995    #[test]
10996    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10997        let registry = Arc::new(Registry::default());
10998        let forwarding = Arc::new(ForwardingTable::default());
10999        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11000        let handler =
11001            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11002                .with_process_liveness(process_liveness);
11003        let (ctx, route_channel, route_epoch) =
11004            bind_liveness_route(&registry, &forwarding, "aft-dead");
11005        let responses = handler
11006            .handle_route_poll(
11007                &ctx,
11008                route_poll_frame(41, PollKind::Liveness, route_channel),
11009                route_channel,
11010                route_epoch,
11011                PollKind::Liveness,
11012            )
11013            .unwrap();
11014
11015        assert_eq!(responses.len(), 1);
11016        assert_eq!(responses[0].header.ty, FrameType::Response);
11017        assert_route_poll_liveness(&responses[0], false);
11018    }
11019
11020    #[test]
11021    fn liveness_poll_without_process_source_uses_bound_route() {
11022        let registry = Arc::new(Registry::default());
11023        let forwarding = Arc::new(ForwardingTable::default());
11024        let handler =
11025            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11026        let (ctx, route_channel, route_epoch) =
11027            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
11028        let responses = handler
11029            .handle_route_poll(
11030                &ctx,
11031                route_poll_frame(42, PollKind::Liveness, route_channel),
11032                route_channel,
11033                route_epoch,
11034                PollKind::Liveness,
11035            )
11036            .unwrap();
11037
11038        assert_route_poll_liveness(&responses[0], true);
11039    }
11040
11041    #[test]
11042    fn liveness_poll_untracked_process_source_uses_bound_route() {
11043        let registry = Arc::new(Registry::default());
11044        let forwarding = Arc::new(ForwardingTable::default());
11045        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11046        let handler =
11047            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11048                .with_process_liveness(process_liveness);
11049        let (ctx, route_channel, route_epoch) =
11050            bind_liveness_route(&registry, &forwarding, "aft-untracked");
11051        let responses = handler
11052            .handle_route_poll(
11053                &ctx,
11054                route_poll_frame(43, PollKind::Liveness, route_channel),
11055                route_channel,
11056                route_epoch,
11057                PollKind::Liveness,
11058            )
11059            .unwrap();
11060
11061        assert_route_poll_liveness(&responses[0], true);
11062    }
11063
11064    #[tokio::test]
11065    async fn unknown_op_returns_unknown_control_op() {
11066        let handler = ControlHandler::default();
11067        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
11068        let request = Frame::build(
11069            FrameType::Request,
11070            control_flags(),
11071            0,
11072            0,
11073            55,
11074            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
11075        )
11076        .unwrap();
11077
11078        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11079
11080        assert_eq!(response.len(), 1);
11081        assert_eq!(response[0].header.ty, FrameType::Error);
11082        assert_eq!(response[0].header.corr, 55);
11083        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
11084    }
11085
11086    #[tokio::test]
11087    async fn supervisor_provenance_rejects_unknown_exact_module() {
11088        let handler = ControlHandler::default();
11089        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
11090        let request = Frame::build(
11091            FrameType::Request,
11092            control_flags(),
11093            0,
11094            0,
11095            57,
11096            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
11097        )
11098        .unwrap();
11099
11100        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11101
11102        assert_eq!(response.len(), 1);
11103        assert_eq!(response[0].header.ty, FrameType::Error);
11104        assert_eq!(response[0].header.corr, 57);
11105        let error = parse_error(&response[0]);
11106        assert_eq!(error["code"], "unknown_module");
11107        assert_eq!(error["message"], "module_id 'missing' is not supervised");
11108    }
11109
11110    #[test]
11111    fn provenance_probe_override_keeps_handler_tests_deterministic() {
11112        let expected = subc_control::RunningImageAgreement::Unavailable {
11113            reason: subc_control::RunningImageUnavailableReason::HashFailed,
11114        };
11115        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
11116        assert_eq!(handler.provenance_probe_override, Some(expected));
11117    }
11118
11119    #[test]
11120    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11121        let verdict = reload_verdict(
11122            std::path::Path::new("/bin/new"),
11123            Some(std::path::Path::new("/bin/old")),
11124            subc_control::RunningImageAgreement::Unavailable {
11125                reason: subc_control::RunningImageUnavailableReason::HashFailed,
11126            },
11127        );
11128        assert!(matches!(
11129            verdict.path,
11130            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11131                if configured == std::path::Path::new("/bin/new")
11132                    && spawned_from == std::path::Path::new("/bin/old")
11133        ));
11134    }
11135
11136    #[test]
11137    fn reload_verdict_detects_replaced_image_at_same_path() {
11138        let image = subc_control::RunningImageAgreement::Mismatch {
11139            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11140                digest: "old".into(),
11141            },
11142            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11143                digest: "new".into(),
11144            },
11145        };
11146        let verdict = reload_verdict(
11147            std::path::Path::new("/bin/same"),
11148            Some(std::path::Path::new("/bin/same")),
11149            image.clone(),
11150        );
11151        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11152        assert_eq!(verdict.image, image);
11153    }
11154
11155    #[test]
11156    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11157        let image = subc_control::RunningImageAgreement::Unavailable {
11158            reason: subc_control::RunningImageUnavailableReason::NotRunning,
11159        };
11160        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11161        assert_eq!(
11162            verdict.path,
11163            subc_control::ReloadPathAgreement::Unavailable {
11164                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11165            }
11166        );
11167        assert_eq!(verdict.image, image);
11168
11169        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11170            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11171        };
11172        let verdict = reload_verdict(
11173            std::path::Path::new("/bin/same"),
11174            Some(std::path::Path::new("/bin/same")),
11175            unconfirmed.clone(),
11176        );
11177        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11178        assert_eq!(verdict.image, unconfirmed);
11179    }
11180
11181    #[test]
11182    fn reload_verdict_preserves_each_image_unavailability_reason() {
11183        use subc_control::RunningImageUnavailableReason as Reason;
11184
11185        for reason in [
11186            Reason::NotRunning,
11187            Reason::UnsupportedPlatform,
11188            Reason::RunningExecutableUnreadable,
11189            Reason::SpawnedPathUnreadable,
11190            Reason::HashFailed,
11191            Reason::ProcessIdentityUnconfirmed,
11192            Reason::Unknown("future_probe_reason".to_string()),
11193        ] {
11194            let image = subc_control::RunningImageAgreement::Unavailable {
11195                reason: reason.clone(),
11196            };
11197            let verdict = reload_verdict(
11198                std::path::Path::new("/bin/same"),
11199                Some(std::path::Path::new("/bin/same")),
11200                image.clone(),
11201            );
11202            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11203            assert_eq!(verdict.image, image, "{reason:?}");
11204        }
11205    }
11206
11207    #[tokio::test]
11208    async fn malformed_control_bodies_return_invalid_control_body() {
11209        let handler = ControlHandler::default();
11210        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11211
11212        for (corr, body) in [
11213            (56, br#"{"route_channel":1}"#.as_slice()),
11214            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11215            (
11216                58,
11217                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11218            ),
11219        ] {
11220            let request = Frame::build(
11221                FrameType::Request,
11222                control_flags(),
11223                0,
11224                0,
11225                corr,
11226                body.to_vec(),
11227            )
11228            .unwrap();
11229            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11230
11231            assert_eq!(response.len(), 1);
11232            assert_eq!(response[0].header.ty, FrameType::Error);
11233            assert_eq!(response[0].header.corr, corr);
11234            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11235        }
11236    }
11237
11238    #[tokio::test]
11239    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11240        let registry = Arc::new(Registry::default());
11241        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
11242        let router = Router::with_control_handler(Arc::clone(&control));
11243        let connection = router.begin_connection();
11244        let (ctx, mut rx) = route_ctx(connection.id());
11245
11246        router
11247            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11248            .await
11249            .unwrap();
11250        let response = rx.recv().await.unwrap();
11251        let ack = parse_ack(&response);
11252        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11253        let channel = 1;
11254
11255        let goodbye =
11256            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11257        router.route_for_connection(&ctx, goodbye).await.unwrap();
11258        assert!(rx.try_recv().is_err());
11259        assert!(registry.get_module("aft").unwrap().is_none());
11260
11261        router
11262            .route_for_connection(&ctx, channel_request(channel, 13))
11263            .await
11264            .unwrap();
11265        let error_frame = rx.recv().await.unwrap();
11266        assert_eq!(error_frame.header.ty, FrameType::Error);
11267        assert_eq!(error_frame.header.channel, channel);
11268    }
11269
11270    #[tokio::test]
11271    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11272        let registry = Arc::new(Registry::default());
11273        let handler = ControlHandler::new(registry).with_capability_config(
11274            [("prov".to_string(), true), ("cons".to_string(), true)],
11275            BTreeMap::new(),
11276        );
11277        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11278        register_capability_manifest(
11279            &handler,
11280            &provider_ctx,
11281            &mut provider_rx,
11282            capability_manifest("prov", &["thing/v1"], &[]),
11283            1,
11284        )
11285        .await;
11286        let mut consumer = capability_manifest("cons", &[], &[]);
11287        consumer.capabilities.as_mut().unwrap().requires.push(
11288            subc_protocol::manifest::CapabilityRequirement {
11289                capability: "thing/v1".to_string(),
11290                need: subc_protocol::manifest::CapabilityNeed::Required,
11291            },
11292        );
11293        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11294        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11295        assert_eq!(
11296            handler.capability_evaluator.verdict("cons", "thing/v1"),
11297            Some(CapabilityVerdict::Provided)
11298        );
11299        let (mut client_rx, _) = open_route_for_capability_test(
11300            &handler,
11301            &provider_ctx,
11302            &mut provider_rx,
11303            703,
11304            3,
11305            "prov",
11306            None,
11307        )
11308        .await;
11309        let goodbye =
11310            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11311        handler
11312            .handle_control_frame(&provider_ctx, goodbye)
11313            .await
11314            .unwrap();
11315        assert_eq!(
11316            handler.capability_evaluator.verdict("cons", "thing/v1"),
11317            Some(CapabilityVerdict::NeverProvided)
11318        );
11319        let closed = client_rx
11320            .try_recv()
11321            .expect("GOODBYE pushes route.closed before route GOODBYE");
11322        assert!(
11323            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11324            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11325        );
11326        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11327        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11328    }
11329
11330    #[tokio::test]
11331    async fn dropping_router_connection_releases_registration() {
11332        let registry = Arc::new(Registry::default());
11333        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
11334        let router = Router::with_control_handler(control);
11335        let connection = router.begin_connection();
11336        let (ctx, mut rx) = route_ctx(connection.id());
11337
11338        router
11339            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11340            .await
11341            .unwrap();
11342        let response = rx.recv().await.unwrap();
11343        let ack = parse_ack(&response);
11344        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11345        assert!(registry.get_module("aft").unwrap().is_some());
11346
11347        drop(connection);
11348
11349        assert!(registry.get_module("aft").unwrap().is_none());
11350        assert_eq!(registry.active_registration_count().unwrap(), 0);
11351    }
11352
11353    fn capability_manifest(
11354        module_id: &str,
11355        provides: &[&str],
11356        must_never_reach: &[&str],
11357    ) -> ModuleManifest {
11358        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11359        manifest.capabilities = Some(CapabilityDeclarations {
11360            provides: provides
11361                .iter()
11362                .map(|capability| (*capability).to_string())
11363                .collect(),
11364            requires: Vec::new(),
11365            must_never_reach: must_never_reach
11366                .iter()
11367                .map(|capability| (*capability).to_string())
11368                .collect(),
11369        });
11370        manifest
11371    }
11372
11373    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11374        Frame::build(
11375            FrameType::Hello,
11376            control_flags(),
11377            0,
11378            0,
11379            corr,
11380            serde_json::to_vec(&ModuleHelloBody {
11381                protocol_ver: manifest.protocol_ver,
11382                manifest,
11383                control_ops: None,
11384                launch_nonce: None,
11385            })
11386            .expect("capability test HELLO serializes"),
11387        )
11388        .expect("capability test HELLO frame builds")
11389    }
11390
11391    fn catalog_update_with_capabilities_frame(
11392        corr: u64,
11393        capabilities: CapabilityDeclarations,
11394    ) -> Frame {
11395        Frame::build(
11396            FrameType::Request,
11397            control_flags(),
11398            0,
11399            0,
11400            corr,
11401            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11402                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11403                capabilities: Some(capabilities),
11404                ready: None,
11405            })
11406            .expect("capability catalog.update serializes"),
11407        )
11408        .expect("capability catalog.update frame builds")
11409    }
11410
11411    async fn register_capability_manifest(
11412        handler: &ControlHandler,
11413        ctx: &RouteCtx,
11414        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11415        manifest: ModuleManifest,
11416        corr: u64,
11417    ) {
11418        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11419    }
11420
11421    async fn open_route_for_capability_test(
11422        handler: &ControlHandler,
11423        target_ctx: &RouteCtx,
11424        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11425        client_connection_id: u64,
11426        corr: u64,
11427        target_module_id: &str,
11428        consumer_identity: Option<ConsumerIdentity>,
11429    ) -> (
11430        mpsc::Receiver<crate::router::OutboundFrame>,
11431        ModuleControlRequest,
11432    ) {
11433        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11434        let route_handler = handler.clone();
11435        let target_module_id = target_module_id.to_string();
11436        let route_task = tokio::spawn(async move {
11437            route_handler
11438                .handle_control_frame(
11439                    &client_ctx,
11440                    route_open_frame_with_admission_facts(
11441                        corr,
11442                        &target_module_id,
11443                        unique_project_root("admission-facts"),
11444                        consumer_identity,
11445                        None,
11446                    ),
11447                )
11448                .await
11449                .expect("capability test route.open succeeds")
11450        });
11451        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
11452            .await
11453            .expect("capability test route.open must reach route.bind")
11454            .expect("target control receiver stays open");
11455        let bind_request: ModuleControlRequest =
11456            serde_json::from_slice(&bind.body).expect("route.bind decodes");
11457        handler
11458            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
11459            .await
11460            .expect("capability test route.bind ACK succeeds");
11461        assert!(route_task.await.expect("route.open task joins").is_empty());
11462        let opened = client_rx
11463            .recv()
11464            .await
11465            .expect("successful route.open publishes a response");
11466        assert!(matches!(
11467            serde_json::from_slice::<ClientControlResponse>(&opened.body),
11468            Ok(ClientControlResponse::RouteOpen { .. })
11469        ));
11470        (client_rx, bind_request)
11471    }
11472
11473    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
11474        assert_eq!(frame.header.ty, FrameType::Push);
11475        assert_eq!(frame.header.channel, 0);
11476        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
11477            .expect("route.closed control push decodes");
11478        let ClientControlPush::RouteClosed { channels, .. } = &push else {
11479            panic!("expected route.closed");
11480        };
11481        assert_eq!(channels.len(), 1, "exactly one violating route closed");
11482        let channels = channels.clone();
11483        assert_eq!(
11484            push,
11485            ClientControlPush::RouteClosed {
11486                module_id: target_module_id.to_string(),
11487                channels,
11488                reason: RouteCloseReason::CapabilityDenied,
11489                drained: false,
11490                abandoned: 0,
11491                excluded_subscriptions: 0,
11492                terminal: Some(false),
11493            }
11494        );
11495    }
11496
11497    #[tokio::test]
11498    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
11499        let registry = Arc::new(Registry::default());
11500        let forwarding = Arc::new(ForwardingTable::default());
11501        let supervisor = SupervisorHandle::new();
11502        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11503        let handler =
11504            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11505                .with_supervisor(supervisor);
11506        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
11507        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
11508        register_capability_manifest(
11509            &handler,
11510            &target_ctx,
11511            &mut target_rx,
11512            capability_manifest("target", &["credentials-provider/v1"], &[]),
11513            1,
11514        )
11515        .await;
11516        register_capability_manifest(
11517            &handler,
11518            &opener_ctx,
11519            &mut opener_rx,
11520            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11521            2,
11522        )
11523        .await;
11524
11525        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
11526        let replies = handler
11527            .handle_control_frame(
11528                &client_ctx,
11529                route_open_frame_with_admission_facts(
11530                    3,
11531                    "target",
11532                    unique_project_root("admission-facts"),
11533                    Some(ConsumerIdentity {
11534                        module_id: "opener".to_string(),
11535                        launch_nonce: "opener-nonce".to_string(),
11536                    }),
11537                    None,
11538                ),
11539            )
11540            .await
11541            .expect("denied route.open returns a typed frame");
11542        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11543        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11544        assert!(
11545            target_rx.try_recv().is_err(),
11546            "forbidden route.open must not relay route.bind"
11547        );
11548    }
11549
11550    #[tokio::test]
11551    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
11552        let registry = Arc::new(Registry::default());
11553        let forwarding = Arc::new(ForwardingTable::default());
11554        let supervisor = SupervisorHandle::new();
11555        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11556        let handler =
11557            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11558                .with_supervisor(supervisor);
11559        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
11560        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
11561        register_capability_manifest(
11562            &handler,
11563            &target_ctx,
11564            &mut target_rx,
11565            capability_manifest("target", &["credentials-provider/v1"], &[]),
11566            1,
11567        )
11568        .await;
11569        register_capability_manifest(
11570            &handler,
11571            &old_opener_ctx,
11572            &mut old_opener_rx,
11573            capability_manifest("opener", &[], &[]),
11574            2,
11575        )
11576        .await;
11577        let (mut client_rx, _) = open_route_for_capability_test(
11578            &handler,
11579            &target_ctx,
11580            &mut target_rx,
11581            712,
11582            3,
11583            "target",
11584            Some(ConsumerIdentity {
11585                module_id: "opener".to_string(),
11586                launch_nonce: "opener-nonce".to_string(),
11587            }),
11588        )
11589        .await;
11590        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11591
11592        handler
11593            .cleanup_connection(old_opener_ctx.connection_id)
11594            .expect("old opener registration cleans up");
11595        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
11596        register_capability_manifest(
11597            &handler,
11598            &new_opener_ctx,
11599            &mut new_opener_rx,
11600            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11601            4,
11602        )
11603        .await;
11604
11605        assert_capability_denied_push(
11606            client_rx
11607                .try_recv()
11608                .expect("HELLO deny addition must emit route.closed")
11609                .frame,
11610            "target",
11611        );
11612        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11613        assert!(matches!(
11614            target_rx.try_recv(),
11615            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11616        ));
11617    }
11618
11619    #[tokio::test]
11620    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
11621        let registry = Arc::new(Registry::default());
11622        let forwarding = Arc::new(ForwardingTable::default());
11623        let supervisor = SupervisorHandle::new();
11624        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11625        let handler =
11626            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11627                .with_supervisor(supervisor);
11628        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
11629        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
11630        register_capability_manifest(
11631            &handler,
11632            &target_ctx,
11633            &mut target_rx,
11634            capability_manifest("target", &[], &[]),
11635            1,
11636        )
11637        .await;
11638        register_capability_manifest(
11639            &handler,
11640            &opener_ctx,
11641            &mut opener_rx,
11642            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11643            2,
11644        )
11645        .await;
11646        let (mut client_rx, _) = open_route_for_capability_test(
11647            &handler,
11648            &target_ctx,
11649            &mut target_rx,
11650            722,
11651            3,
11652            "target",
11653            Some(ConsumerIdentity {
11654                module_id: "opener".to_string(),
11655                launch_nonce: "opener-nonce".to_string(),
11656            }),
11657        )
11658        .await;
11659        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11660
11661        let replies = handler
11662            .handle_control_frame(
11663                &target_ctx,
11664                catalog_update_with_capabilities_frame(
11665                    4,
11666                    CapabilityDeclarations {
11667                        provides: vec!["credentials-provider/v1".to_string()],
11668                        requires: Vec::new(),
11669                        must_never_reach: Vec::new(),
11670                    },
11671                ),
11672            )
11673            .await
11674            .expect("claim catalog.update succeeds");
11675        assert!(matches!(
11676            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
11677            Ok(ModuleControlResponseToModule::CatalogUpdate {})
11678        ));
11679        assert_capability_denied_push(
11680            client_rx
11681                .try_recv()
11682                .expect("claim addition must emit route.closed")
11683                .frame,
11684            "target",
11685        );
11686        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11687        assert!(matches!(
11688            target_rx.try_recv(),
11689            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11690        ));
11691    }
11692
11693    #[tokio::test]
11694    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
11695        let registry = Arc::new(Registry::default());
11696        let forwarding = Arc::new(ForwardingTable::default());
11697        let supervisor = SupervisorHandle::new();
11698        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11699        let handler =
11700            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11701                .with_supervisor(supervisor);
11702        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11703        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11704        register_capability_manifest(
11705            &handler,
11706            &target_ctx,
11707            &mut target_rx,
11708            capability_manifest("target", &["credentials-provider/v1"], &[]),
11709            1,
11710        )
11711        .await;
11712        register_capability_manifest(
11713            &handler,
11714            &opener_ctx,
11715            &mut opener_rx,
11716            capability_manifest("opener", &[], &[]),
11717            2,
11718        )
11719        .await;
11720        let (mut client_rx, _) = open_route_for_capability_test(
11721            &handler,
11722            &target_ctx,
11723            &mut target_rx,
11724            732,
11725            3,
11726            "target",
11727            Some(ConsumerIdentity {
11728                module_id: "opener".to_string(),
11729                launch_nonce: "opener-nonce".to_string(),
11730            }),
11731        )
11732        .await;
11733        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11734
11735        handler
11736            .handle_control_frame(
11737                &target_ctx,
11738                catalog_update_with_capabilities_frame(
11739                    4,
11740                    CapabilityDeclarations {
11741                        provides: Vec::new(),
11742                        requires: Vec::new(),
11743                        must_never_reach: Vec::new(),
11744                    },
11745                ),
11746            )
11747            .await
11748            .expect("claim removal catalog.update succeeds");
11749        assert_eq!(
11750            forwarding.active_binding_count().unwrap(),
11751            1,
11752            "removing an attested target claim must leave the route census unchanged"
11753        );
11754        assert!(
11755            client_rx.try_recv().is_err(),
11756            "claim removal must not emit route.closed capability_denied"
11757        );
11758        assert!(
11759            target_rx.try_recv().is_err(),
11760            "claim removal must not send the target a route GOODBYE"
11761        );
11762    }
11763
11764    /// A direct client may open a route to a denied capability provider; this
11765    /// policy applies only to attested supervised module origins, not to direct clients.
11766    #[tokio::test]
11767    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11768        let registry = Arc::new(Registry::default());
11769        let forwarding = Arc::new(ForwardingTable::default());
11770        let supervisor = SupervisorHandle::new();
11771        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11772        let handler =
11773            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11774                .with_supervisor(supervisor);
11775        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11776        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11777        register_capability_manifest(
11778            &handler,
11779            &target_ctx,
11780            &mut target_rx,
11781            capability_manifest("target", &["credentials-provider/v1"], &[]),
11782            1,
11783        )
11784        .await;
11785        register_capability_manifest(
11786            &handler,
11787            &opener_ctx,
11788            &mut opener_rx,
11789            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11790            2,
11791        )
11792        .await;
11793
11794        let (_client_rx, bind) = open_route_for_capability_test(
11795            &handler,
11796            &target_ctx,
11797            &mut target_rx,
11798            742,
11799            3,
11800            "target",
11801            None,
11802        )
11803        .await;
11804        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11805            panic!("direct scope-honesty route must bind");
11806        };
11807        assert_eq!(principal, Some(Principal::Direct));
11808        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11809    }
11810
11811    /// A module that denies a capability receives no self-route exemption when it
11812    /// also attestedly provides that capability.
11813    #[tokio::test]
11814    async fn must_never_reach_self_route_is_capability_forbidden() {
11815        let registry = Arc::new(Registry::default());
11816        let forwarding = Arc::new(ForwardingTable::default());
11817        let supervisor = SupervisorHandle::new();
11818        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11819        let handler =
11820            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11821                .with_supervisor(supervisor);
11822        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11823        register_capability_manifest(
11824            &handler,
11825            &self_ctx,
11826            &mut self_rx,
11827            capability_manifest(
11828                "self-provider",
11829                &["credentials-provider/v1"],
11830                &["credentials-provider/v1"],
11831            ),
11832            1,
11833        )
11834        .await;
11835
11836        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11837        let replies = handler
11838            .handle_control_frame(
11839                &client_ctx,
11840                route_open_frame_with_admission_facts(
11841                    2,
11842                    "self-provider",
11843                    unique_project_root("admission-facts"),
11844                    Some(ConsumerIdentity {
11845                        module_id: "self-provider".to_string(),
11846                        launch_nonce: "self-nonce".to_string(),
11847                    }),
11848                    None,
11849                ),
11850            )
11851            .await
11852            .expect("self-route refusal returns a typed frame");
11853        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11854        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11855        assert!(
11856            self_rx.try_recv().is_err(),
11857            "self denial must not relay route.bind"
11858        );
11859    }
11860
11861    #[test]
11862    fn unsupported_channel_zero_frame_returns_error() {
11863        let handler = ControlHandler::default();
11864        let request = Frame::build(
11865            FrameType::Request,
11866            control_flags(),
11867            0,
11868            0,
11869            21,
11870            b"opaque".to_vec(),
11871        )
11872        .unwrap();
11873
11874        let response = handler
11875            .handle_control(ConnectionId::new(1), request)
11876            .unwrap();
11877
11878        assert_eq!(response[0].header.ty, FrameType::Error);
11879        assert_eq!(
11880            parse_error(&response[0])["code"],
11881            "unsupported_control_frame"
11882        );
11883    }
11884
11885    /// Blue/green swap at the control-plane boundary. The supervisor that opens
11886    /// a swap is not wired yet, so the candidate is registered here directly
11887    /// into the registry and forwarding candidate slots, the way the swap's
11888    /// HELLO admission will.
11889    mod swap {
11890        use super::*;
11891
11892        const INCUMBENT: ConnectionId = ConnectionId::new(30);
11893        const CANDIDATE: ConnectionId = ConnectionId::new(40);
11894
11895        struct Swap {
11896            registry: Arc<Registry>,
11897            forwarding: Arc<ForwardingTable>,
11898            handler: ControlHandler,
11899            incumbent_ctx: RouteCtx,
11900            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11901            candidate_ctx: RouteCtx,
11902            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11903        }
11904
11905        async fn swap_with_incumbent() -> Swap {
11906            let registry = Arc::new(Registry::default());
11907            let forwarding = Arc::new(ForwardingTable::default());
11908            let handler =
11909                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11910            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11911            hello_via_sink(
11912                &handler,
11913                &incumbent_ctx,
11914                &mut incumbent_rx,
11915                hello_frame("aft", PROTOCOL_VERSION, 7),
11916            )
11917            .await;
11918            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11919            Swap {
11920                registry,
11921                forwarding,
11922                handler,
11923                incumbent_ctx,
11924                incumbent_rx,
11925                candidate_ctx,
11926                candidate_rx,
11927            }
11928        }
11929
11930        fn register_candidate(swap: &Swap, ready: Option<bool>) {
11931            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11932            candidate_manifest.ready = ready;
11933            let registration = swap
11934                .registry
11935                .register_candidate_with_control_ops(
11936                    candidate_manifest,
11937                    PROTOCOL_VERSION,
11938                    CANDIDATE,
11939                    module_baseline_control_ops(),
11940                )
11941                .unwrap();
11942            swap.forwarding
11943                .register_candidate_module_connection(
11944                    CANDIDATE,
11945                    "aft".to_string(),
11946                    PROTOCOL_VERSION,
11947                    manifest_concurrency(&registration.manifest),
11948                    swap.candidate_ctx.egress.clone(),
11949                )
11950                .unwrap();
11951        }
11952
11953        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11954            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11955            swap.registry.promote_candidate("aft").unwrap().unwrap();
11956            cutover.incumbent.unwrap()
11957        }
11958
11959        fn keyed_total(counters: &Value, key: &str) -> u64 {
11960            counters[key]
11961                .as_object()
11962                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11963                .unwrap_or(0)
11964        }
11965
11966        /// An ack from the incumbent for a bind it was sent before cutover,
11967        /// arriving before the incumbent is drained. The incumbent is the live
11968        /// connection carrying every other client's routes, so the ack must
11969        /// not end it: the waiting client is told to retry, the reservation is
11970        /// given back, and the incumbent is told to drop just that binding.
11971        #[tokio::test]
11972        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11973            let mut swap = swap_with_incumbent().await;
11974            let handler = swap.handler.clone();
11975
11976            // A co-tenant route, bound on the incumbent before the swap.
11977            let cotenant = ConnectionId::new(31);
11978            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11979            let (cotenant_task, cotenant_bind) = relay_route_open(
11980                &handler,
11981                cotenant,
11982                &cotenant_ctx.egress,
11983                &mut swap.incumbent_rx,
11984                100,
11985                "aft",
11986                "swap-cotenant",
11987            )
11988            .await;
11989            handler
11990                .handle_control_frame(
11991                    &swap.incumbent_ctx,
11992                    route_bind_ack(cotenant_bind.header.corr),
11993                )
11994                .await
11995                .unwrap();
11996            assert!(cotenant_task.await.unwrap().is_empty());
11997            let (cotenant_channel, cotenant_epoch) =
11998                published_route(&cotenant_rx.recv().await.unwrap());
11999
12000            // A second route.open, relayed to the incumbent and not yet acked.
12001            let caller = ConnectionId::new(32);
12002            let (caller_ctx, mut caller_rx) = route_ctx(caller);
12003            let (caller_task, caller_bind) = relay_route_open(
12004                &handler,
12005                caller,
12006                &caller_ctx.egress,
12007                &mut swap.incumbent_rx,
12008                101,
12009                "aft",
12010                "swap-caller",
12011            )
12012            .await;
12013            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
12014
12015            register_candidate(&swap, None);
12016            cutover(&swap);
12017
12018            // The incumbent acks after promotion and before any drain.
12019            let ack = handler
12020                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
12021                .await;
12022            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12023            if module_loop_error.is_some() {
12024                // What the connection loop does with an untranslated router
12025                // error: end the connection, releasing every route on it.
12026                handler.cleanup_connection(INCUMBENT).unwrap();
12027            }
12028
12029            // 1. The incumbent's other routes survive.
12030            assert!(
12031                cotenant_rx.try_recv().is_err(),
12032                "the co-tenant route on the incumbent was torn down by one late ack: \
12033                 {module_loop_error:?}"
12034            );
12035            assert!(matches!(
12036                swap.forwarding
12037                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12038                    .unwrap(),
12039                DataRoute::Client(DataRouteState::Bound(_))
12040            ));
12041            assert_eq!(module_loop_error, None);
12042            assert!(swap
12043                .registry
12044                .get_module_by_connection(INCUMBENT)
12045                .unwrap()
12046                .is_some());
12047
12048            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
12049            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
12050                .await
12051                .expect("the incumbent is told to drop the abandoned binding")
12052                .unwrap()
12053                .frame;
12054            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12055            assert_eq!(goodbye.header.channel, abandoned_channel);
12056            assert_eq!(goodbye.header.epoch, abandoned_epoch);
12057            assert!(swap.incumbent_rx.try_recv().is_err());
12058
12059            // 3. The waiting client gets a retryable refusal and no route.
12060            let response = caller_task.await.unwrap();
12061            assert_eq!(response.len(), 1);
12062            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
12063            assert!(caller_rx.try_recv().is_err());
12064
12065            // 4. The reservation pair is given back, and the pending bind
12066            //    settled exactly once: one accepted open (the co-tenant) and one
12067            //    refused open (the caller), nothing counted twice.
12068            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
12069            let counters = handler.counters().snapshot();
12070            assert_eq!(
12071                keyed_total(&counters, "route_open_accepted_by_principal"),
12072                1
12073            );
12074            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
12075            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
12076        }
12077
12078        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
12079        /// id would resolve to the promoted candidate and every new route.open
12080        /// would be refused as reloading, leaving neither process routable.
12081        #[tokio::test]
12082        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
12083            let mut swap = swap_with_incumbent().await;
12084            register_candidate(&swap, None);
12085            let incumbent = cutover(&swap);
12086            swap.forwarding
12087                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
12088                .unwrap()
12089                .expect("the incumbent is still registered");
12090
12091            let client = ConnectionId::new(33);
12092            let (client_ctx, mut client_rx) = route_ctx(client);
12093            let route_handler = swap.handler.clone();
12094            let open_ctx = RouteCtx {
12095                connection_id: client,
12096                egress: client_ctx.egress.clone(),
12097            };
12098            let mut route_task = tokio::spawn(async move {
12099                route_handler
12100                    .handle_control_frame(
12101                        &open_ctx,
12102                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
12103                    )
12104                    .await
12105                    .unwrap()
12106            });
12107            let bind = tokio::select! {
12108                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
12109                response = &mut route_task => {
12110                    let response = response.unwrap();
12111                    panic!(
12112                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
12113                        parse_error(&response[0])["code"]
12114                    );
12115                }
12116            };
12117            swap.handler
12118                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
12119                .await
12120                .unwrap();
12121            assert!(route_task.await.unwrap().is_empty());
12122            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
12123            match swap
12124                .forwarding
12125                .lookup_data_route(client, channel, epoch)
12126                .unwrap()
12127            {
12128                DataRoute::Client(DataRouteState::Bound(route)) => {
12129                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
12130                }
12131                other => panic!("expected a bound route on the candidate, got {other:?}"),
12132            }
12133            assert!(swap.incumbent_rx.try_recv().is_err());
12134        }
12135
12136        /// A candidate declares itself ready with `catalog.update` on its own
12137        /// connection. If the connection-keyed registry lookups searched only the
12138        /// active slot, this would answer `not_registered` and the candidate
12139        /// would never become ready.
12140        #[tokio::test]
12141        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
12142            let swap = swap_with_incumbent().await;
12143            register_candidate(&swap, Some(false));
12144            let update = Frame::build(
12145                FrameType::Request,
12146                control_flags(),
12147                0,
12148                0,
12149                55,
12150                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12151                    provides: manifest("aft", PROTOCOL_VERSION).provides,
12152                    capabilities: None,
12153                    ready: Some(true),
12154                })
12155                .unwrap(),
12156            )
12157            .unwrap();
12158
12159            let replies = swap
12160                .handler
12161                .handle_control_frame(&swap.candidate_ctx, update)
12162                .await
12163                .unwrap();
12164
12165            assert_eq!(replies.len(), 1);
12166            assert_eq!(
12167                replies[0].header.ty,
12168                FrameType::Response,
12169                "candidate catalog.update was refused: {:?}",
12170                serde_json::from_slice::<Value>(&replies[0].body).ok()
12171            );
12172            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
12173            assert_eq!(
12174                swap.registry
12175                    .get_module("aft")
12176                    .unwrap()
12177                    .unwrap()
12178                    .connection_id,
12179                INCUMBENT
12180            );
12181        }
12182    }
12183
12184    /// The HELLO gate while the supervisor has a swap open: only the nonce it
12185    /// minted for the candidate admits a second process, into the candidate
12186    /// slot, and that check runs ahead of the reserved-module gate.
12187    mod swap_admission {
12188        use super::*;
12189
12190        const INCUMBENT_NONCE: &str = "incumbent-nonce";
12191        const CANDIDATE_NONCE: &str = "candidate-nonce";
12192
12193        fn handler_with_incumbent(
12194            module_id: &str,
12195            reserved: bool,
12196        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
12197            let registry = Arc::new(Registry::default());
12198            let supervisor = SupervisorHandle::new();
12199            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
12200            if reserved {
12201                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
12202            }
12203            let handler =
12204                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
12205            let incumbent = handler
12206                .handle_control(
12207                    ConnectionId::new(1),
12208                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
12209                )
12210                .unwrap();
12211            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
12212            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
12213            (registry, supervisor, handler)
12214        }
12215
12216        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
12217        /// admits every nonce, so while a swap is open the swap gate is the only
12218        /// thing between a key-holder and the candidate slot. A nonce the
12219        /// supervisor did not mint, or none at all, is refused, and neither the
12220        /// incumbent's registration nor the candidate slot moves.
12221        #[test]
12222        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
12223            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
12224
12225            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
12226                let replies = handler
12227                    .handle_control(
12228                        ConnectionId::new(connection),
12229                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
12230                    )
12231                    .unwrap();
12232                assert_eq!(replies[0].header.ty, FrameType::Error);
12233                assert_eq!(
12234                    parse_error(&replies[0])["code"],
12235                    "swap_token_invalid",
12236                    "nonce {nonce:?}"
12237                );
12238            }
12239            assert!(registry.get_candidate("aft").unwrap().is_none());
12240            assert_eq!(
12241                registry.get_module("aft").unwrap().unwrap().connection_id,
12242                ConnectionId::new(1)
12243            );
12244
12245            // Control: the minted token is admitted, into the candidate slot,
12246            // and only once.
12247            let admitted = handler
12248                .handle_control(
12249                    ConnectionId::new(4),
12250                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12251                )
12252                .unwrap();
12253            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12254            assert_eq!(
12255                registry
12256                    .get_candidate("aft")
12257                    .unwrap()
12258                    .unwrap()
12259                    .connection_id,
12260                ConnectionId::new(4)
12261            );
12262            assert_eq!(
12263                registry.get_module("aft").unwrap().unwrap().connection_id,
12264                ConnectionId::new(1),
12265                "the candidate must not take the active slot"
12266            );
12267            let replayed = handler
12268                .handle_control(
12269                    ConnectionId::new(5),
12270                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12271                )
12272                .unwrap();
12273            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12274
12275            // The case only this gate covers: the incumbent has died mid-swap,
12276            // so its duplicate refusal is gone too, and without the gate a
12277            // key-holder would take the id's ACTIVE slot.
12278            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12279            let squatter = handler
12280                .handle_control(
12281                    ConnectionId::new(6),
12282                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12283                )
12284                .unwrap();
12285            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12286            assert!(
12287                registry.get_module("aft").unwrap().is_none(),
12288                "a squatter took the active slot of an id being swapped"
12289            );
12290        }
12291
12292        /// Design mutation arm (iii). A reserved module's candidate presents a
12293        /// nonce the reserved gate has never seen (that gate holds the
12294        /// incumbent's), so the swap gate must run first or the candidate is
12295        /// refused `reserved_module` and a reserved module can never be swapped.
12296        #[test]
12297        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12298            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12299
12300            let replies = handler
12301                .handle_control(
12302                    ConnectionId::new(2),
12303                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12304                )
12305                .unwrap();
12306
12307            assert_eq!(
12308                replies[0].header.ty,
12309                FrameType::HelloAck,
12310                "reserved candidate refused: {:?}",
12311                serde_json::from_slice::<Value>(&replies[0].body).ok()
12312            );
12313            assert_eq!(
12314                registry
12315                    .get_candidate("vault")
12316                    .unwrap()
12317                    .unwrap()
12318                    .connection_id,
12319                ConnectionId::new(2)
12320            );
12321        }
12322
12323        /// With no swap open the gate is inert: the incumbent's reserved gate
12324        /// and duplicate refusal behave exactly as before.
12325        #[test]
12326        fn without_an_open_swap_the_ordinary_gates_decide() {
12327            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12328            supervisor.close_swap("vault");
12329
12330            let candidate = handler
12331                .handle_control(
12332                    ConnectionId::new(2),
12333                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12334                )
12335                .unwrap();
12336            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12337            let duplicate = handler
12338                .handle_control(
12339                    ConnectionId::new(3),
12340                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12341                )
12342                .unwrap();
12343            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12344            assert!(registry.get_candidate("vault").unwrap().is_none());
12345        }
12346    }
12347
12348    /// `scope.sync` and `scope.describe` through the real control handler: who
12349    /// may sync is decided by the registration and launch nonce of the module
12350    /// connection, never by the request body.
12351    mod scopes {
12352        use subc_protocol::scope::{
12353            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12354            ScopeStatus,
12355        };
12356
12357        use super::*;
12358
12359        const OWNER: &str = "prefrontal-core";
12360
12361        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12362            ScopeRecord {
12363                scope_ref: scope_ref.to_string(),
12364                scope_epoch,
12365                kind: ScopeKind::Head,
12366                parent: None,
12367                child_owners: Vec::new(),
12368                carriers: Vec::new(),
12369                attributes: Default::default(),
12370            }
12371        }
12372
12373        async fn call(
12374            handler: &ControlHandler,
12375            ctx: &RouteCtx,
12376            request: &ModuleControlRequestFromModule,
12377        ) -> Frame {
12378            let body = serde_json::to_vec(request).unwrap();
12379            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12380            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12381            assert_eq!(replies.len(), 1, "{replies:?}");
12382            replies.pop().unwrap()
12383        }
12384
12385        async fn sync(
12386            handler: &ControlHandler,
12387            ctx: &RouteCtx,
12388            generation: u64,
12389            scopes: Vec<ScopeRecord>,
12390        ) -> Result<ModuleControlResponseToModule, String> {
12391            let reply = call(
12392                handler,
12393                ctx,
12394                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
12395            )
12396            .await;
12397            match reply.header.ty {
12398                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
12399                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
12400            }
12401        }
12402
12403        async fn describe(
12404            handler: &ControlHandler,
12405            ctx: &RouteCtx,
12406            owner: &str,
12407            scope_ref: &str,
12408        ) -> ModuleControlResponseToModule {
12409            let reply = call(
12410                handler,
12411                ctx,
12412                &ModuleControlRequestFromModule::ScopeDescribe {
12413                    owner: Principal::Reserved {
12414                        module_id: owner.to_string(),
12415                    },
12416                    scope_ref: scope_ref.to_string(),
12417                },
12418            )
12419            .await;
12420            assert_eq!(
12421                reply.header.ty,
12422                FrameType::Response,
12423                "{:?}",
12424                parse_error(&reply)
12425            );
12426            serde_json::from_slice(&reply.body).unwrap()
12427        }
12428
12429        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
12430        async fn module(
12431            handler: &ControlHandler,
12432            connection: u64,
12433            module_id: &str,
12434            nonce: Option<&str>,
12435        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12436            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
12437            hello_via_sink(
12438                handler,
12439                &ctx,
12440                &mut rx,
12441                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
12442            )
12443            .await;
12444            (ctx, rx)
12445        }
12446
12447        /// `direct` and every other client connection has no registration, so
12448        /// it can neither sync nor own a scope.
12449        #[tokio::test]
12450        async fn a_client_connection_cannot_sync_or_describe() {
12451            let handler = ControlHandler::new(Arc::new(Registry::default()));
12452            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
12453            for request in [
12454                ModuleControlRequestFromModule::ScopeSync {
12455                    generation: 1,
12456                    scopes: vec![head("s", 1)],
12457                },
12458                ModuleControlRequestFromModule::ScopeDescribe {
12459                    owner: Principal::Direct,
12460                    scope_ref: "s".to_string(),
12461                },
12462            ] {
12463                let reply = call(&handler, &ctx, &request).await;
12464                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
12465            }
12466            assert!(
12467                !handler
12468                    .scopes
12469                    .read()
12470                    .unwrap()
12471                    .describe(
12472                        &Principal::Reserved {
12473                            module_id: OWNER.to_string()
12474                        },
12475                        "s"
12476                    )
12477                    .owner_synced
12478            );
12479        }
12480
12481        /// A module the supervisor did not spawn registers without a launch
12482        /// nonce, so it is never an owner's current launch.
12483        #[tokio::test]
12484        async fn a_module_without_a_supervised_launch_cannot_sync() {
12485            let handler = ControlHandler::new(Arc::new(Registry::default()));
12486            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
12487            assert_eq!(
12488                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
12489                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12490            );
12491        }
12492
12493        #[tokio::test]
12494        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
12495            let supervisor = SupervisorHandle::new();
12496            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12497            let handler = ControlHandler::new(Arc::new(Registry::default()))
12498                .with_supervisor(supervisor.clone());
12499            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12500            sync(&handler, &incumbent, 1, vec![head("s", 1)])
12501                .await
12502                .expect("the current launch syncs");
12503
12504            // A swap candidate registers with the swap token and is refused
12505            // while the incumbent keeps syncing.
12506            supervisor.open_swap(OWNER, "n2".to_string());
12507            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
12508            assert_eq!(
12509                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12510                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12511            );
12512            sync(&handler, &incumbent, 2, vec![head("s", 1)])
12513                .await
12514                .expect("the serving owner syncs during the swap");
12515
12516            // The swap fails and is rolled back. The candidate never held sync
12517            // authority, and still cannot sync.
12518            supervisor.close_swap(OWNER);
12519            assert_eq!(
12520                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12521                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12522            );
12523            sync(&handler, &incumbent, 3, vec![head("s", 1)])
12524                .await
12525                .expect("the serving owner syncs after the rollback");
12526            handler.cleanup_connection(candidate.connection_id).unwrap();
12527
12528            // A swap that cuts over. Promotion records the candidate's nonce as
12529            // the module's spawn nonce, which is what `set_spawn_nonce` does
12530            // here; the promoted connection then takes authority at any
12531            // generation and the superseded incumbent is refused.
12532            supervisor.open_swap(OWNER, "n3".to_string());
12533            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
12534            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
12535            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
12536                .await
12537                .expect("the promoted launch takes authority");
12538            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12539                panic!("unexpected reply {reply:?}");
12540            };
12541            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
12542            assert_eq!(
12543                sync(&handler, &incumbent, 4, Vec::new()).await,
12544                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12545            );
12546        }
12547
12548        /// Authority dies with its connection: the cleanup path releases it,
12549        /// so the owner's next connection takes it at any generation.
12550        #[tokio::test]
12551        async fn closing_the_authority_connection_frees_sync_authority() {
12552            let supervisor = SupervisorHandle::new();
12553            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12554            let handler = ControlHandler::new(Arc::new(Registry::default()))
12555                .with_supervisor(supervisor.clone());
12556            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12557            sync(&handler, &first, 10, vec![head("s", 1)])
12558                .await
12559                .unwrap();
12560            handler.cleanup_connection(first.connection_id).unwrap();
12561
12562            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
12563            sync(&handler, &second, 1, vec![head("s", 1)])
12564                .await
12565                .expect("the next connection takes the released authority");
12566        }
12567
12568        #[tokio::test]
12569        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
12570            let supervisor = SupervisorHandle::new();
12571            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12572            let handler =
12573                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
12574            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
12575            sync(&handler, &first, 10, vec![head("s", 1)])
12576                .await
12577                .unwrap();
12578            handler
12579                .handle_control_frame(
12580                    &first,
12581                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
12582                )
12583                .await
12584                .unwrap();
12585            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
12586            sync(&handler, &second, 1, vec![head("s", 1)])
12587                .await
12588                .expect("GOODBYE releases authority even if the old socket remains open");
12589        }
12590
12591        #[tokio::test]
12592        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
12593            let registry = Arc::new(Registry::default());
12594            let supervisor_handle = SupervisorHandle::new();
12595            let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
12596                .with_handle(supervisor_handle.clone())
12597                .with_daemon_incarnation("incarnation-7".to_string());
12598            // Configured with enabled: false, so the supervisor lists the
12599            // module without spawning a process for it.
12600            supervisor
12601                .supervise_configured(
12602                    ModuleSpec {
12603                        module_id: OWNER.to_string(),
12604                        program: PathBuf::from("/nonexistent/prefrontal-core"),
12605                        args: Vec::new(),
12606                        env: Vec::new(),
12607                        reserved: false,
12608                        reserved_prefixes: Vec::new(),
12609                        protocol: ModuleProtocol::Subc,
12610                        overlap: Default::default(),
12611                    },
12612                    false,
12613                )
12614                .unwrap();
12615            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
12616            let handler =
12617                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12618            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
12619
12620            // Configured but not yet synced: a reader waits for the owner.
12621            let ModuleControlResponseToModule::ScopeDescribe {
12622                status,
12623                daemon_incarnation,
12624                owner_synced,
12625                owner_configured,
12626                scope,
12627                ..
12628            } = describe(&handler, &reader, OWNER, "s").await
12629            else {
12630                panic!("not a describe reply");
12631            };
12632            assert_eq!(status, ScopeStatus::NotLive);
12633            assert_eq!(daemon_incarnation, "incarnation-7");
12634            assert!(!owner_synced);
12635            assert!(owner_configured);
12636            assert!(scope.is_none());
12637
12638            // Not a supervised module: the owner will never sync, and a reader
12639            // refuses rather than waits.
12640            let ModuleControlResponseToModule::ScopeDescribe {
12641                status,
12642                owner_configured,
12643                ..
12644            } = describe(&handler, &reader, "ghost", "s").await
12645            else {
12646                panic!("not a describe reply");
12647            };
12648            assert_eq!(status, ScopeStatus::NotLive);
12649            assert!(!owner_configured);
12650
12651            // Live, with the stamp fields and the computed owner_authorized.
12652            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
12653            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
12654            let ModuleControlResponseToModule::ScopeDescribe {
12655                status,
12656                scope_epoch,
12657                owner_synced,
12658                scope,
12659                ..
12660            } = describe(&handler, &reader, OWNER, "s").await
12661            else {
12662                panic!("not a describe reply");
12663            };
12664            assert_eq!(status, ScopeStatus::Live);
12665            assert_eq!(scope_epoch, Some(4));
12666            assert!(owner_synced);
12667            let stamp = scope.expect("a live scope carries its stamp");
12668            assert!(
12669                stamp.owner_authorized,
12670                "prefrontal-core is the default authority"
12671            );
12672            assert_eq!(stamp.kind, ScopeKind::Head);
12673        }
12674
12675        #[tokio::test]
12676        async fn scope_authority_owners_decides_owner_authorized() {
12677            let supervisor = SupervisorHandle::new();
12678            supervisor.set_spawn_nonce("broca", "b1".to_string());
12679            let handler = ControlHandler::new(Arc::new(Registry::default()))
12680                .with_supervisor(supervisor)
12681                .with_scope_authority_owners(vec!["broca".to_string()]);
12682            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
12683            let mut gated = head("s", 1);
12684            gated.attributes.agent_id = Some("agent".to_string());
12685            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
12686            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
12687                describe(&handler, &broca, "broca", "s").await
12688            else {
12689                panic!("not a describe reply");
12690            };
12691            assert!(scope.unwrap().owner_authorized);
12692        }
12693
12694        /// With route admission, the stamp, the commit re-check and drains in
12695        /// place, the feature is advertised: the module ops in HELLO_ACK, and
12696        /// `scopes/v1` in HELLO_ACK and `server.describe`.
12697        #[tokio::test]
12698        async fn scope_ops_and_the_scopes_capability_are_advertised() {
12699            let handler = ControlHandler::new(Arc::new(Registry::default()));
12700            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
12701            let ack = hello_via_sink(
12702                &handler,
12703                &ctx,
12704                &mut rx,
12705                hello_frame("m", PROTOCOL_VERSION, 1),
12706            )
12707            .await;
12708            let ack = parse_ack(&ack);
12709            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
12710                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
12711            }
12712            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
12713
12714            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
12715            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12716            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12717            let reply = handler
12718                .handle_control_frame(&client, frame)
12719                .await
12720                .unwrap()
12721                .pop()
12722                .unwrap();
12723            let ClientControlResponse::ServerDescribe { capabilities, .. } =
12724                serde_json::from_slice(&reply.body).unwrap()
12725            else {
12726                panic!("not a server.describe reply");
12727            };
12728            assert!(
12729                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12730                "{capabilities:?}"
12731            );
12732        }
12733
12734        // ---- route admission, stamps, commit re-check and drains ----------
12735
12736        const PLEXUS: &str = "plexus";
12737        const OTHER: &str = "other";
12738        const AFT: &str = "aft";
12739        const BROCA: &str = "broca";
12740        const MAGIC: &str = "magic-context";
12741
12742        fn nonce(module_id: &str) -> String {
12743            format!("nonce-{module_id}")
12744        }
12745
12746        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12747            let (tx, rx) = mpsc::channel(64);
12748            (
12749                RouteCtx {
12750                    connection_id: ConnectionId::new(connection),
12751                    egress: FrameSink::new(tx),
12752                },
12753                rx,
12754            )
12755        }
12756
12757        /// A daemon with a configured owner (prefrontal-core) registered on its
12758        /// own module connection, two routable targets (plexus, other), and
12759        /// launch nonces minted for the modules that open routes as carriers.
12760        struct Rig {
12761            handler: ControlHandler,
12762            forwarding: Arc<ForwardingTable>,
12763            owner: RouteCtx,
12764            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12765            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12766            generation: u64,
12767            next_connection: u64,
12768            _supervisor: Supervisor,
12769        }
12770
12771        async fn rig() -> Rig {
12772            rig_with_flow_support(true).await
12773        }
12774
12775        async fn rig_with_flow_support(flow_support: bool) -> Rig {
12776            let registry = Arc::new(Registry::default());
12777            let forwarding = Arc::new(ForwardingTable::default());
12778            let supervisor_handle = SupervisorHandle::new();
12779            let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
12780                .with_handle(supervisor_handle.clone());
12781            supervisor
12782                .supervise_configured(
12783                    ModuleSpec {
12784                        module_id: OWNER.to_string(),
12785                        program: PathBuf::from("/nonexistent/prefrontal-core"),
12786                        args: Vec::new(),
12787                        env: Vec::new(),
12788                        reserved: false,
12789                        reserved_prefixes: Vec::new(),
12790                        protocol: ModuleProtocol::Subc,
12791                        overlap: Default::default(),
12792                    },
12793                    false,
12794                )
12795                .unwrap();
12796            for module_id in [OWNER, AFT, BROCA, MAGIC] {
12797                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12798            }
12799            let handler =
12800                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12801                    .with_supervisor(supervisor_handle);
12802            let (owner, mut owner_rx) = wide_ctx(1);
12803            hello_via_sink(
12804                &handler,
12805                &owner,
12806                &mut owner_rx,
12807                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12808            )
12809            .await;
12810            let mut modules = BTreeMap::new();
12811            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12812                let (ctx, mut rx) = wide_ctx(connection);
12813                let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
12814                let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12815                // A decoder version alone must not admit flow routes. Every
12816                // target here declares wire crate version 0.29.0; only one that
12817                // declares `flow-scopes/v1` promises flow behaviour.
12818                body["manifest"]["provenance"] =
12819                    serde_json::json!({"wire_crate_version": "0.29.0"});
12820                if flow_support {
12821                    body["manifest"]["capabilities"] =
12822                        serde_json::json!({"provides": ["flow-scopes/v1"]});
12823                }
12824                let hello = Frame::build(
12825                    FrameType::Hello,
12826                    control_flags(),
12827                    0,
12828                    0,
12829                    connection,
12830                    serde_json::to_vec(&body).unwrap(),
12831                )
12832                .unwrap();
12833                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
12834                modules.insert(module_id.to_string(), (ctx, rx));
12835            }
12836            Rig {
12837                handler,
12838                forwarding,
12839                owner,
12840                _owner_rx: owner_rx,
12841                modules,
12842                generation: 0,
12843                next_connection: 100,
12844                _supervisor: supervisor,
12845            }
12846        }
12847
12848        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12849            ScopeCarrier {
12850                principal: Principal::Reserved {
12851                    module_id: module_id.to_string(),
12852                },
12853                targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12854            }
12855        }
12856
12857        /// The scope most tests open under: aft carries to any module, broca
12858        /// only to plexus and other, and the owner delegates as agent-1.
12859        fn session(scope_epoch: u64) -> ScopeRecord {
12860            let mut record = head("s", scope_epoch);
12861            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12862            record.attributes.agent_id = Some("agent-1".to_string());
12863            record.attributes.delegates = true;
12864            record
12865        }
12866
12867        impl Rig {
12868            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12869                self.generation += 1;
12870                sync(&self.handler, &self.owner, self.generation, scopes)
12871                    .await
12872                    .expect("the owner's sync is accepted");
12873            }
12874
12875            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12876                ScopeSelector {
12877                    owner: Principal::Reserved {
12878                        module_id: OWNER.to_string(),
12879                    },
12880                    scope_ref: scope_ref.to_string(),
12881                    scope_epoch,
12882                }
12883            }
12884
12885            fn open_frame(
12886                &mut self,
12887                opener: Option<&str>,
12888                target: &str,
12889                scope: Option<ScopeSelector>,
12890            ) -> (
12891                RouteCtx,
12892                mpsc::Receiver<crate::router::OutboundFrame>,
12893                Frame,
12894            ) {
12895                self.next_connection += 1;
12896                let (ctx, rx) = wide_ctx(self.next_connection);
12897                let root = unique_project_root("scoped-open");
12898                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12899                    target: RouteTarget::ToolProvider {
12900                        module_id: target.to_string(),
12901                    },
12902                    identity: BindIdentity::new(
12903                        root.path().to_path_buf(),
12904                        "unit".to_string(),
12905                        "session".to_string(),
12906                    ),
12907                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
12908                        module_id: module_id.to_string(),
12909                        launch_nonce: nonce(module_id),
12910                    }),
12911                    consumer_capabilities: None,
12912                    role_versions: None,
12913                    admission_facts: None,
12914                    scope,
12915                })
12916                .unwrap();
12917                let frame = Frame::build(
12918                    FrameType::Request,
12919                    control_flags(),
12920                    0,
12921                    0,
12922                    self.next_connection,
12923                    body,
12924                )
12925                .unwrap();
12926                (ctx, rx, frame)
12927            }
12928
12929            /// Open and expect a refusal before anything is relayed.
12930            async fn refused(
12931                &mut self,
12932                opener: Option<&str>,
12933                target: &str,
12934                scope: Option<ScopeSelector>,
12935            ) -> String {
12936                self.refusal_body(opener, target, scope).await["code"]
12937                    .as_str()
12938                    .unwrap()
12939                    .to_string()
12940            }
12941
12942            async fn refusal_body(
12943                &mut self,
12944                opener: Option<&str>,
12945                target: &str,
12946                scope: Option<ScopeSelector>,
12947            ) -> Value {
12948                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12949                let replies = tokio::time::timeout(
12950                    Duration::from_secs(2),
12951                    self.handler.handle_control_frame(&ctx, frame),
12952                )
12953                .await
12954                .expect("the open must be refused before waiting for a bind ack")
12955                .unwrap();
12956                assert_eq!(replies.len(), 1, "{replies:?}");
12957                assert_eq!(replies[0].header.ty, FrameType::Error);
12958                let (_, module_rx) = self.modules.get_mut(target).unwrap();
12959                assert!(
12960                    module_rx.try_recv().is_err(),
12961                    "a refused open relays nothing"
12962                );
12963                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
12964                parse_error(&replies[0])
12965            }
12966
12967            /// Start an open and return its task and the bind the target got.
12968            async fn relayed(
12969                &mut self,
12970                opener: Option<&str>,
12971                target: &str,
12972                scope: Option<ScopeSelector>,
12973            ) -> Relayed {
12974                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12975                let handler = self.handler.clone();
12976                let task_ctx = ctx.clone();
12977                let task = tokio::spawn(async move {
12978                    handler
12979                        .handle_control_frame(&task_ctx, frame)
12980                        .await
12981                        .unwrap()
12982                });
12983                let (_, module_rx) = self.modules.get_mut(target).unwrap();
12984                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12985                    .await
12986                    .expect("the target receives the relayed route.bind")
12987                    .unwrap()
12988                    .frame;
12989                Relayed {
12990                    target: target.to_string(),
12991                    client: ctx,
12992                    client_rx: rx,
12993                    task,
12994                    bind,
12995                }
12996            }
12997
12998            async fn ack(&self, relayed: &Relayed) {
12999                let (module, _) = &self.modules[&relayed.target];
13000                self.handler
13001                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
13002                    .await
13003                    .unwrap();
13004            }
13005
13006            /// Open, ack and return the bound route.
13007            async fn bound(
13008                &mut self,
13009                opener: Option<&str>,
13010                target: &str,
13011                scope: Option<ScopeSelector>,
13012            ) -> Bound {
13013                let relayed = self.relayed(opener, target, scope).await;
13014                self.ack(&relayed).await;
13015                let Relayed {
13016                    target,
13017                    client,
13018                    mut client_rx,
13019                    task,
13020                    bind,
13021                } = relayed;
13022                assert!(
13023                    task.await.unwrap().is_empty(),
13024                    "the open is answered by commit"
13025                );
13026                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
13027                Bound {
13028                    target,
13029                    client,
13030                    client_rx,
13031                    channel,
13032                    epoch,
13033                    bind,
13034                }
13035            }
13036
13037            fn live(&self, route: &Bound) -> bool {
13038                matches!(
13039                    self.forwarding
13040                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13041                        .unwrap(),
13042                    DataRoute::Client(DataRouteState::Bound(_))
13043                )
13044            }
13045        }
13046
13047        struct Relayed {
13048            target: String,
13049            client: RouteCtx,
13050            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13051            task: tokio::task::JoinHandle<Vec<Frame>>,
13052            bind: Frame,
13053        }
13054
13055        struct Bound {
13056            target: String,
13057            client: RouteCtx,
13058            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13059            channel: u16,
13060            epoch: u32,
13061            bind: Frame,
13062        }
13063
13064        impl Bound {
13065            /// The reason of the `route.closed` this client was sent, after
13066            /// checking it also got a GOODBYE on exactly this route.
13067            fn closed_reason(&mut self) -> RouteCloseReason {
13068                let mut reason = None;
13069                let mut goodbye = false;
13070                while let Ok(outbound) = self.client_rx.try_recv() {
13071                    let frame = outbound.frame;
13072                    match frame.header.ty {
13073                        FrameType::Goodbye => {
13074                            assert_eq!(
13075                                (frame.header.channel, frame.header.epoch),
13076                                (self.channel, self.epoch)
13077                            );
13078                            goodbye = true;
13079                        }
13080                        FrameType::Push => {
13081                            let ClientControlPush::RouteClosed {
13082                                reason: r,
13083                                module_id,
13084                                ..
13085                            } = serde_json::from_slice(&frame.body).unwrap()
13086                            else {
13087                                panic!("unexpected push");
13088                            };
13089                            assert_eq!(module_id, self.target);
13090                            reason = Some(r);
13091                        }
13092                        other => panic!("unexpected frame {other:?}"),
13093                    }
13094                }
13095                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
13096                reason.expect("the client is told why the route closed")
13097            }
13098
13099            fn untouched(&mut self) -> bool {
13100                self.client_rx.try_recv().is_err()
13101            }
13102
13103            fn stamp(&self) -> Option<ScopeStamp> {
13104                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
13105                    ModuleControlRequest::RouteBind { scope, .. } => scope,
13106                    other => panic!("expected a route.bind, got {other:?}"),
13107                }
13108            }
13109        }
13110
13111        #[tokio::test]
13112        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
13113        ) {
13114            let mut rig = rig().await;
13115            let mut record = session(1);
13116            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
13117            record.child_owners = vec![Principal::Reserved {
13118                module_id: MAGIC.to_string(),
13119            }];
13120            rig.sync(vec![record]).await;
13121            let scope = || Some(rig_selector("s", Some(1)));
13122
13123            // Admitted: the owner, a bare carrier to any module, a targeted
13124            // carrier to its listed module.
13125            rig.bound(Some(OWNER), PLEXUS, scope()).await;
13126            rig.bound(Some(AFT), OTHER, scope()).await;
13127            rig.bound(Some(BROCA), PLEXUS, scope()).await;
13128
13129            // Refused scope_not_carrier: a targeted carrier to an unlisted
13130            // module, a module that is not listed at all (a child owner is not
13131            // a carrier), and a direct key-holder.
13132            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
13133                assert_eq!(
13134                    rig.refused(opener, target, scope()).await,
13135                    error_codes::SCOPE_NOT_CARRIER,
13136                    "{opener:?} -> {target}"
13137                );
13138            }
13139        }
13140
13141        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13142            ScopeSelector {
13143                owner: Principal::Reserved {
13144                    module_id: OWNER.to_string(),
13145                },
13146                scope_ref: scope_ref.to_string(),
13147                scope_epoch,
13148            }
13149        }
13150
13151        #[tokio::test]
13152        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
13153            let mut rig = rig().await;
13154            rig.sync(vec![session(1)]).await;
13155            for opener in [OWNER, AFT] {
13156                assert_eq!(
13157                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
13158                        .await,
13159                    error_codes::SCOPE_EPOCH_REQUIRED,
13160                    "{opener}"
13161                );
13162            }
13163        }
13164
13165        #[tokio::test]
13166        async fn admission_separates_not_synced_not_live_and_ended() {
13167            let mut rig = rig().await;
13168            // Before the configured owner's first sync: retryable.
13169            let code = rig
13170                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13171                .await;
13172            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
13173            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
13174
13175            // An owner that is not configured will never sync: terminal.
13176            let ghost = ScopeSelector {
13177                owner: Principal::Reserved {
13178                    module_id: "ghost".to_string(),
13179                },
13180                scope_ref: "s".to_string(),
13181                scope_epoch: Some(1),
13182            };
13183            assert_eq!(
13184                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
13185                error_codes::SCOPE_NOT_LIVE
13186            );
13187
13188            rig.sync(vec![session(2)]).await;
13189            assert_eq!(
13190                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
13191                    .await,
13192                error_codes::SCOPE_NOT_LIVE
13193            );
13194            for epoch in [1, 3] {
13195                assert_eq!(
13196                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
13197                        .await,
13198                    error_codes::SCOPE_ENDED,
13199                    "epoch {epoch}"
13200                );
13201            }
13202            // Control: the live epoch is admitted.
13203            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
13204                .await;
13205        }
13206
13207        #[tokio::test]
13208        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
13209            let mut rig = rig().await;
13210            rig.sync(vec![session(1)]).await;
13211            let route = rig
13212                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13213                .await;
13214            let stamp = route.stamp().expect("a scoped bind carries the stamp");
13215            assert_eq!(stamp.scope_ref, "s");
13216            assert_eq!(stamp.scope_epoch, 1);
13217            assert_eq!(stamp.kind, ScopeKind::Head);
13218            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
13219            assert!(stamp.attributes.delegates);
13220            assert!(stamp.owner_authorized);
13221            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13222            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
13223
13224            // broca owns a scope of its own on its own module connection; it is
13225            // not in scope_authority_owners, so its stamp is not authorized.
13226            let (broca, mut broca_rx) = wide_ctx(50);
13227            hello_via_sink(
13228                &rig.handler,
13229                &broca,
13230                &mut broca_rx,
13231                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13232            )
13233            .await;
13234            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13235                .await
13236                .unwrap();
13237            let own = ScopeSelector {
13238                owner: Principal::Reserved {
13239                    module_id: BROCA.to_string(),
13240                },
13241                scope_ref: "b".to_string(),
13242                scope_epoch: Some(1),
13243            };
13244            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13245            assert!(!route.stamp().unwrap().owner_authorized);
13246        }
13247
13248        #[tokio::test]
13249        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13250            let mut rig = rig().await;
13251            let mut record = head("s", 1);
13252            record.carriers = vec![carrier(AFT, None)];
13253            let flow_id = "Flow:run-7/step_2!~";
13254            record.attributes.flow_id = Some(flow_id.to_string());
13255            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13256                .await
13257                .unwrap();
13258            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13259                panic!("not a sync reply");
13260            };
13261            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
13262            let route = rig
13263                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13264                .await;
13265            assert!(rig.live(&route), "the stamped bind committed");
13266            let stamp = route.stamp().expect("a flow scope carries a stamp");
13267            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
13268            assert_eq!(stamp.attributes.agent_id, None);
13269            assert!(!stamp.attributes.delegates);
13270            assert!(stamp.owner_authorized);
13271            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13272            assert_eq!(unscoped.stamp(), None);
13273        }
13274
13275        #[tokio::test]
13276        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
13277            let mut rig = rig_with_flow_support(false).await;
13278            let mut record = session(1);
13279            record.attributes.flow_id = Some("flow:7".to_string());
13280            rig.sync(vec![record]).await;
13281            for opener in [OWNER, AFT] {
13282                let body = rig
13283                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13284                    .await;
13285                assert_eq!(body["code"], "target_flow_unsupported");
13286                let message = body["message"].as_str().unwrap();
13287                for required in [PLEXUS, "flow-scopes/v1"] {
13288                    assert!(message.contains(required), "{message}");
13289                }
13290            }
13291        }
13292
13293        #[tokio::test]
13294        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
13295            let mut rig = rig_with_flow_support(true).await;
13296            let mut record = session(1);
13297            record.attributes.flow_id = Some("flow:7".to_string());
13298            rig.sync(vec![record]).await;
13299            for opener in [OWNER, AFT] {
13300                let route = rig
13301                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13302                    .await;
13303                assert!(rig.live(&route));
13304                assert_eq!(
13305                    route.stamp().unwrap().attributes.flow_id.as_deref(),
13306                    Some("flow:7")
13307                );
13308            }
13309        }
13310
13311        #[tokio::test]
13312        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
13313            use std::future::Future;
13314
13315            let mut rig = rig_with_flow_support(true).await;
13316            let mut record = session(1);
13317            record.attributes.flow_id = Some("flow:7".to_string());
13318            rig.sync(vec![record]).await;
13319            let (client, mut client_rx, frame) =
13320                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
13321            // Hold the route-open response permit so admission sees the first
13322            // target but relay reservation cannot capture an endpoint yet.
13323            for _ in 0..64 {
13324                client.egress.try_send(route_bind_ack(1)).unwrap();
13325            }
13326            let handler = rig.handler.clone();
13327            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
13328            std::future::poll_fn(|cx| {
13329                assert!(open.as_mut().poll(cx).is_pending());
13330                std::task::Poll::Ready(())
13331            })
13332            .await;
13333            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13334
13335            let old_connection = rig.modules[PLEXUS].0.connection_id;
13336            rig.handler.cleanup_connection(old_connection).unwrap();
13337            let (replacement, mut replacement_rx) = wide_ctx(200);
13338            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
13339            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13340            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13341            let hello = Frame::build(
13342                FrameType::Hello,
13343                control_flags(),
13344                0,
13345                0,
13346                200,
13347                serde_json::to_vec(&body).unwrap(),
13348            )
13349            .unwrap();
13350            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
13351
13352            client_rx.try_recv().unwrap();
13353            let replies = tokio::time::timeout(Duration::from_secs(2), open)
13354                .await
13355                .expect("the replacement is refused without waiting for a bind ack")
13356                .unwrap();
13357            assert_eq!(replies.len(), 1);
13358            let body = parse_error(&replies[0]);
13359            assert_eq!(body["code"], "target_flow_unsupported");
13360            for required in [PLEXUS, "flow-scopes/v1"] {
13361                assert!(body["message"].as_str().unwrap().contains(required));
13362            }
13363            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
13364            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
13365            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13366            assert!(rig
13367                .handler
13368                .registry
13369                .get_module_by_connection(replacement.connection_id)
13370                .unwrap()
13371                .is_some());
13372        }
13373
13374        #[tokio::test]
13375        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
13376        {
13377            let mut rig = rig_with_flow_support(false).await;
13378            rig.sync(vec![session(1)]).await;
13379            let route = rig
13380                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13381                .await;
13382            assert!(rig.live(&route));
13383            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
13384            let mut record = session(1);
13385            record.attributes.flow_id = Some("flow:7".to_string());
13386            rig.sync(vec![record]).await;
13387            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
13388            assert!(rig.live(&unscoped));
13389            assert_eq!(unscoped.stamp(), None);
13390        }
13391
13392        #[tokio::test]
13393        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
13394            let mut rig = rig().await;
13395            let mut record = session(1);
13396            record.attributes.flow_id = Some("flow:7".to_string());
13397            rig.sync(vec![record.clone()]).await;
13398            let before = rig.forwarding.published_scope_tag(OWNER, "s");
13399            let mut owner_route = rig
13400                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13401                .await;
13402            let mut carrier_route = rig
13403                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
13404                .await;
13405            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13406            rig.sync(vec![record.clone()]).await;
13407            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13408            assert!(rig.live(&owner_route) && owner_route.untouched());
13409            assert!(rig.live(&carrier_route) && carrier_route.untouched());
13410
13411            record.attributes.flow_id = Some("flow:8".to_string());
13412            rig.sync(vec![record]).await;
13413            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
13414            let before = before.unwrap();
13415            assert_eq!(after.scope_epoch, before.scope_epoch);
13416            assert!(after.version > before.version);
13417            for route in [&mut owner_route, &mut carrier_route] {
13418                assert!(!rig.live(route));
13419                assert_eq!(
13420                    route.closed_reason(),
13421                    RouteCloseReason::ScopeDelegationChanged
13422                );
13423            }
13424            assert!(rig.live(&unscoped) && unscoped.untouched());
13425            // Each provider also receives a GOODBYE for its drained route;
13426            // consume it before expecting the next route.bind on that sink.
13427            for target in [PLEXUS, OTHER] {
13428                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
13429                let goodbye = module_rx
13430                    .try_recv()
13431                    .expect("the provider sees the drain")
13432                    .frame;
13433                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13434                assert!(module_rx.try_recv().is_err());
13435            }
13436            let rebound = rig
13437                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13438                .await;
13439            assert_eq!(
13440                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
13441                Some("flow:8")
13442            );
13443        }
13444
13445        /// The owner's sync lands between admission and the module's ack. The
13446        /// open is refused by name, the module's other routes stay up, and the
13447        /// reserved pair is released. Changed content is retryable; an ended
13448        /// scope is not.
13449        #[tokio::test]
13450        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
13451            let mut rig = rig().await;
13452            rig.sync(vec![session(1)]).await;
13453            let mut cotenant = rig
13454                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13455                .await;
13456
13457            let mut changed = session(1);
13458            changed.child_owners.push(Principal::Reserved {
13459                module_id: MAGIC.to_string(),
13460            });
13461            let mut ended = None;
13462            for (code, next) in [
13463                (error_codes::SCOPE_CHANGED, vec![changed]),
13464                (error_codes::SCOPE_ENDED, Vec::new()),
13465            ] {
13466                let relayed = rig
13467                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13468                    .await;
13469                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
13470                ended = Some(next.is_empty());
13471                rig.sync(next).await;
13472                rig.ack(&relayed).await;
13473                let replies = relayed.task.await.unwrap();
13474                assert_eq!(replies.len(), 1, "{replies:?}");
13475                assert_eq!(parse_error(&replies[0])["code"], code);
13476                assert_eq!(
13477                    subc_protocol::error_codes::is_retryable_route_open(code),
13478                    code == error_codes::SCOPE_CHANGED
13479                );
13480                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13481                // The module is told to drop just the binding it created.
13482                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13483                // Collected, because ending the scope also closes the co-tenant
13484                // route, whose GOODBYE comes first.
13485                let mut goodbyes = Vec::new();
13486                while let Ok(outbound) = plexus_rx.try_recv() {
13487                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
13488                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
13489                }
13490                assert!(
13491                    goodbyes.contains(&(bind_channel, bind_epoch)),
13492                    "{goodbyes:?}"
13493                );
13494                assert!(rig
13495                    .handler
13496                    .registry
13497                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
13498                    .unwrap()
13499                    .is_some());
13500            }
13501            assert_eq!(ended, Some(true));
13502            // The co-tenant stayed up through the change, and closed only when
13503            // the scope ended, by the drain rule rather than by the commit.
13504            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
13505        }
13506
13507        /// Each row of the drain table on one set of routes: the owner's, a
13508        /// bare carrier's, and a targeted carrier's to each of its targets.
13509        #[tokio::test]
13510        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
13511            struct Case {
13512                name: &'static str,
13513                change: fn(&mut ScopeRecord),
13514                /// Closed routes by index: owner->plexus, aft->plexus,
13515                /// broca->plexus, broca->other.
13516                closed: [Option<RouteCloseReason>; 4],
13517            }
13518            use RouteCloseReason::*;
13519            let cases = [
13520                Case {
13521                    name: "a carrier entry removed",
13522                    change: |r| {
13523                        r.carriers.retain(|c| {
13524                            c.principal
13525                                != Principal::Reserved {
13526                                    module_id: AFT.to_string(),
13527                                }
13528                        })
13529                    },
13530                    closed: [None, Some(ScopeCarrierRemoved), None, None],
13531                },
13532                Case {
13533                    name: "a target removed from a carrier",
13534                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
13535                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
13536                },
13537                Case {
13538                    name: "a bare carrier narrowed to targets",
13539                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
13540                    closed: [None, Some(ScopeCarrierRemoved), None, None],
13541                },
13542                Case {
13543                    name: "delegates turned off",
13544                    change: |r| r.attributes.delegates = false,
13545                    closed: [Some(ScopeDelegationChanged); 4],
13546                },
13547                Case {
13548                    name: "agent_id changed",
13549                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
13550                    closed: [Some(ScopeDelegationChanged); 4],
13551                },
13552                Case {
13553                    name: "a carrier added, child owners changed, the record re-sent",
13554                    change: |r| {
13555                        r.carriers.push(carrier(MAGIC, None));
13556                        r.child_owners.push(Principal::Reserved {
13557                            module_id: MAGIC.to_string(),
13558                        });
13559                    },
13560                    closed: [None; 4],
13561                },
13562                Case {
13563                    name: "a target added",
13564                    change: |r| {
13565                        r.carriers[1]
13566                            .targets
13567                            .as_mut()
13568                            .unwrap()
13569                            .push("third".to_string())
13570                    },
13571                    closed: [None; 4],
13572                },
13573                Case {
13574                    name: "delegates turned on",
13575                    change: |r| r.attributes.delegates = true,
13576                    closed: [None; 4],
13577                },
13578            ];
13579            for case in cases {
13580                let mut rig = rig().await;
13581                rig.sync(vec![session(1)]).await;
13582                let scope = || Some(rig_selector("s", Some(1)));
13583                let mut routes = [
13584                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
13585                    rig.bound(Some(AFT), PLEXUS, scope()).await,
13586                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
13587                    rig.bound(Some(BROCA), OTHER, scope()).await,
13588                ];
13589                let mut record = session(1);
13590                (case.change)(&mut record);
13591                rig.sync(vec![record]).await;
13592                for (index, expected) in case.closed.iter().enumerate() {
13593                    let route = &mut routes[index];
13594                    match expected {
13595                        Some(reason) => {
13596                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
13597                            assert_eq!(
13598                                route.closed_reason(),
13599                                *reason,
13600                                "{}: route {index}",
13601                                case.name
13602                            );
13603                        }
13604                        None => {
13605                            assert!(rig.live(route), "{}: route {index} closed", case.name);
13606                            assert!(
13607                                route.untouched(),
13608                                "{}: route {index} was told something",
13609                                case.name
13610                            );
13611                        }
13612                    }
13613                }
13614            }
13615        }
13616
13617        #[tokio::test]
13618        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
13619            // Removed, and replaced by a higher epoch.
13620            for next in [Vec::new(), vec![session(2)]] {
13621                let mut rig = rig().await;
13622                rig.sync(vec![session(1)]).await;
13623                let mut route = rig
13624                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13625                    .await;
13626                rig.sync(next).await;
13627                assert!(!rig.live(&route));
13628                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
13629            }
13630
13631            // A child whose parent ends: its routes close as parent-ended, the
13632            // child stays live, and routes under the parent close as ended.
13633            let mut rig = rig().await;
13634            let mut child = session(1);
13635            child.scope_ref = "child".to_string();
13636            child.kind = ScopeKind::Worker;
13637            child.parent = Some(ScopeParent {
13638                owner: Principal::Reserved {
13639                    module_id: OWNER.to_string(),
13640                },
13641                scope_ref: "s".to_string(),
13642                scope_epoch: 1,
13643            });
13644            rig.sync(vec![session(1), child.clone()]).await;
13645            let mut child_route = rig
13646                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
13647                .await;
13648            assert_eq!(
13649                child_route.stamp().unwrap().parent_state,
13650                Some(ParentState::Linked)
13651            );
13652            rig.sync(vec![child]).await;
13653            assert!(!rig.live(&child_route));
13654            assert_eq!(
13655                child_route.closed_reason(),
13656                RouteCloseReason::ScopeParentEnded
13657            );
13658        }
13659
13660        #[tokio::test]
13661        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
13662        ) {
13663            let mut rig = rig().await;
13664            rig.sync(vec![session(1)]).await;
13665            let mut route = rig
13666                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13667                .await;
13668            let before = rig.forwarding.published_scope_tag(OWNER, "s");
13669            rig.sync(vec![session(1)]).await;
13670            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13671            assert!(rig.live(&route) && route.untouched());
13672
13673            // A call in flight on the route when another carrier is added. A
13674            // forwarded REQUEST holds one credit on the route's flow until the
13675            // module answers; the router takes it exactly like this.
13676            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
13677                .forwarding
13678                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13679                .unwrap()
13680            else {
13681                panic!("the route is bound");
13682            };
13683            binding.flow.acquire_tagged(9, false).await.unwrap();
13684            let mut widened = session(1);
13685            widened.carriers.push(carrier(MAGIC, None));
13686            rig.sync(vec![widened]).await;
13687            assert!(rig.live(&route) && route.untouched());
13688            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13689            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
13690            // The call's credit is still held on an open flow, so its answer
13691            // will be delivered: closing the route would have closed the flow.
13692            assert_eq!(binding.flow.in_flight(), 1);
13693            binding
13694                .flow
13695                .acquire_tagged(10, false)
13696                .await
13697                .expect("the flow is still open");
13698        }
13699
13700        /// A swap's superseded endpoint keeps its routes until drained; ending
13701        /// the scope closes them there too.
13702        #[tokio::test]
13703        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
13704            let mut rig = rig().await;
13705            rig.sync(vec![session(1)]).await;
13706            let mut on_incumbent = rig
13707                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13708                .await;
13709
13710            // Swap plexus: register a candidate and cut over, leaving the
13711            // incumbent superseded with the route still on it.
13712            let (candidate, _candidate_rx) = wide_ctx(9);
13713            let registration = rig
13714                .handler
13715                .registry
13716                .register_candidate_with_control_ops(
13717                    manifest(PLEXUS, PROTOCOL_VERSION),
13718                    PROTOCOL_VERSION,
13719                    candidate.connection_id,
13720                    module_baseline_control_ops(),
13721                )
13722                .unwrap();
13723            rig.forwarding
13724                .register_candidate_module_connection(
13725                    candidate.connection_id,
13726                    PLEXUS.to_string(),
13727                    PROTOCOL_VERSION,
13728                    manifest_concurrency(&registration.manifest),
13729                    candidate.egress.clone(),
13730                )
13731                .unwrap();
13732            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
13733            rig.handler
13734                .registry
13735                .promote_candidate(PLEXUS)
13736                .unwrap()
13737                .unwrap();
13738            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
13739
13740            rig.sync(Vec::new()).await;
13741            assert!(!rig.live(&on_incumbent));
13742            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
13743            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13744            let goodbye = incumbent_rx
13745                .try_recv()
13746                .expect("the superseded endpoint is told")
13747                .frame;
13748            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13749        }
13750    }
13751}
13752
13753#[cfg(test)]
13754mod concurrency_default_exposure_tests {
13755    use super::*;
13756
13757    fn hello_body(role_json: &str) -> Vec<u8> {
13758        format!(
13759            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
13760        )
13761        .into_bytes()
13762    }
13763
13764    fn manifest_from(body: &[u8]) -> ModuleManifest {
13765        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
13766        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
13767            .expect("manifest parses")
13768    }
13769
13770    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
13771
13772    #[test]
13773    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
13774        let body = hello_body(&format!(
13775            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
13776        ));
13777        let manifest = manifest_from(&body);
13778        // Precondition: serde really resolved it to the default, so the typed
13779        // manifest alone cannot answer the question this probe exists for.
13780        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13781        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
13782    }
13783
13784    #[test]
13785    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
13786        let body = hello_body(&format!(
13787            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
13788        ));
13789        let manifest = manifest_from(&body);
13790        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13791        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13792    }
13793
13794    #[test]
13795    fn non_management_roles_are_never_reported() {
13796        let body = hello_body(
13797            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
13798        );
13799        let manifest = manifest_from(&body);
13800        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13801    }
13802}