Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::{Path, PathBuf},
5    sync::{Arc, Mutex, RwLock},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20    error_codes,
21    manifest::{
22        validate_hello_capability_grammar, validate_hello_event_declarations,
23        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
24        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
25    },
26    scope::{
27        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
29        SCOPE_SYNC_OP,
30    },
31    session::{
32        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
33        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
34        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
35        ROLE_VERSIONS_FIELD,
36    },
37    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
38    Priority, RouteTarget, PROTOCOL_VERSION,
39};
40use tokio::time::{timeout_at, Instant};
41use tracing::{debug, info, warn};
42
43use crate::{
44    capability_requirements::{
45        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
46        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
47        RuntimeModule,
48    },
49    daemon_config::RestartRequiredSection,
50    forwarding::{
51        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
52        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
53        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
54    },
55    observability::{
56        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
57    },
58    provenance::{
59        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
60    },
61    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
62    router::{RouteCtx, RouterError},
63    scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
64    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
65    stderr_tail::{CaptureState, TailEntry},
66    supervise::{
67        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
68        SupervisorHandle, SwapHelloAdmission,
69    },
70    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
71};
72
73/// Lowest envelope version this subc build will negotiate.
74///
75/// Module HELLO negotiation is exact: peers must use the daemon's locked
76/// protocol version. Older and newer peers receive `version_unsupported` and
77/// are not registered.
78pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
79
80const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
81const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
82const CAP_PING_PONG: &str = "ping_pong_v1";
83const CAP_SESSION_ATTACH: &str = "session_attach_v1";
84const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
85
86const SUBC_CONTROL_OPS: &[&str] = &[
87    ops::ACTIVITY_RECORD,
88    ops::SERVER_DESCRIBE,
89    ops::CATALOG_LIST,
90    ops::ROUTE_OPEN,
91    ops::ROUTE_POLL,
92    ops::ROUTE_CLOSING,
93    ops::ROUTE_CLOSED,
94    ops::SUPERVISOR_LIST,
95    ops::SUPERVISOR_RESTART,
96    ops::SUPERVISOR_SWAP,
97    ops::SUPERVISOR_RELOAD,
98    ops::SUPERVISOR_RESCAN,
99    ops::SUPERVISOR_RELEASE_RESERVED,
100    ops::SUPERVISOR_SET_ENABLED,
101    ops::SUPERVISOR_HEALTH_PROBE,
102    ops::SUPERVISOR_HEALTH,
103    ops::SUPERVISOR_STDERR_TAIL,
104    ops::SUPERVISOR_TERMINALS,
105    ops::SUPERVISOR_ROUTES,
106    ops::SUPERVISOR_PROVENANCE,
107    ops::SUPERVISOR_SPAWN_SNAPSHOT,
108    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
109];
110
111const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
112    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
113    "supervisor.live_roots",
114    SCOPE_SYNC_OP,
115    SCOPE_APPLY_OP,
116    SCOPE_DESCRIBE_OP,
117    "operator.confirm",
118];
119
120/// Module-originated ops the daemon answers but does not advertise in
121/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
122/// belongs to is incomplete, so no module is told it works before it does.
123const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
124
125const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
126
127/// How long subc waits for a module to ack a relayed route.bind before returning
128/// `module_timeout`. The ack waits on the module's own configure, which for AFT
129/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
130/// and DB-open work — on a cold page cache or a large repo that legitimately
131/// exceeds a couple of seconds. The default is generous because rejecting a VALID
132/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
133/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
134pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
135
136/// How many CONSECUTIVE full-budget relay timeouts against one target module
137/// open that module's bind-relay breaker.
138///
139/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
140/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
141/// so three consecutive full-budget timeouts take ~36s to observe: every client
142/// whose open contributed to opening the breaker had already given up on its
143/// own. That is what makes opening the breaker unable to turn a call that would
144/// have succeeded into a refusal — it can only make an already-failing module
145/// fail faster.
146///
147/// Two would be reachable inside one default deadline. One would convict a
148/// module on a single cold-cache bind, which is exactly the valid-but-slow case
149/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
150pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
151
152/// How long a module's bind-relay breaker stays open before exactly one
153/// `route.open` is let through as a probe.
154///
155/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
156/// re-pays a full-budget stall almost continuously, and the breaker stops being
157/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
158/// deadline: a client that starts retrying after the module recovers has to get
159/// a probe opportunity inside its own deadline, or the breaker converts a
160/// recovered module into a failed call — the failure it exists to prevent,
161/// pointed the other way.
162///
163/// 20s sits between those with room on both sides, and it caps what a wedged
164/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
165/// rather than one per `route.open` per connection. The stall that motivated
166/// this, with its measurements, is written up in
167/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
168/// waited the whole budget out.
169pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
170
171const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
172const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
173
174fn reload_verdict(
175    configured: &Path,
176    spawned_from: Option<&Path>,
177    image: subc_control::RunningImageAgreement,
178) -> PendingReloadVerdict {
179    let path = match spawned_from {
180        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
181            ReloadPathAgreement::Match
182        }
183        Some(spawned_from) => ReloadPathAgreement::Mismatch {
184            configured: configured.to_path_buf(),
185            spawned_from: spawned_from.to_path_buf(),
186        },
187        None => ReloadPathAgreement::Unavailable {
188            reason: if matches!(
189                image,
190                subc_control::RunningImageAgreement::Unavailable {
191                    reason: subc_control::RunningImageUnavailableReason::NotRunning
192                }
193            ) {
194                ReloadPathUnavailableReason::NotRunning
195            } else {
196                ReloadPathUnavailableReason::SpawnedPathUnavailable
197            },
198        },
199    };
200    PendingReloadVerdict { path, image }
201}
202
203fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
204    if configured == spawned_from {
205        return true;
206    }
207    #[cfg(windows)]
208    {
209        // Windows spawn records the resolved verbatim path (\\?\...), while
210        // configuration keeps its original spelling. Resolve both with the OS
211        // instead of stripping prefixes or folding case, which could conflate
212        // distinct names in case-sensitive directories. Different resolved
213        // paths still differ even if their files have identical content.
214        match (
215            std::fs::canonicalize(configured),
216            std::fs::canonicalize(spawned_from),
217        ) {
218            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
219            _ => false,
220        }
221    }
222    #[cfg(not(windows))]
223    {
224        false
225    }
226}
227
228#[derive(Clone)]
229struct DaemonProvenanceFacts {
230    build: DaemonBuildProvenance,
231    pid: Option<u32>,
232    started_at_ms: Option<u64>,
233    start_clock: Option<crate::clock::StartClock>,
234    executable_path: Option<PathBuf>,
235    executable_identity: Option<SpawnedFileIdentity>,
236    process_start_time: Option<u64>,
237    probe: ExecutableIdentityProbe,
238}
239
240impl Default for DaemonProvenanceFacts {
241    fn default() -> Self {
242        Self {
243            build: DaemonBuildProvenance {
244                build_git_sha: None,
245                build_lock_digest: None,
246            },
247            pid: None,
248            started_at_ms: None,
249            start_clock: None,
250            executable_path: None,
251            executable_identity: None,
252            process_start_time: None,
253            probe: ExecutableIdentityProbe::default(),
254        }
255    }
256}
257
258#[derive(Debug, Clone)]
259struct SupervisorRescanContext {
260    supervisor: Supervisor,
261    config_path: PathBuf,
262    configured_port: Option<u16>,
263    storage_config: Option<crate::daemon_config::StorageConfig>,
264    admission_facts_carrier_module_id: Option<String>,
265    admission_facts_targets: Option<Vec<String>>,
266    scope_authority_owners: Vec<String>,
267}
268
269/// Refusal labels passed to `observe_route_open_refusal` that mean the target
270/// module is not serving right now, and so open or extend an outage in the
271/// route outage tracker. Every one of them is only reachable after the target
272/// was found in the registry, which is what keeps an arbitrary client-chosen
273/// id from ever creating tracker state.
274///
275/// Deliberately absent: `not_registered` and `removed` (the id may be
276/// anything a client sent, and a removed module is gone on purpose),
277/// `protocol_none` (such a module never serves routes, so nothing is out),
278/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
279/// capability and admission-facts refusals (they refuse the caller, not a
280/// module outage), and `relay_reservation_failed` (its code ranges over
281/// capacity limits as well as a vanished connection). Capacity, breaker,
282/// relay-timeout and module-rejection refusals do not pass through that
283/// function at all; the breaker logs its own transitions.
284///
285/// The two not-serving refusals that bypass that function record themselves
286/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
287/// `required_capability_unprovided` is not tracked: the module itself is up,
288/// and the outage belongs to the missing provider.
289const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
290    "reloading",
291    "supervisor_not_live",
292    "registration_not_active",
293    "no_forwarding_connection",
294    "relay_send_failed",
295];
296
297/// Real channel-0 control handler for subc itself.
298#[derive(Clone)]
299pub struct ControlHandler {
300    registry: Arc<Registry>,
301    activity: Option<crate::activity::ActivityLedger>,
302    forwarding: Arc<ForwardingTable>,
303    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
304    supervisor: SupervisorHandle,
305    subc_capabilities: Arc<[String]>,
306    /// Daemon-wide route.bind relay budget. Used as the fallback when the
307    /// target module has no per-module override in
308    /// `route_bind_relay_timeouts`.
309    route_bind_relay_timeout: Duration,
310    /// Per-module route.bind relay budget overrides, keyed by module id. When
311    /// `handle_route_open` resolves the deadline for a target module, a
312    /// per-module entry wins over the daemon-wide value above.
313    route_bind_relay_timeouts: BTreeMap<String, Duration>,
314    /// Per-target-module bind-relay breaker state. Shared with the forwarding
315    /// table, which is where a new module connection resets it.
316    route_bind_breakers: RouteBindBreakers,
317    /// Live relay admissions keyed by target module. Shared through the
318    /// forwarding table so cloned or separately built handlers enforce one cap.
319    route_bind_concurrency: RouteBindConcurrency,
320    /// Start and end of each module's not-serving period as seen by
321    /// `route.open`, so an outage gets one line at each edge instead of only
322    /// the per-refusal INFO lines. Taken from the forwarding table, so every
323    /// handler built over one table shares it.
324    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
325    /// Consecutive relay timeouts that open a module's breaker.
326    route_bind_breaker_threshold: u32,
327    /// How long a breaker stays open before one probe is admitted.
328    route_bind_breaker_cooldown: Duration,
329    health_probe_timeout: Duration,
330    /// Central storage policy. When set, each registering module receives its
331    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
332    storage_config: Option<crate::daemon_config::StorageConfig>,
333    /// The machine id established at boot, served on every HELLO_ACK and on
334    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
335    /// changes the file, never this value. `None` serves no id.
336    machine_id: Option<crate::machine_id::MachineId>,
337    admission_facts_carrier_module_id: Option<String>,
338    admission_facts_targets: Option<Vec<String>>,
339    /// Scope records with their sync authorities and tombstones; see
340    /// `crate::scopes`. Shared by clones of this handler, so every connection
341    /// reads and writes one table.
342    scopes: Arc<RwLock<ScopeTable>>,
343    /// Wakes the expiry loop when an accepted scope change may have added a
344    /// deadline, so the loop can wait without ticking while no live scope has
345    /// one. A permit is stored when nothing waits, so no wake is lost.
346    scope_deadline_added: Arc<tokio::sync::Notify>,
347    /// The configured `scope_authority_owners`, kept so a rescan can report a
348    /// changed value as needing a daemon restart; rescan never applies it.
349    scope_authority_owners: Vec<String>,
350    /// The launch nonce each module connection presented at HELLO, which is how
351    /// a `scope.sync` is matched to the owner's current launch.
352    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
353    rescan: Option<SupervisorRescanContext>,
354    connected_clients: ConnectedClients,
355    counters: DaemonCounters,
356    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
357    daemon_provenance: DaemonProvenanceFacts,
358    #[cfg(test)]
359    control_dispatch_delay: Option<Duration>,
360    #[cfg(test)]
361    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
362    /// Holds `route.open` after the route.bind is queued to the module and
363    /// before it is marked enqueued, so a test can make the module's reply (or
364    /// a client teardown) land exactly inside that window.
365    #[cfg(test)]
366    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
367}
368
369impl fmt::Debug for ControlHandler {
370    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
371        f.debug_struct("ControlHandler")
372            .field("registry", &self.registry)
373            .field("forwarding", &self.forwarding)
374            .field("process_liveness", &self.process_liveness.is_some())
375            .field("supervisor", &self.supervisor)
376            .field("subc_capabilities", &self.subc_capabilities)
377            .finish()
378    }
379}
380
381struct RouteOpenRequest {
382    target: RouteTarget,
383    identity: BindIdentity,
384    consumer_identity: Option<ConsumerIdentity>,
385    consumer_capabilities: Option<Vec<String>>,
386    role_versions: Option<BTreeMap<String, String>>,
387    admission_facts: Option<serde_json::Value>,
388    scope: Option<ScopeSelector>,
389}
390
391struct RouteBindReservationGuard {
392    forwarding: Arc<ForwardingTable>,
393    endpoint: ModuleEndpointId,
394    relay_corr: u64,
395    armed: bool,
396}
397
398struct ModuleControlRpcGuard {
399    forwarding: Arc<ForwardingTable>,
400    endpoint: ModuleEndpointId,
401    corr: u64,
402    armed: bool,
403}
404
405impl ModuleControlRpcGuard {
406    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
407        Self {
408            forwarding,
409            endpoint,
410            corr,
411            armed: true,
412        }
413    }
414
415    fn disarm(&mut self) {
416        self.armed = false;
417    }
418}
419
420impl Drop for ModuleControlRpcGuard {
421    fn drop(&mut self) {
422        if self.armed {
423            let _ = self
424                .forwarding
425                .cancel_module_control_rpc(self.endpoint, self.corr);
426        }
427    }
428}
429
430impl RouteBindReservationGuard {
431    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
432        Self {
433            forwarding,
434            endpoint,
435            relay_corr,
436            armed: true,
437        }
438    }
439
440    fn release_and_disarm(&mut self) {
441        if !self.armed {
442            return;
443        }
444        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
445            self.endpoint,
446            self.relay_corr,
447            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
448        ) {
449            send_goodbye_target_best_effort(
450                &self.forwarding.counters(),
451                &target,
452                "canceled route.bind",
453            );
454        }
455        self.armed = false;
456    }
457
458    fn disarm(&mut self) {
459        self.armed = false;
460    }
461}
462
463impl Drop for RouteBindReservationGuard {
464    fn drop(&mut self) {
465        self.release_and_disarm();
466    }
467}
468
469/// Per-target-module circuit breaker around the `route.bind` relay.
470///
471/// The connection reader is serial per connection, so a module whose `on_bind`
472/// sits on the ack blocks every LATER frame on the connections that call it,
473/// including calls to unrelated modules. This does not make any module's bind
474/// fast; it stops the daemon paying the full budget again and again for a
475/// condition it has already observed.
476///
477/// State is keyed by TARGET MODULE and shared by every connection: a wedged
478/// module wedges everyone, so what one connection learned should protect the
479/// rest.
480///
481/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
482/// relay to that module has actually timed out, and is removed again when a
483/// relay is accepted or the module reconnects, so it cannot grow with traffic
484/// or with modules that behave.
485///
486/// # Why a `std` mutex here is not the head-of-line defect again
487///
488/// Acquisition never awaits. The critical section is a hash lookup plus a few
489/// integer updates, with no I/O and no `.await` inside it, so a reader task
490/// cannot be descheduled behind it the way it can behind
491/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
492/// primitive, held for the same kind of work, as the refusal counter this very
493/// path already increments.
494///
495/// It is also NOT on the data-plane splice path: only `route.open` and module
496/// registration touch it, so bound-route frames gain no state check and no
497/// contention.
498#[derive(Debug, Clone, Default)]
499pub(crate) struct RouteBindBreakers {
500    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
501}
502
503#[derive(Debug, Clone, Default)]
504pub(crate) struct RouteBindConcurrency {
505    modules: Arc<Mutex<HashMap<String, usize>>>,
506}
507
508struct RouteBindConcurrencyGuard {
509    concurrency: RouteBindConcurrency,
510    module_id: String,
511}
512
513impl RouteBindConcurrency {
514    /// Admit without waiting. Waiting here would move the bind stall from the
515    /// module reply to a semaphore and restore reader head-of-line blocking.
516    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
517        let mut modules = self
518            .modules
519            .lock()
520            .expect("route.bind concurrency mutex poisoned");
521        let in_flight = modules.entry(module_id.to_string()).or_default();
522        if *in_flight >= limit {
523            return Err(*in_flight);
524        }
525        *in_flight += 1;
526        Ok(RouteBindConcurrencyGuard {
527            concurrency: self.clone(),
528            module_id: module_id.to_string(),
529        })
530    }
531}
532
533impl Drop for RouteBindConcurrencyGuard {
534    fn drop(&mut self) {
535        let mut modules = self
536            .concurrency
537            .modules
538            .lock()
539            .expect("route.bind concurrency mutex poisoned");
540        let remove = {
541            let in_flight = modules
542                .get_mut(&self.module_id)
543                .expect("admitted route.bind has a concurrency entry");
544            *in_flight -= 1;
545            *in_flight == 0
546        };
547        if remove {
548            modules.remove(&self.module_id);
549        }
550    }
551}
552
553#[derive(Debug, Default)]
554struct ModuleBreakerState {
555    /// Relay timeouts observed with no accepted relay in between.
556    consecutive_timeouts: u32,
557    /// `Some` while the breaker is open: the instant the cooldown expires and
558    /// the next arrival may probe. `None` means closed.
559    cooldown_until: Option<Instant>,
560    /// A half-open probe has been admitted and has not settled yet. This is
561    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
562    /// that read the cooldown, so concurrent opens arriving at the moment the
563    /// cooldown expires cannot all decide that they are the probe.
564    probe_in_flight: Option<Arc<()>>,
565}
566
567/// What the breaker decided for one `route.open`, before any relay work.
568enum RouteBindAdmission<'a> {
569    Admitted {
570        guard: RouteBindBreakerGuard<'a>,
571        /// This open is the single half-open probe, so the transition is worth
572        /// one log line.
573        probe: bool,
574    },
575    Refused {
576        consecutive_timeouts: u32,
577        /// What is left of the cooldown. Zero when the refusal is because the
578        /// one probe is already in flight rather than because the cooldown has
579        /// not elapsed.
580        retry_in: Duration,
581        probe_in_flight: bool,
582    },
583}
584
585/// An outstanding admission, which must be told how its relay settled.
586///
587/// `Drop` settles it as inconclusive, so an early return between admission and
588/// the relay -- or the whole handler being cancelled when the client
589/// disconnects -- releases a half-open probe slot instead of leaving the
590/// breaker wedged half-open with no further probes.
591struct RouteBindBreakerGuard<'a> {
592    breakers: RouteBindBreakers,
593    module_id: &'a str,
594    probe_token: Option<Arc<()>>,
595    settled: bool,
596}
597
598impl RouteBindBreakerGuard<'_> {
599    /// The module answered within the budget and took the bind. THE ONLY
600    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
601    /// breaker, which is a transition worth logging.
602    fn record_accepted(&mut self) -> bool {
603        self.settled = true;
604        self.breakers.record_accepted(self.module_id)
605    }
606
607    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
608    /// COUNTS TOWARD OPENING.
609    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
610        self.settled = true;
611        self.breakers.record_timeout(
612            self.module_id,
613            self.probe_token.as_ref(),
614            threshold,
615            cooldown,
616        )
617    }
618
619    /// Everything else: the module REJECTED the bind, its connection went away
620    /// mid-relay, or the waiter was cancelled.
621    ///
622    /// None of these is evidence that a module is slow, and each already has
623    /// its own refusal with its own code. A module that rejects a bind in
624    /// microseconds is healthy and must never be convicted for it; a module
625    /// that died has said nothing about the module that replaces it. So these
626    /// neither increment nor reset the count -- they only release a probe slot.
627    fn record_inconclusive(&mut self) {
628        self.settled = true;
629        self.breakers
630            .record_inconclusive(self.module_id, self.probe_token.as_ref());
631    }
632}
633
634impl Drop for RouteBindBreakerGuard<'_> {
635    fn drop(&mut self) {
636        if !self.settled {
637            self.breakers
638                .record_inconclusive(self.module_id, self.probe_token.as_ref());
639        }
640    }
641}
642
643/// The breaker moved to open, reported so the caller can log it outside the
644/// lock. Opening is rare and load-bearing; the refusals that follow are
645/// frequent and are counted rather than logged.
646struct BreakerOpened {
647    consecutive_timeouts: u32,
648    /// True when a failed probe re-opened an already-open breaker, which reads
649    /// very differently in a log from a first opening.
650    reopened_after_probe: bool,
651}
652
653impl RouteBindBreakers {
654    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
655        self.modules
656            .lock()
657            .expect("route.bind breaker mutex poisoned")
658    }
659
660    /// Decide whether this `route.open` may attempt its relay. Takes the map
661    /// lock and nothing else, and never awaits.
662    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
663        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
664            probe: probe_token.is_some(),
665            guard: RouteBindBreakerGuard {
666                breakers: self.clone(),
667                module_id,
668                probe_token,
669                settled: false,
670            },
671        };
672
673        let mut modules = self.lock();
674        let Some(state) = modules.get_mut(module_id) else {
675            return admitted(None);
676        };
677        let Some(cooldown_until) = state.cooldown_until else {
678            return admitted(None);
679        };
680        if state.probe_in_flight.is_some() {
681            return RouteBindAdmission::Refused {
682                consecutive_timeouts: state.consecutive_timeouts,
683                retry_in: Duration::ZERO,
684                probe_in_flight: true,
685            };
686        }
687        let now = Instant::now();
688        if now < cooldown_until {
689            return RouteBindAdmission::Refused {
690                consecutive_timeouts: state.consecutive_timeouts,
691                retry_in: cooldown_until - now,
692                probe_in_flight: false,
693            };
694        }
695        let token = Arc::new(());
696        state.probe_in_flight = Some(Arc::clone(&token));
697        admitted(Some(token))
698    }
699
700    fn record_accepted(&self, module_id: &str) -> bool {
701        self.lock()
702            .remove(module_id)
703            .is_some_and(|state| state.cooldown_until.is_some())
704    }
705
706    fn record_timeout(
707        &self,
708        module_id: &str,
709        probe_token: Option<&Arc<()>>,
710        threshold: u32,
711        cooldown: Duration,
712    ) -> Option<BreakerOpened> {
713        let mut modules = self.lock();
714        let state = modules.entry(module_id.to_string()).or_default();
715        let was_open = state.cooldown_until.is_some();
716        let was_probe = Self::owns_probe(state, probe_token);
717        if was_probe {
718            state.probe_in_flight = None;
719        }
720        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
721        if state.consecutive_timeouts < threshold {
722            return None;
723        }
724        state.cooldown_until = Some(Instant::now() + cooldown);
725        Some(BreakerOpened {
726            consecutive_timeouts: state.consecutive_timeouts,
727            reopened_after_probe: was_open && was_probe,
728        })
729    }
730
731    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
732        // A relay can finish after the breaker was reset or after it opened
733        // again and started a new probe. Only the guard whose token matches the
734        // active probe may release it, so a late relay never frees a newer probe.
735        state
736            .probe_in_flight
737            .as_ref()
738            .zip(token)
739            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
740    }
741
742    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
743        if let Some(state) = self.lock().get_mut(module_id) {
744            if Self::owns_probe(state, probe_token) {
745                state.probe_in_flight = None;
746            }
747        }
748    }
749
750    /// Discard what was learned about a module, because the process it was
751    /// learned about is gone. Returns the discarded count when it was non-zero.
752    ///
753    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
754    /// `module_id` is a configuration identity that outlives any particular
755    /// child; what the breaker observed was the process behind the module
756    /// connection of the moment. When a new connection registers under that id
757    /// the verdict's subject no longer exists, so the verdict is stale by
758    /// construction rather than merely likely to be wrong. Keeping it would
759    /// apply a dead process's record to a live one, which is the same defect
760    /// class this breaker exists to stop the daemon committing.
761    ///
762    /// A half-open probe in flight is discarded with the rest: it was a
763    /// question about the old process.
764    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
765        self.lock()
766            .remove(module_id)
767            .map(|state| state.consecutive_timeouts)
768            .filter(|discarded| *discarded > 0)
769    }
770
771    /// Open breakers, for the `server.describe` counters object. `None` when
772    /// none is open, so the key stays absent rather than present-and-empty.
773    ///
774    /// This is the operator's answer to "is this module refusing instantly or
775    /// is it fine?", which look identical from a client that retries and then
776    /// succeeds.
777    fn open_snapshot(&self) -> Option<serde_json::Value> {
778        let now = Instant::now();
779        let modules = self.lock();
780        let open = modules
781            .iter()
782            .filter_map(|(module_id, state)| {
783                let cooldown_until = state.cooldown_until?;
784                Some((
785                    module_id.clone(),
786                    serde_json::json!({
787                        "consecutive_timeouts": state.consecutive_timeouts,
788                        "cooldown_remaining_ms":
789                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
790                        "probe_in_flight": state.probe_in_flight.is_some(),
791                    }),
792                ))
793            })
794            .collect::<serde_json::Map<String, serde_json::Value>>();
795        (!open.is_empty()).then_some(serde_json::Value::Object(open))
796    }
797}
798
799impl ControlHandler {
800    pub fn new(registry: Arc<Registry>) -> Self {
801        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
802    }
803
804    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
805        let counters = forwarding.counters();
806        // Taken from the forwarding table rather than created here, so that the
807        // breaker a `route.open` consults is the same one a module's
808        // registration resets, however many handlers are built over one table.
809        let route_bind_breakers = forwarding.route_bind_breakers();
810        let route_bind_concurrency = forwarding.route_bind_concurrency();
811        let route_outages = forwarding.route_outages();
812        Self {
813            registry,
814            activity: None,
815            forwarding,
816            process_liveness: None,
817            supervisor: SupervisorHandle::new(),
818            subc_capabilities: Arc::from([
819                CAP_MANIFEST_REGISTRATION.to_string(),
820                CAP_CHANNEL_LIFECYCLE.to_string(),
821                CAP_PING_PONG.to_string(),
822                CAP_SESSION_ATTACH.to_string(),
823                CAP_ADMISSION_FACTS_RELAY.to_string(),
824                CAP_SCOPES_V1.to_string(),
825                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
826                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
827                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
828            ]),
829            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
830            route_bind_relay_timeouts: BTreeMap::new(),
831            route_bind_breakers,
832            route_bind_concurrency,
833            route_outages,
834            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
835            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
836            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
837            storage_config: None,
838            machine_id: None,
839            admission_facts_carrier_module_id: None,
840            admission_facts_targets: None,
841            scopes: Arc::new(RwLock::new(ScopeTable::new(
842                crate::daemon_config::default_scope_authority_owners(),
843            ))),
844            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
845            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
846            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
847            rescan: None,
848            connected_clients: ConnectedClients::new(),
849            counters,
850            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
851            daemon_provenance: DaemonProvenanceFacts::default(),
852            #[cfg(test)]
853            control_dispatch_delay: None,
854            #[cfg(test)]
855            provenance_probe_override: None,
856            #[cfg(test)]
857            route_bind_relay_sent_hold: None,
858        }
859    }
860
861    /// Set the central storage policy: registering modules then receive their
862    /// resolved storage descriptor in HELLO_ACK.
863    pub fn with_storage_config(
864        mut self,
865        storage_config: Option<crate::daemon_config::StorageConfig>,
866    ) -> Self {
867        self.storage_config = storage_config;
868        self
869    }
870
871    /// Open the ledger on its own writer thread; an unavailable sink does not
872    /// prevent the daemon from serving routing or supervisor requests.
873    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
874        self.activity = Some(crate::activity::ActivityLedger::open(
875            run_dir.join("activity.jsonl"),
876            incarnation,
877        ));
878        self
879    }
880
881    /// Set the machine id served to every registering module (HELLO_ACK) and on
882    /// `server.describe`.
883    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
884        self.machine_id = machine_id;
885        self
886    }
887
888    /// Configure the exact reserved module and target ids permitted to relay
889    /// opaque admission facts. Config-file loading validates this authority;
890    /// this builder keeps the same policy available to embedded test daemons.
891    pub fn with_admission_facts_config(
892        mut self,
893        carrier_module_id: Option<String>,
894        targets: Option<Vec<String>>,
895    ) -> Self {
896        self.admission_facts_carrier_module_id = carrier_module_id;
897        self.admission_facts_targets = targets;
898        self
899    }
900
901    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
902    /// Replaces the scope table with an empty one under the new list, so call it
903    /// while building the handler, before any module can sync.
904    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
905        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
906        self.scope_authority_owners = owners;
907        self
908    }
909
910    /// Override the route.bind relay timeout. Used by tests that assert the
911    /// timeout path so they don't block on the production-safe default.
912    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
913        self.route_bind_relay_timeout = timeout;
914        self
915    }
916
917    /// Install per-module route.bind relay budget overrides. A module id
918    /// listed here wins over the daemon-wide default set via
919    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
920    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
921    /// the same `Duration` the bind path will use.
922    pub fn with_route_bind_relay_timeouts(
923        mut self,
924        timeouts: impl IntoIterator<Item = (String, Duration)>,
925    ) -> Self {
926        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
927        self
928    }
929
930    /// Resolve the route.bind relay budget for a specific target module id.
931    /// Per-module overrides win; the daemon-wide value (set via
932    /// `with_route_bind_relay_timeout` or the built-in default) is the
933    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
934    /// the same resolution `handle_route_open` will use.
935    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
936        self.route_bind_relay_timeouts
937            .get(module_id)
938            .copied()
939            .unwrap_or(self.route_bind_relay_timeout)
940    }
941
942    /// Override the per-module bind-relay breaker policy.
943    ///
944    /// Used by tests, which cannot spend three production budgets opening a
945    /// breaker or twenty seconds waiting for its cooldown. The production
946    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
947    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
948    /// reasoning for the numbers.
949    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
950        self.route_bind_breaker_threshold = threshold.max(1);
951        self.route_bind_breaker_cooldown = cooldown;
952        self
953    }
954
955    #[cfg(test)]
956    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
957        self.health_probe_timeout = timeout;
958        self
959    }
960
961    #[cfg(test)]
962    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
963        self.control_dispatch_delay = Some(delay);
964        self
965    }
966
967    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
968    /// announce it is parked, once to be released. Build the barrier for two
969    /// parties, the test being the other one.
970    #[cfg(test)]
971    pub(crate) fn with_route_bind_relay_sent_hold(
972        mut self,
973        hold: Arc<tokio::sync::Barrier>,
974    ) -> Self {
975        self.route_bind_relay_sent_hold = Some(hold);
976        self
977    }
978
979    pub fn with_process_liveness(
980        mut self,
981        process_liveness: Arc<dyn ModuleProcessLiveness>,
982    ) -> Self {
983        self.process_liveness = Some(process_liveness);
984        self
985    }
986
987    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
988        self.supervisor = supervisor;
989        self
990    }
991
992    pub fn with_daemon_provenance(
993        mut self,
994        pid: u32,
995        started_at_ms: u64,
996        executable_path: Option<PathBuf>,
997        build_git_sha: Option<String>,
998        build_lock_digest: Option<String>,
999    ) -> Self {
1000        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1001        let process_start_time = process_start_time(pid);
1002        self.daemon_provenance = DaemonProvenanceFacts {
1003            build: DaemonBuildProvenance {
1004                build_git_sha,
1005                build_lock_digest,
1006            },
1007            pid: Some(pid),
1008            started_at_ms: Some(started_at_ms),
1009            start_clock: None,
1010            executable_path,
1011            executable_identity,
1012            process_start_time,
1013            probe: ExecutableIdentityProbe::default(),
1014        };
1015        self
1016    }
1017
1018    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1019        self.daemon_provenance.start_clock = Some(clock);
1020        self
1021    }
1022
1023    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1024    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1025        self.scopes
1026            .write()
1027            .unwrap_or_else(|poisoned| poisoned.into_inner())
1028            .set_wall_clock(Arc::new(clock));
1029        self
1030    }
1031
1032    #[cfg(test)]
1033    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1034        self.provenance_probe_override = Some(result);
1035        self
1036    }
1037
1038    /// Install the configured module set and its reserved capability bindings.
1039    /// Bindings are configuration-scoped and may point at a provider that has not
1040    /// been installed yet, so this does not require the bound module to exist.
1041    pub fn with_capability_config(
1042        self,
1043        modules: impl IntoIterator<Item = (String, bool)>,
1044        reserved_capabilities: BTreeMap<String, String>,
1045    ) -> Self {
1046        self.capability_evaluator
1047            .configure(modules, reserved_capabilities);
1048        self
1049    }
1050
1051    pub fn with_supervisor_rescan(
1052        mut self,
1053        supervisor: Supervisor,
1054        config_path: impl Into<PathBuf>,
1055        configured_port: Option<u16>,
1056    ) -> Self {
1057        self.rescan = Some(SupervisorRescanContext {
1058            supervisor,
1059            config_path: config_path.into(),
1060            configured_port,
1061            storage_config: self.storage_config.clone(),
1062            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1063            admission_facts_targets: self.admission_facts_targets.clone(),
1064            scope_authority_owners: self.scope_authority_owners.clone(),
1065        });
1066        self
1067    }
1068
1069    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1070        self.connected_clients = connected_clients;
1071        self
1072    }
1073
1074    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1075        Arc::clone(&self.forwarding)
1076    }
1077
1078    pub(crate) fn counters(&self) -> DaemonCounters {
1079        self.counters.clone()
1080    }
1081
1082    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1083    /// requirement event without depending on an operator polling a status command.
1084    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1085        tokio::spawn(async move {
1086            loop {
1087                self.capability_evaluator
1088                    .wait_for_change_or_deadline()
1089                    .await;
1090                self.refresh_capability_requirements();
1091            }
1092        });
1093    }
1094
1095    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1096    /// deadline passed during sleep is enforced on the first tick after wake.
1097    /// While no live scope carries a deadline the loop waits for an accepted
1098    /// scope change instead of ticking, so a daemon without deadlines never
1099    /// wakes for expiry.
1100    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1101        tokio::spawn(async move {
1102            loop {
1103                let has_deadlines = self
1104                    .scopes
1105                    .read()
1106                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1107                    .has_deadlines();
1108                if !has_deadlines {
1109                    self.scope_deadline_added.notified().await;
1110                    continue;
1111                }
1112                tokio::time::sleep(Duration::from_secs(1)).await;
1113                if let Err(error) = self.sweep_expired_scopes() {
1114                    warn!(%error, "scope expiry sweep failed");
1115                }
1116            }
1117        });
1118    }
1119
1120    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1121        let mut table = self
1122            .scopes
1123            .write()
1124            .unwrap_or_else(|poisoned| poisoned.into_inner());
1125        let swept = table.sweep_expired();
1126        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1127        drop(table);
1128        self.close_scope_drained_routes(drained);
1129        Ok(())
1130    }
1131
1132    fn publish_scope_changes(
1133        &self,
1134        changes: &[crate::scopes::ScopeTagChange],
1135        expired: &[crate::scopes::ScopeExpired],
1136    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1137        if changes.is_empty() {
1138            return Ok(Vec::new());
1139        }
1140        let drained = self
1141            .forwarding
1142            .publish_scope_changes(changes)
1143            .map_err(RouterError::Forwarding)?;
1144        let mut counts = HashMap::new();
1145        for route in &drained {
1146            *counts
1147                .entry((
1148                    route.scope.owner.as_str(),
1149                    route.scope.scope_ref.as_str(),
1150                    route.scope.tag.scope_epoch,
1151                ))
1152                .or_insert(0usize) += 1;
1153        }
1154        for scope in expired {
1155            let routes_closed = counts
1156                .get(&(
1157                    scope.owner.as_str(),
1158                    scope.scope_ref.as_str(),
1159                    scope.scope_epoch,
1160                ))
1161                .copied()
1162                .unwrap_or(0);
1163            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1164                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1165                routes_closed, "scope expired");
1166        }
1167        Ok(drained)
1168    }
1169
1170    fn runtime_capability_snapshot(
1171        &self,
1172    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1173        let runtime = self
1174            .supervisor
1175            .list()
1176            .into_iter()
1177            .map(|module| {
1178                let status = module.status().map_err(|err| {
1179                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1180                })?;
1181                Ok(RuntimeModule {
1182                    module_id: status.module_id,
1183                    state: status.state,
1184                    enabled: status.enabled,
1185                })
1186            })
1187            .collect::<Result<Vec<_>, RouterError>>()?;
1188        let (_, registrations) = self.registry.list_modules().map_err(|err| {
1189            RouterError::backend(
1190                0,
1191                0,
1192                format!("failed to list capability registrations: {err}"),
1193            )
1194        })?;
1195        let registrations = registrations
1196            .into_iter()
1197            .map(|registration| RegisteredModule {
1198                module_id: registration.manifest.module_id,
1199                module_version: registration.manifest.module_version,
1200                capabilities: registration.manifest.capabilities,
1201            })
1202            .collect();
1203        Ok((runtime, registrations))
1204    }
1205
1206    /// The capability side effects of a module becoming the active registration
1207    /// for its id: cache its manifest (warning if its claims drifted), run the
1208    /// deny census when its declarations call for one, and recompute the
1209    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1210    /// candidate's does not, and the supervisor does it at promotion instead,
1211    /// through [`crate::supervise::SwapPromotionObserver`].
1212    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1213        let cached_registration = RegisteredModule {
1214            module_id: registration.manifest.module_id.clone(),
1215            module_version: registration.manifest.module_version.clone(),
1216            capabilities: registration.manifest.capabilities.clone(),
1217        };
1218        if self.capability_evaluator.record_hello(&cached_registration) {
1219            warn!(
1220                module_id = %cached_registration.module_id,
1221                "capability claims drifted from the cached manifest"
1222            );
1223        }
1224        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1225            self.enforce_capability_denies();
1226        }
1227        self.refresh_capability_requirements();
1228    }
1229
1230    /// Point the shared supervisor handle at this handler for swap promotions.
1231    /// Called wherever a handler is put behind the `Arc` the router serves, so
1232    /// it can be held weakly.
1233    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1234        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1235            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1236        self.supervisor.set_swap_promotion_observer(observer);
1237    }
1238
1239    pub fn refresh_capability_requirements(&self) {
1240        match self.runtime_capability_snapshot() {
1241            Ok((runtime, registrations)) => {
1242                log_requirement_events(
1243                    self.capability_evaluator
1244                        .evaluate_now(&runtime, &registrations),
1245                );
1246            }
1247            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1248        }
1249    }
1250
1251    /// Reconcile only live, attested route bindings after a capability deny edge
1252    /// or target claim was added. This is deliberately a control-plane census:
1253    /// the opaque forwarding hot path must not grow a per-frame capability check.
1254    fn enforce_capability_denies(&self) {
1255        let (_, registrations) = match self.registry.list_modules() {
1256            Ok(snapshot) => snapshot,
1257            Err(err) => {
1258                warn!(error = %err, "failed to read registrations for capability deny census");
1259                return;
1260            }
1261        };
1262        let manifests = registrations
1263            .into_iter()
1264            .map(|registration| {
1265                (
1266                    registration.manifest.module_id.clone(),
1267                    registration.manifest,
1268                )
1269            })
1270            .collect::<BTreeMap<_, _>>();
1271        let census = match self.forwarding.route_census(None) {
1272            Ok(census) => census,
1273            Err(err) => {
1274                warn!(error = %err, "failed to read route census for capability deny enforcement");
1275                return;
1276            }
1277        };
1278
1279        for (target_module_id, routes) in census {
1280            let Some(target_manifest) = manifests.get(&target_module_id) else {
1281                continue;
1282            };
1283            let mut closed_routes = Vec::new();
1284            let mut module_goodbyes = Vec::new();
1285            for route in routes {
1286                let Principal::Reserved {
1287                    module_id: opening_module_id,
1288                } = &route.principal
1289                else {
1290                    continue;
1291                };
1292                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1293                    continue;
1294                };
1295                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1296                    continue;
1297                };
1298
1299                match self.forwarding.release_client_route(
1300                    route.goodbye_target.connection_id,
1301                    route.goodbye_target.channel,
1302                    route.goodbye_target.epoch,
1303                ) {
1304                    Ok(RouteRelease::Removed(module_goodbye)) => {
1305                        warn!(
1306                            opening_module_id,
1307                            target_module_id,
1308                            capability,
1309                            "force-closing route because an attested capability deny edge now matches"
1310                        );
1311                        closed_routes.push(route);
1312                        module_goodbyes.push(module_goodbye);
1313                    }
1314                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1315                    Err(err) => warn!(
1316                        opening_module_id,
1317                        target_module_id,
1318                        capability,
1319                        error = %err,
1320                        "failed to force-close capability-denied route"
1321                    ),
1322                }
1323            }
1324
1325            if closed_routes.is_empty() {
1326                continue;
1327            }
1328            send_route_control_pushes(
1329                &self.forwarding,
1330                closed_routes,
1331                ClientControlPush::RouteClosed {
1332                    module_id: target_module_id,
1333                    channels: Vec::new(),
1334                    reason: RouteCloseReason::CapabilityDenied,
1335                    drained: false,
1336                    abandoned: 0,
1337                    excluded_subscriptions: 0,
1338                    terminal: Some(false),
1339                },
1340            );
1341            self.emit_route_goodbyes(module_goodbyes);
1342        }
1343    }
1344
1345    /// Why a registered module is not accepting new route binds, or `None` when
1346    /// it is. This is the module's effective readiness: its declared readiness
1347    /// first, then every `need: required` capability it declares evaluating to
1348    /// `provided`. `route.open` and `catalog.list` both read it here so the
1349    /// catalog never reports a module routable that `route.open` would refuse.
1350    fn not_ready_reason(
1351        &self,
1352        registration: &crate::registry::ModuleRegistration,
1353    ) -> Option<NotReadyReason> {
1354        if !registration.ready {
1355            return Some(NotReadyReason {
1356                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1357                capability: None,
1358            });
1359        }
1360        self.first_unprovided_required_capability(registration)
1361            .map(|capability| NotReadyReason {
1362                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1363                capability: Some(capability),
1364            })
1365    }
1366
1367    /// The lexicographically first capability this registration declares
1368    /// `need: required` whose evaluator verdict is not `provided`.
1369    ///
1370    /// The verdicts are the capability evaluator's own; nothing here decides
1371    /// what "provided" means. The evaluator counts a capability provided as
1372    /// soon as a module claiming it has REGISTERED, not once that module is
1373    /// ready. That distinction is what keeps two modules that require each
1374    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1375    /// is ready", each would wait for the other to become ready first and
1376    /// neither ever would. Do not tighten it to readiness.
1377    ///
1378    /// A required capability with no verdict at all means this registration's
1379    /// HELLO or catalog.update landed after the last recompute; recompute once
1380    /// rather than let a missing verdict read as either answer. If it is still
1381    /// missing (the recompute itself failed) the capability counts as
1382    /// unprovided: the refusal is retryable, and routing a module whose
1383    /// required provider is unknown is the outcome this check exists to stop.
1384    fn first_unprovided_required_capability(
1385        &self,
1386        registration: &crate::registry::ModuleRegistration,
1387    ) -> Option<String> {
1388        let required = registration
1389            .manifest
1390            .capabilities
1391            .iter()
1392            .flat_map(|declarations| declarations.requires.iter())
1393            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1394            .map(|requirement| requirement.capability.as_str())
1395            .collect::<BTreeSet<_>>();
1396        if required.is_empty() {
1397            return None;
1398        }
1399        let module_id = registration.manifest.module_id.as_str();
1400        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1401        if required
1402            .iter()
1403            .any(|capability| verdict(capability).is_none())
1404        {
1405            self.refresh_capability_requirements();
1406        }
1407        required
1408            .into_iter()
1409            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1410            .map(str::to_string)
1411    }
1412
1413    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1414        self.capability_evaluator
1415            .statuses()
1416            .into_iter()
1417            .map(capability_requirement_status)
1418            .collect()
1419    }
1420
1421    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1422    /// registration-release watch. The signal is what the supervisor waits on
1423    /// before spawning a replacement, so it must only fire once forwarding
1424    /// teardown is also done (see [`Self::cleanup_connection`] /
1425    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1426    /// admitted but its forwarding endpoint could not be installed.
1427    fn deregister_connection(
1428        &self,
1429        connection_id: ConnectionId,
1430        reason: RegistrationEndReason,
1431    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1432        self.registry
1433            .deregister_connection_with_reason(connection_id, reason)
1434    }
1435
1436    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1437        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1438            return None;
1439        }
1440        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1441            parse_client_control_request(&frame.body)
1442        else {
1443            return None;
1444        };
1445        Some(target_module_id(&target).to_string())
1446    }
1447
1448    pub(crate) fn route_open_capacity_refusal(
1449        &self,
1450        ctx: &RouteCtx,
1451        frame: &Frame,
1452        target_module_id: &str,
1453        in_flight: usize,
1454        limit: usize,
1455    ) -> Result<Frame, RouterError> {
1456        self.route_open_admission_refusal_frame(
1457            ctx,
1458            frame,
1459            target_module_id,
1460            "open_admission_full",
1461            (in_flight, limit),
1462            format!(
1463                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1464            ),
1465        )
1466    }
1467
1468    fn route_open_target_capacity_refusal(
1469        &self,
1470        ctx: &RouteCtx,
1471        frame: &Frame,
1472        target_module_id: &str,
1473        in_flight: usize,
1474    ) -> Result<Frame, RouterError> {
1475        self.route_open_admission_refusal_frame(
1476            ctx,
1477            frame,
1478            target_module_id,
1479            "target_binds_full",
1480            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1481            format!(
1482                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1483            ),
1484        )
1485    }
1486
1487    /// Admission pressure clears as existing binds settle, so its refusal must
1488    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1489    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1490    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1491    /// cannot currently reach its target; `module_timeout` would falsely claim
1492    /// that a wait expired. A new, cleaner code would be terminal to deployed
1493    /// clients, so it requires a client-tolerance rollout before daemon emission.
1494    fn route_open_admission_refusal_frame(
1495        &self,
1496        ctx: &RouteCtx,
1497        frame: &Frame,
1498        target_module_id: &str,
1499        reason: &'static str,
1500        (in_flight, limit): (usize, usize),
1501        message: impl Into<String>,
1502    ) -> Result<Frame, RouterError> {
1503        let code = error_codes::TARGET_UNAVAILABLE;
1504        self.counters.increment_route_open_refused(code);
1505        info!(
1506            target: "control",
1507            code,
1508            reason,
1509            module_id = ?target_module_id,
1510            connection_id = ctx.connection_id.get(),
1511            in_flight,
1512            limit,
1513            "route.open refused"
1514        );
1515        control_error_frame(frame, code, message.into())
1516    }
1517
1518    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1519    ///
1520    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1521    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1522    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1523    #[cfg(test)]
1524    pub fn handle_control(
1525        &self,
1526        connection_id: ConnectionId,
1527        frame: Frame,
1528    ) -> Result<Vec<Frame>, RouterError> {
1529        match frame.header.ty {
1530            FrameType::Ping => Ok(vec![pong(&frame)?]),
1531            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1532            FrameType::Goodbye => self.handle_goodbye(connection_id),
1533            ty => Ok(vec![control_error_frame(
1534                &frame,
1535                "unsupported_control_frame",
1536                format!("unsupported channel-0 frame {ty:?}"),
1537            )?]),
1538        }
1539    }
1540
1541    pub async fn handle_control_frame(
1542        &self,
1543        ctx: &RouteCtx,
1544        frame: Frame,
1545    ) -> Result<Vec<Frame>, RouterError> {
1546        self.handle_control_frame_timed(ctx, frame, None).await
1547    }
1548
1549    pub(crate) async fn handle_control_frame_timed(
1550        &self,
1551        ctx: &RouteCtx,
1552        frame: Frame,
1553        dispatch_started_at: Option<StdInstant>,
1554    ) -> Result<Vec<Frame>, RouterError> {
1555        match frame.header.ty {
1556            FrameType::Ping => Ok(vec![pong(&frame)?]),
1557            FrameType::Hello => {
1558                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1559            }
1560            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1561            FrameType::Cancel => {
1562                // A Cancel on channel 0 names either a waiting operator.confirm
1563                // or a spawn-event subscription; both answer nothing on success.
1564                if self
1565                    .forwarding
1566                    .operator_confirms()
1567                    .cancel(ctx.connection_id, frame.header.corr)
1568                    || self
1569                        .supervisor
1570                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1571                {
1572                    Ok(Vec::new())
1573                } else {
1574                    Ok(vec![control_error_frame(
1575                        &frame,
1576                        "unknown_subscription",
1577                        "no supervisor spawn subscription has this correlation id",
1578                    )?])
1579                }
1580            }
1581            FrameType::Request => {
1582                // This additive operation is not part of the existing exhaustive
1583                // module-control enum. Probe the op before decoding that enum.
1584                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1585                if op
1586                    .as_ref()
1587                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1588                {
1589                    return self.handle_activity_record(ctx, frame).await;
1590                }
1591                if op
1592                    .as_ref()
1593                    .is_some_and(|probe| probe.op == "operator.confirm")
1594                {
1595                    return self.handle_operator_confirm(ctx, frame);
1596                }
1597                if self
1598                    .forwarding
1599                    .module_endpoint_for_connection(ctx.connection_id)
1600                    .map_err(RouterError::Forwarding)?
1601                    .is_some()
1602                {
1603                    if !is_known_module_request_op(&frame.body) {
1604                        return Ok(vec![control_error_frame(
1605                            &frame,
1606                            "unsupported_control_frame",
1607                            "module-originated channel-0 REQUEST is not supported",
1608                        )?]);
1609                    }
1610                    let request = match parse_module_control_request_from_module(&frame.body) {
1611                        Ok(request) => request,
1612                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1613                            return Ok(vec![control_error_frame(
1614                                &frame,
1615                                "unsupported_control_frame",
1616                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1617                            )?])
1618                        }
1619                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1620                            return Ok(vec![control_error_frame(
1621                                &frame,
1622                                "invalid_control_body",
1623                                format!("malformed module control body: {err}"),
1624                            )?])
1625                        }
1626                    };
1627                    let op = module_control_request_op(&request);
1628                    let corr = frame.header.corr;
1629                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1630                    let result =
1631                        self.handle_module_control_request(ctx.connection_id, frame, request);
1632                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1633                    return result;
1634                }
1635
1636                if is_known_module_request_op(&frame.body) {
1637                    return Ok(vec![control_error_frame(
1638                        &frame,
1639                        "not_registered",
1640                        "catalog.update requires an active module registration owned by this connection",
1641                    )?]);
1642                }
1643
1644                let request = match parse_client_control_request(&frame.body) {
1645                    Ok(request) => request,
1646                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1647                        return Ok(vec![control_error_frame(
1648                            &frame,
1649                            "unknown_control_op",
1650                            format!("unknown client control op: {err}"),
1651                        )?])
1652                    }
1653                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1654                        return Ok(vec![control_error_frame(
1655                            &frame,
1656                            "invalid_control_body",
1657                            format!("malformed client control body: {err}"),
1658                        )?])
1659                    }
1660                };
1661                let op = client_control_request_op(&request);
1662                let corr = frame.header.corr;
1663                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1664                #[cfg(test)]
1665                if let Some(delay) = self.control_dispatch_delay {
1666                    tokio::time::sleep(delay).await;
1667                }
1668                let result = self
1669                    .handle_client_control_request(ctx, frame, request)
1670                    .await;
1671                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1672                result
1673            }
1674            FrameType::Push => {
1675                let Some(endpoint) = self
1676                    .forwarding
1677                    .module_endpoint_for_connection(ctx.connection_id)
1678                    .map_err(RouterError::Forwarding)?
1679                else {
1680                    return Ok(vec![control_error_frame(
1681                        &frame,
1682                        "unsupported_control_frame",
1683                        "client-originated channel-0 PUSH is not supported",
1684                    )?]);
1685                };
1686                self.handle_status_update(endpoint, frame)
1687            }
1688            FrameType::Response | FrameType::Error
1689                if self
1690                    .forwarding
1691                    .module_endpoint_for_connection(ctx.connection_id)
1692                    .map_err(RouterError::Forwarding)?
1693                    .is_some() =>
1694            {
1695                self.handle_module_relay_response(ctx.connection_id, frame)
1696            }
1697            ty => Ok(vec![control_error_frame(
1698                &frame,
1699                "unsupported_control_frame",
1700                format!("unsupported channel-0 frame {ty:?}"),
1701            )?]),
1702        }
1703    }
1704
1705    pub fn cleanup_connection(
1706        &self,
1707        connection_id: ConnectionId,
1708    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1709        self.cleanup_connection_with_end_reason(
1710            connection_id,
1711            RegistrationEndReason::ConnectionClosed,
1712        )
1713    }
1714
1715    fn cleanup_connection_with_end_reason(
1716        &self,
1717        connection_id: ConnectionId,
1718        requested_reason: RegistrationEndReason,
1719    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1720        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1721            self.registry
1722                .get_module_by_connection(connection_id)?
1723                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
1724                .and_then(|module| module.registration_end_reason().ok().flatten())
1725                .unwrap_or(requested_reason)
1726        } else {
1727            requested_reason
1728        };
1729        let crash_closed = self
1730            .registry
1731            .get_module_by_connection(connection_id)?
1732            .and_then(|registration| {
1733                self.forwarding
1734                    .module_endpoint_for_connection(connection_id)
1735                    .ok()
1736                    .flatten()
1737                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1738                    .map(|routes| (registration.manifest.module_id, routes))
1739            });
1740        let crash_closed = crash_closed.map(|(module_id, routes)| {
1741            let terminal = match self.supervisor.get(&module_id) {
1742                None => false,
1743                Some(module) => match module.will_recover_after_connection_loss() {
1744                    Ok(will_recover) => !will_recover,
1745                    Err(err) => {
1746                        warn!(
1747                            %module_id,
1748                            error = %err,
1749                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1750                        );
1751                        false
1752                    }
1753                },
1754            };
1755            // The forwarding table gates all providers at the start of daemon
1756            // shutdown, before their connections are closed. An ordinary
1757            // module disconnect still reports crash if that gate is not set.
1758            let (reason, drained) = match self
1759                .forwarding
1760                .daemon_drain_result_for_connection(connection_id)
1761            {
1762                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1763                Ok(None) => (RouteCloseReason::Crash, false),
1764                Err(err) => {
1765                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1766                    (RouteCloseReason::Crash, false)
1767                }
1768            };
1769            (module_id, routes, reason, drained, terminal)
1770        });
1771        let registrations = self.deregister_connection(connection_id, end_reason);
1772        let cleanup = if crash_closed.is_some() {
1773            self.forwarding.cleanup_connection_counted(connection_id)
1774        } else {
1775            // A module connection's teardown needs the count of abandoned
1776            // route.bind relays for its route.closed notice below. Any other
1777            // connection, such as a client's, sends no such notice and needs
1778            // only its routes released, so it uses the route-only wrapper and
1779            // reports zero.
1780            self.forwarding
1781                .cleanup_connection(connection_id)
1782                .map(|released| crate::forwarding::ConnectionCleanup {
1783                    released,
1784                    abandoned_relays: 0,
1785                })
1786        };
1787        // The route.closed push waits for forwarding teardown because only
1788        // teardown knows how many pending route.bind relays it aborted. It still
1789        // goes out before the GOODBYEs for the released routes, and its targets
1790        // were captured above, before teardown removed those routes.
1791        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1792            let abandoned = cleanup
1793                .as_ref()
1794                .map_or(0, |cleanup| cleanup.abandoned_relays);
1795            send_route_control_pushes(
1796                &self.forwarding,
1797                routes,
1798                ClientControlPush::RouteClosed {
1799                    module_id,
1800                    channels: Vec::new(),
1801                    reason,
1802                    drained,
1803                    abandoned,
1804                    excluded_subscriptions: 0,
1805                    terminal: Some(terminal),
1806                },
1807            );
1808        }
1809        if let Ok(cleanup) = cleanup {
1810            self.emit_route_goodbyes(cleanup.released);
1811        }
1812        // Signal the registration-release watch only now that BOTH registry and
1813        // forwarding teardown are done, so a supervisor waiting to spawn a
1814        // replacement never observes release while old routes still exist.
1815        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1816            crate::supervise::notify_registration_release();
1817            self.capability_evaluator.wake_deadline_loop();
1818            self.refresh_capability_requirements();
1819        }
1820        self.supervisor.remove_spawn_subscribers(connection_id);
1821        // Sync authority dies with its connection, so the owner's next
1822        // connection can take it; the owner's scopes stay as they are.
1823        self.hello_launch_nonces
1824            .lock()
1825            .unwrap_or_else(|poisoned| poisoned.into_inner())
1826            .forget(connection_id);
1827        self.scopes
1828            .write()
1829            .unwrap_or_else(|poisoned| poisoned.into_inner())
1830            .release_connection(connection_id);
1831        registrations
1832    }
1833
1834    pub(crate) fn handle_route_goodbye(
1835        &self,
1836        connection_id: ConnectionId,
1837        route_channel: u16,
1838        route_epoch: u32,
1839    ) -> Result<bool, RouterError> {
1840        debug!(
1841            connection_id = connection_id.get(),
1842            route_channel, route_epoch, "handling route GOODBYE"
1843        );
1844        let RouteRelease::Removed(released_route) = self
1845            .forwarding
1846            .release_client_route(connection_id, route_channel, route_epoch)
1847            .map_err(RouterError::Forwarding)?
1848        else {
1849            return Ok(false);
1850        };
1851        self.emit_route_goodbyes(vec![released_route]);
1852        Ok(true)
1853    }
1854
1855    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1856        for released in released_routes {
1857            let frame = match Frame::build_with_version(
1858                released.negotiated_ver,
1859                FrameType::Goodbye,
1860                control_flags(),
1861                released.channel,
1862                released.epoch,
1863                0,
1864                Vec::new(),
1865            ) {
1866                Ok(frame) => frame,
1867                Err(err) => {
1868                    warn!(
1869                        route_channel = released.channel,
1870                        error = %err,
1871                        "failed to build route GOODBYE frame"
1872                    );
1873                    continue;
1874                }
1875            };
1876            if !released.close_on_delivery_failure() {
1877                crate::forwarding::send_module_route_goodbye(
1878                    &self.counters,
1879                    &released.sink,
1880                    frame,
1881                    released.module_id.as_deref(),
1882                    "client route released",
1883                );
1884                continue;
1885            }
1886            if let Err(err) = released.sink.try_send(frame) {
1887                warn!(
1888                    target_connection_id = released.connection_id.get(),
1889                    route_channel = released.channel,
1890                    error = %err,
1891                    "route GOODBYE was not delivered to client; closing target connection"
1892                );
1893                if self
1894                    .forwarding
1895                    .escalate_client_delivery_failure(
1896                        released.connection_id,
1897                        released.channel,
1898                        released.epoch,
1899                        CloseReason::new(
1900                            "route_goodbye_delivery_failed",
1901                            format!(
1902                                "failed to enqueue route GOODBYE for channel {}: {err}",
1903                                released.channel
1904                            ),
1905                        ),
1906                        crate::forwarding::UndeliveredFrame {
1907                            module_id: released.module_id.as_deref(),
1908                            sink: &released.sink,
1909                        },
1910                    )
1911                    .unwrap_or(false)
1912                {
1913                    self.counters.increment_goodbye_relay_client_failed();
1914                }
1915            }
1916        }
1917    }
1918
1919    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1920    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1921    /// own commit failed after the module had already accepted). Without this, a
1922    /// module that accepts late keeps a binding subc has torn down, so a later
1923    /// frame on that module channel could misdeliver if the channel is reused.
1924    ///
1925    /// Never closes the shared module connection on failure: a dropped notification
1926    /// only wastes a bounded amount of warm module-side state, which the module's
1927    /// own idle reaper reclaims. Only call this once the route.bind relay was
1928    /// actually enqueued to the module — if the relay send itself failed, the
1929    /// module never created a binding and there is nothing to tear down.
1930    fn send_abandoned_route_bind_goodbye(
1931        &self,
1932        module_sink: &crate::FrameSink,
1933        negotiated_ver: u8,
1934        module_channel: u16,
1935        module_epoch: u32,
1936    ) {
1937        let frame = match Frame::build_with_version(
1938            negotiated_ver,
1939            FrameType::Goodbye,
1940            control_flags(),
1941            module_channel,
1942            module_epoch,
1943            0,
1944            Vec::new(),
1945        ) {
1946            Ok(frame) => frame,
1947            Err(err) => {
1948                warn!(
1949                    route_channel = module_channel,
1950                    error = %err,
1951                    "failed to build GOODBYE for abandoned route.bind"
1952                );
1953                return;
1954            }
1955        };
1956        crate::forwarding::send_module_route_goodbye(
1957            &self.counters,
1958            module_sink,
1959            frame,
1960            None,
1961            "abandoned route.bind",
1962        );
1963    }
1964
1965    fn handle_hello(
1966        &self,
1967        connection_id: ConnectionId,
1968        sink: Option<crate::FrameSink>,
1969        frame: Frame,
1970    ) -> Result<Vec<Frame>, RouterError> {
1971        debug!(
1972            connection_id = connection_id.get(),
1973            corr = frame.header.corr,
1974            "handling HELLO"
1975        );
1976        // A module connection has one identity for its entire lifetime. A second
1977        // registration would leave the old registry owner behind while replacing
1978        // its forwarding endpoint and launch nonce.
1979        if self
1980            .registry
1981            .get_module_by_connection(connection_id)
1982            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1983            .is_some()
1984        {
1985            return Ok(vec![control_error_frame(
1986                &frame,
1987                "invalid_hello",
1988                "connection is already registered as a module",
1989            )?]);
1990        }
1991        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1992            Ok(value) => value,
1993            Err(err) => {
1994                return Ok(vec![control_error_frame(
1995                    &frame,
1996                    "invalid_hello",
1997                    format!("malformed HELLO body: {err}"),
1998                )?])
1999            }
2000        };
2001        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2002            return Ok(vec![control_error_frame(
2003                &frame,
2004                "invalid_capability_grammar",
2005                err.to_string(),
2006            )?]);
2007        }
2008        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2009            return Ok(vec![control_error_frame(
2010                &frame,
2011                "invalid_manifest",
2012                err.to_string(),
2013            )?]);
2014        }
2015        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2016            return Ok(vec![control_error_frame(
2017                &frame,
2018                "invalid_event_declaration",
2019                err.to_string(),
2020            )?]);
2021        }
2022        if let Some(provenance) = hello_value
2023            .get("manifest")
2024            .and_then(|manifest| manifest.get("provenance"))
2025        {
2026            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2027                return Ok(vec![control_error_frame(
2028                    &frame,
2029                    "invalid_manifest",
2030                    format!("malformed manifest provenance: {err}"),
2031                )?]);
2032            }
2033        }
2034        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2035            Ok(hello) => hello,
2036            Err(err) => {
2037                return Ok(vec![control_error_frame(
2038                    &frame,
2039                    "invalid_hello",
2040                    format!("malformed HELLO body: {err}"),
2041                )?])
2042            }
2043        };
2044
2045        if hello.protocol_ver != hello.manifest.protocol_ver {
2046            return Ok(vec![control_error_frame(
2047                &frame,
2048                "invalid_manifest",
2049                format!(
2050                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2051                    hello.protocol_ver, hello.manifest.protocol_ver
2052                ),
2053            )?]);
2054        }
2055
2056        if hello.manifest.module_id.trim().is_empty() {
2057            return Ok(vec![control_error_frame(
2058                &frame,
2059                "invalid_manifest",
2060                "manifest module_id must not be empty",
2061            )?]);
2062        }
2063
2064        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2065            Ok(negotiated_ver) => negotiated_ver,
2066            Err(message) => {
2067                return Ok(vec![control_error_frame(
2068                    &frame,
2069                    "version_unsupported",
2070                    message,
2071                )?])
2072            }
2073        };
2074
2075        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2076        // swap is open for this id, the only HELLO admitted as a second process
2077        // is the one carrying the candidate's launch nonce (the swap token), and
2078        // it registers into the candidate slot rather than being refused as a
2079        // duplicate. Run after the reserved gate, a reserved module's candidate
2080        // would be refused `reserved_module` for presenting a nonce that gate
2081        // does not know. See `SupervisorHandle::swap_hello_admission`.
2082        let swap_admission = self
2083            .supervisor
2084            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2085        if swap_admission == SwapHelloAdmission::Refused {
2086            warn!(
2087                module_id = %hello.manifest.module_id,
2088                connection_id = connection_id.get(),
2089                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2090            );
2091            return Ok(vec![control_error_frame(
2092                &frame,
2093                "swap_token_invalid",
2094                format!(
2095                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2096                    hello.manifest.module_id
2097                ),
2098            )?]);
2099        }
2100        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2101
2102        // Reserved-module identity gate: a module_id configured `reserved` may be
2103        // registered ONLY by the process subc spawned for it, proven by echoing the
2104        // one-time launch nonce subc injected. A non-reserved id has no recorded
2105        // nonce and always passes. This blocks a key-holder from impersonating a
2106        // security-boundary module (e.g. the credential vault) while the real one is
2107        // down/restarting and its registration slot is momentarily free. A swap
2108        // candidate has already proven the same thing with its own nonce above.
2109        if let Some(rejection) = (!swap_candidate)
2110            .then(|| {
2111                self.supervisor.reserved_hello_rejection(
2112                    &hello.manifest.module_id,
2113                    hello.launch_nonce.as_deref(),
2114                )
2115            })
2116            .flatten()
2117        {
2118            let message = match rejection {
2119                ReservedHelloRejection::Exact { module_id } => format!(
2120                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2121                ),
2122                ReservedHelloRejection::Prefix {
2123                    prefix,
2124                    owner_module_id,
2125                } => format!(
2126                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2127                    hello.manifest.module_id
2128                ),
2129            };
2130            return Ok(vec![control_error_frame(
2131                &frame,
2132                "reserved_module",
2133                message,
2134            )?]);
2135        }
2136
2137        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2138            &hello.manifest.module_id,
2139            hello.manifest.capabilities.as_ref(),
2140        );
2141        if let Some(refusal) = reserved_capability_refusals.first() {
2142            let capability = refusal.capability.clone();
2143            let bound_module = refusal.claimants[0].clone();
2144            log_duplicate_claim_events(reserved_capability_refusals);
2145            return Ok(vec![control_error_frame(
2146                &frame,
2147                "reserved_capability",
2148                format!(
2149                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2150                    capability, bound_module, hello.manifest.module_id
2151                ),
2152            )?]);
2153        }
2154
2155        // A connection that already opened client routes must not also register as
2156        // a module: cleanup would then release only one side and leak the other.
2157        if self
2158            .forwarding
2159            .connection_has_client_routes(connection_id)
2160            .map_err(RouterError::Forwarding)?
2161        {
2162            return Ok(vec![control_error_frame(
2163                &frame,
2164                "invalid_hello",
2165                "connection has open client routes and cannot also register as a module",
2166            )?]);
2167        }
2168
2169        // Kept for scope sync authority, which goes only to the connection that
2170        // presented the module's current launch nonce. Recorded before the
2171        // registration is attempted: a connection whose registration then fails
2172        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2173        self.hello_launch_nonces
2174            .lock()
2175            .unwrap_or_else(|poisoned| poisoned.into_inner())
2176            .record(connection_id, hello.launch_nonce.as_deref());
2177        let control_ops = effective_module_control_ops(hello.control_ops);
2178        // Built before anything is registered so an encoding failure leaves no
2179        // registry or forwarding state behind.
2180        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2181        if swap_candidate {
2182            return self.register_swap_candidate(
2183                connection_id,
2184                sink,
2185                &frame,
2186                hello.manifest,
2187                negotiated_ver,
2188                control_ops,
2189                hello_ack,
2190            );
2191        }
2192        let registration = match self.registry.register_with_control_ops(
2193            hello.manifest,
2194            negotiated_ver,
2195            connection_id,
2196            control_ops,
2197        ) {
2198            Ok(registration) => registration,
2199            Err(RegistryError::DuplicateModuleId { module_id }) => {
2200                return Ok(vec![control_error_frame(
2201                    &frame,
2202                    "duplicate_module_id",
2203                    format!(
2204                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2205                    ),
2206                )?])
2207            }
2208            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2209                return Ok(vec![control_error_frame(
2210                    &frame,
2211                    "invalid_module_id",
2212                    err.to_string(),
2213                )?])
2214            }
2215            Err(err) => {
2216                return Ok(vec![control_error_frame(
2217                    &frame,
2218                    "registry_error",
2219                    err.to_string(),
2220                )?])
2221            }
2222        };
2223
2224        let reply = if let Some(sink) = sink {
2225            // The forwarding table's module store is also the daemon-to-module
2226            // control-RPC lane, so every HELLO gets a live endpoint even when the
2227            // manifest has no routable provider role. Non-routable modules still
2228            // cannot receive route.bind in production: `handle_route_open` checks
2229            // the registry manifest with `target_has_required_role` before the
2230            // only production call to `begin_route_bind_relay_for` below that
2231            // route.open path. The remaining direct relay callers are unit tests
2232            // and benchmark harnesses that construct forwarding state explicitly.
2233            //
2234            // The HELLO_ACK is queued by the forwarding table itself, before the
2235            // endpoint becomes visible, and is NOT returned as a reply. A module
2236            // reads HELLO_ACK first and exits on anything else; a reply is only
2237            // written after this handler returns, by which time a route.open on
2238            // another connection could already have queued a route.bind request
2239            // for this module ahead of it.
2240            let concurrency = manifest_concurrency(&registration.manifest);
2241            if let Err(err) = self.forwarding.register_module_connection_acked(
2242                connection_id,
2243                registration.manifest.module_id.clone(),
2244                negotiated_ver,
2245                concurrency,
2246                sink,
2247                hello_ack,
2248            ) {
2249                // Forwarding registration failed, so there is no forwarding
2250                // state to tear down. Remove the registry entry and signal the
2251                // release watch directly.
2252                if matches!(
2253                    self.deregister_connection(
2254                        connection_id,
2255                        RegistrationEndReason::RegistrationFailed,
2256                    ),
2257                    Ok(r) if !r.is_empty()
2258                ) {
2259                    crate::supervise::notify_registration_release();
2260                }
2261                return Ok(vec![control_error_frame(
2262                    &frame,
2263                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2264                        "invalid_hello"
2265                    } else {
2266                        forwarding_error_code(&err)
2267                    },
2268                    err.to_string(),
2269                )?]);
2270            }
2271            Vec::new()
2272        } else {
2273            // No sink means no forwarding endpoint, so nothing can be routed
2274            // ahead of the ack; it goes out as the reply.
2275            vec![hello_ack]
2276        };
2277
2278        // Exposure over assumption: Concurrency's serde default is pinned to the
2279        // pre-field behavior (ModuleManaged), so a management surface that is
2280        // genuinely Serial and just never declared it inherits concurrent
2281        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2282        // turns "no module has been bitten yet" into the checkable claim "no
2283        // module is exposed" -- one read of the boot log instead of a fleet
2284        // audit. Detected from the raw HELLO bytes because the serde default
2285        // deliberately erases the absent/declared distinction from the type.
2286        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2287            info!(
2288                module_id = %registration.manifest.module_id,
2289                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2290            );
2291        }
2292
2293        self.apply_registration_capabilities(&registration);
2294
2295        info!(
2296            module_id = %registration.manifest.module_id,
2297            module_version = %registration.manifest.module_version,
2298            negotiated_ver,
2299            routable_provider = manifest_provides_routable_role(&registration.manifest),
2300            connection_id = connection_id.get(),
2301            "module registered"
2302        );
2303
2304        Ok(reply)
2305    }
2306
2307    /// Register a HELLO the swap gate admitted into the candidate slot of the
2308    /// registry and of forwarding, where it is reachable over its own
2309    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2310    /// nothing routes to it until the supervisor cuts over.
2311    ///
2312    /// Registry first, then forwarding, the same order as an ordinary HELLO;
2313    /// a forwarding failure removes the registry entry again. The capability
2314    /// census is not run: it describes routable modules, and this one is not
2315    /// routable until promotion.
2316    #[allow(clippy::too_many_arguments)]
2317    fn register_swap_candidate(
2318        &self,
2319        connection_id: ConnectionId,
2320        sink: Option<crate::FrameSink>,
2321        frame: &Frame,
2322        manifest: ModuleManifest,
2323        negotiated_ver: u8,
2324        control_ops: Vec<String>,
2325        hello_ack: Frame,
2326    ) -> Result<Vec<Frame>, RouterError> {
2327        let module_id = manifest.module_id.clone();
2328        let registration = match self.registry.register_candidate_with_control_ops(
2329            manifest,
2330            negotiated_ver,
2331            connection_id,
2332            control_ops,
2333        ) {
2334            Ok(registration) => registration,
2335            Err(RegistryError::DuplicateModuleId { module_id }) => {
2336                return Ok(vec![control_error_frame(
2337                    frame,
2338                    "duplicate_module_id",
2339                    format!(
2340                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2341                    ),
2342                )?])
2343            }
2344            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2345                return Ok(vec![control_error_frame(
2346                    frame,
2347                    "invalid_module_id",
2348                    err.to_string(),
2349                )?])
2350            }
2351            Err(err) => {
2352                return Ok(vec![control_error_frame(
2353                    frame,
2354                    "registry_error",
2355                    err.to_string(),
2356                )?])
2357            }
2358        };
2359        let reply = if let Some(sink) = sink {
2360            // Same ordering as an ordinary HELLO: the forwarding table queues
2361            // the HELLO_ACK before the candidate endpoint is inserted, because
2362            // a module exits if its first frame after HELLO is anything else.
2363            let concurrency = manifest_concurrency(&registration.manifest);
2364            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2365                connection_id,
2366                module_id.clone(),
2367                negotiated_ver,
2368                concurrency,
2369                sink,
2370                hello_ack,
2371            ) {
2372                if matches!(
2373                    self.deregister_connection(
2374                        connection_id,
2375                        RegistrationEndReason::RegistrationFailed,
2376                    ),
2377                    Ok(r) if !r.is_empty()
2378                ) {
2379                    crate::supervise::notify_registration_release();
2380                }
2381                return Ok(vec![control_error_frame(
2382                    frame,
2383                    forwarding_error_code(&err),
2384                    err.to_string(),
2385                )?]);
2386            }
2387            Vec::new()
2388        } else {
2389            vec![hello_ack]
2390        };
2391        self.supervisor.mark_swap_candidate_admitted(&module_id);
2392        info!(
2393            module_id = %module_id,
2394            module_version = %registration.manifest.module_version,
2395            negotiated_ver,
2396            ready = registration.ready,
2397            connection_id = connection_id.get(),
2398            "swap candidate registered; not routable until cutover"
2399        );
2400        Ok(reply)
2401    }
2402
2403    fn build_hello_ack(
2404        &self,
2405        frame: &Frame,
2406        negotiated_ver: u8,
2407        module_id: &str,
2408    ) -> Result<Frame, RouterError> {
2409        let ack = ModuleHelloAckBody {
2410            negotiated_ver,
2411            subc_ops: module_subc_ops(),
2412            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2413            storage: self
2414                .storage_config
2415                .as_ref()
2416                .map(|cfg| cfg.descriptor_for(module_id)),
2417            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2418        };
2419        let body = serde_json::to_vec(&ack).map_err(|err| {
2420            RouterError::backend(
2421                0,
2422                frame.header.corr,
2423                format!("failed to encode HELLO_ACK: {err}"),
2424            )
2425        })?;
2426
2427        Frame::build_with_version(
2428            negotiated_ver,
2429            FrameType::HelloAck,
2430            control_flags(),
2431            0,
2432            0,
2433            frame.header.corr,
2434            body,
2435        )
2436        .map_err(RouterError::FrameBuild)
2437    }
2438
2439    async fn handle_client_control_request(
2440        &self,
2441        ctx: &RouteCtx,
2442        frame: Frame,
2443        request: ClientControlRequest,
2444    ) -> Result<Vec<Frame>, RouterError> {
2445        match request {
2446            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2447            ClientControlRequest::CatalogList { module_id } => {
2448                self.handle_catalog_list(frame, module_id)
2449            }
2450            ClientControlRequest::RouteOpen {
2451                target,
2452                identity,
2453                consumer_identity,
2454                consumer_capabilities,
2455                role_versions,
2456                admission_facts,
2457                scope,
2458            } => {
2459                self.handle_route_open(
2460                    ctx,
2461                    frame,
2462                    RouteOpenRequest {
2463                        target,
2464                        identity,
2465                        consumer_identity,
2466                        consumer_capabilities,
2467                        role_versions,
2468                        admission_facts,
2469                        scope,
2470                    },
2471                )
2472                .await
2473            }
2474            ClientControlRequest::RoutePoll {
2475                route_channel,
2476                route_epoch,
2477                kind,
2478            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2479            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2480            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2481                self.handle_supervisor_spawn_snapshot(frame)
2482            }
2483            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2484                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2485            }
2486            ClientControlRequest::SupervisorRestart {
2487                module_id,
2488                drain_timeout_ms,
2489            } => {
2490                self.log_supervisor_request_received(
2491                    ctx,
2492                    frame.header.corr,
2493                    ops::SUPERVISOR_RESTART,
2494                    Some(&module_id),
2495                    None,
2496                )?;
2497                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2498                    .await
2499            }
2500            ClientControlRequest::SupervisorSwap {
2501                module_id,
2502                ready_timeout_ms,
2503            } => {
2504                self.log_supervisor_request_received(
2505                    ctx,
2506                    frame.header.corr,
2507                    ops::SUPERVISOR_SWAP,
2508                    Some(&module_id),
2509                    None,
2510                )?;
2511                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2512                    .await
2513            }
2514            ClientControlRequest::SupervisorReload { module_id } => {
2515                self.log_supervisor_request_received(
2516                    ctx,
2517                    frame.header.corr,
2518                    ops::SUPERVISOR_RELOAD,
2519                    Some(&module_id),
2520                    None,
2521                )?;
2522                self.handle_supervisor_reload(frame, module_id).await
2523            }
2524            ClientControlRequest::SupervisorRescan { preview } => {
2525                if !preview {
2526                    self.log_supervisor_request_received(
2527                        ctx,
2528                        frame.header.corr,
2529                        ops::SUPERVISOR_RESCAN,
2530                        None,
2531                        None,
2532                    )?;
2533                }
2534                self.handle_supervisor_rescan(frame, preview).await
2535            }
2536            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2537                self.log_supervisor_request_received(
2538                    ctx,
2539                    frame.header.corr,
2540                    ops::SUPERVISOR_RELEASE_RESERVED,
2541                    Some(&module_id),
2542                    None,
2543                )?;
2544                self.handle_supervisor_release_reserved(frame, module_id)
2545                    .await
2546            }
2547            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2548                self.log_supervisor_request_received(
2549                    ctx,
2550                    frame.header.corr,
2551                    ops::SUPERVISOR_SET_ENABLED,
2552                    Some(&module_id),
2553                    Some(enabled),
2554                )?;
2555                self.handle_supervisor_set_enabled(frame, module_id, enabled)
2556                    .await
2557            }
2558            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2559                self.handle_supervisor_health_probe(frame, module_id).await
2560            }
2561            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2562            ClientControlRequest::SupervisorRoutes { module_id } => {
2563                self.handle_supervisor_routes(frame, module_id)
2564            }
2565            ClientControlRequest::SupervisorProvenance { module_id } => {
2566                self.handle_supervisor_provenance(frame, module_id).await
2567            }
2568            ClientControlRequest::SupervisorStderrTail {
2569                module_id,
2570                max_lines,
2571                max_bytes,
2572            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2573            ClientControlRequest::SupervisorTerminals { module_id } => {
2574                self.handle_supervisor_terminals(frame, module_id).await
2575            }
2576        }
2577    }
2578
2579    fn log_supervisor_request_received(
2580        &self,
2581        ctx: &RouteCtx,
2582        corr: u64,
2583        op: &'static str,
2584        module_id: Option<&str>,
2585        enabled: Option<bool>,
2586    ) -> Result<(), RouterError> {
2587        let caller = self
2588            .registry
2589            .get_module_by_connection(ctx.connection_id)
2590            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2591            .map(|registration| Principal::Reserved {
2592                module_id: registration.manifest.module_id,
2593            })
2594            .unwrap_or(Principal::Direct);
2595        let caller = principal_label(&caller);
2596        // `Option` fields are recorded only when present, so a request with no
2597        // module id (rescan) or no enabled flag simply omits that field.
2598        info!(
2599            target: "control",
2600            op,
2601            module_id,
2602            enabled,
2603            connection_id = ctx.connection_id.get(),
2604            caller = %caller,
2605            "supervisor request received"
2606        );
2607        Ok(())
2608    }
2609
2610    fn handle_module_control_request(
2611        &self,
2612        connection_id: ConnectionId,
2613        frame: Frame,
2614        request: ModuleControlRequestFromModule,
2615    ) -> Result<Vec<Frame>, RouterError> {
2616        match request {
2617            ModuleControlRequestFromModule::CatalogUpdate {
2618                provides,
2619                capabilities,
2620                ready,
2621            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2622            ModuleControlRequestFromModule::LiveRoots {} => {
2623                let registered = self
2624                    .registry
2625                    .get_module_by_connection(connection_id)
2626                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2627                let Some(registration) = registered else {
2628                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2629                };
2630                let response = self
2631                    .forwarding
2632                    .live_roots(&registration.manifest.module_id)
2633                    .map_err(RouterError::Forwarding)?;
2634                Ok(vec![control_response_body_frame(
2635                    &frame,
2636                    &response,
2637                    "ModuleControlResponseToModule::LiveRoots",
2638                )?])
2639            }
2640            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2641                self.handle_scope_sync(connection_id, frame, generation, scopes)
2642            }
2643            ModuleControlRequestFromModule::ScopeApply {
2644                generation,
2645                upsert,
2646                end,
2647            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2648            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2649                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2650            }
2651        }
2652    }
2653
2654    fn handle_operator_confirm(
2655        &self,
2656        ctx: &RouteCtx,
2657        frame: Frame,
2658    ) -> Result<Vec<Frame>, RouterError> {
2659        use crate::operator_confirm::{audit, Outcome};
2660        let registration = self
2661            .registry
2662            .get_module_by_connection(ctx.connection_id)
2663            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2664        let Some(registration) = registration else {
2665            let outcome = Outcome::refusal("not_registered");
2666            audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2667            return Ok(vec![outcome.frame(&frame)]);
2668        };
2669        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2670            Ok(request) => request,
2671            Err(_) => {
2672                let outcome = Outcome::refusal("invalid_control_body");
2673                audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2674                return Ok(vec![outcome.frame(&frame)]);
2675            }
2676        };
2677        let module_id = registration.manifest.module_id;
2678        // Read the launch nonce (under its own lock) before taking the forwarding
2679        // table's lock below: holding forwarding while waiting on another daemon
2680        // lock risks a lock-order deadlock with paths that take them the other way.
2681        let nonce = self
2682            .hello_launch_nonces
2683            .lock()
2684            .unwrap_or_else(|p| p.into_inner())
2685            .nonce(ctx.connection_id)
2686            .map(str::to_owned);
2687        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2688            self.supervisor
2689                .spawned_consumer_authorized(&module_id, nonce)
2690        });
2691        let confirms = self.forwarding.operator_confirms();
2692        self.forwarding
2693            .with_operator_route(
2694                ctx.connection_id,
2695                request.route_channel,
2696                request.route_epoch,
2697                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2698            )
2699            .map_err(RouterError::Forwarding)
2700    }
2701
2702    /// `scope.sync`: the owner is the module registered on this connection.
2703    /// A connection with no registration (every client connection, `direct`
2704    /// included) is refused `not_registered` before the table is consulted.
2705    fn handle_scope_sync(
2706        &self,
2707        connection_id: ConnectionId,
2708        frame: Frame,
2709        generation: u64,
2710        scopes: Vec<ScopeRecord>,
2711    ) -> Result<Vec<Frame>, RouterError> {
2712        self.handle_scope_change(connection_id, frame, generation, scopes, None)
2713    }
2714
2715    fn handle_scope_change(
2716        &self,
2717        connection_id: ConnectionId,
2718        frame: Frame,
2719        generation: u64,
2720        scopes: Vec<ScopeRecord>,
2721        end: Option<Vec<ScopeEnd>>,
2722    ) -> Result<Vec<Frame>, RouterError> {
2723        let is_apply = end.is_some();
2724        let Some(registration) = self
2725            .registry
2726            .get_module_by_connection(connection_id)
2727            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2728        else {
2729            return Ok(vec![control_error_frame(
2730                &frame,
2731                "not_registered",
2732                "scope.sync and scope.apply require an active module registration owned by this connection",
2733            )?]);
2734        };
2735        let owner = registration.manifest.module_id;
2736        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2737        let is_current_launch = |connection: ConnectionId| {
2738            self.hello_launch_nonces
2739                .lock()
2740                .unwrap_or_else(|poisoned| poisoned.into_inner())
2741                .presented(connection, current_nonce.as_deref())
2742        };
2743        // Lock order is the scope table, then the forwarding table: the new
2744        // tags are published, and the routes the change closes are selected,
2745        // while the scope table is still write-locked, so no admission can read
2746        // a record whose tag is not yet published.
2747        let mut table = self
2748            .scopes
2749            .write()
2750            .unwrap_or_else(|poisoned| poisoned.into_inner());
2751        let outcome = match end {
2752            Some(end) => table.apply(
2753                &owner,
2754                connection_id,
2755                is_current_launch,
2756                generation,
2757                scopes,
2758                end,
2759            ),
2760            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2761        };
2762        let drained = match &outcome {
2763            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2764            Err(_) => Vec::new(),
2765        };
2766        let has_deadlines = outcome.is_ok() && table.has_deadlines();
2767        drop(table);
2768        if has_deadlines {
2769            self.scope_deadline_added.notify_one();
2770        }
2771        match outcome {
2772            Ok(applied) => {
2773                let counts = ScopeOutcomeCounts::of(&applied.results);
2774                info!(
2775                    owner = %owner,
2776                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2777                    generation,
2778                    records = applied.results.len(),
2779                    created = counts.created,
2780                    replaced = counts.replaced,
2781                    updated = counts.updated,
2782                    unchanged = counts.unchanged,
2783                    refused = counts.refused,
2784                    ended = applied.ended.len(),
2785                    tag_changes = applied.tag_changes.len(),
2786                    routes_closed = drained.len(),
2787                    "scope change accepted"
2788                );
2789                // An accepted scope change can still refuse individual records, and the
2790                // owner is the only party that sees the reply. Name them here so
2791                // an operator can tell a refused session from a missing one
2792                // without the owner's logs. Capped so a call that refuses
2793                // thousands cannot flood the log; the count above is complete.
2794                for refused in applied
2795                    .results
2796                    .iter()
2797                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2798                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2799                {
2800                    warn!(
2801                        owner = %owner,
2802                        generation,
2803                        scope_ref = %refused.scope_ref,
2804                        scope_epoch = refused.scope_epoch,
2805                        code = refused.code.as_deref().unwrap_or(""),
2806                        "scope record refused"
2807                    );
2808                }
2809                self.close_scope_drained_routes(drained);
2810                let response = if is_apply {
2811                    ModuleControlResponseToModule::ScopeApply {
2812                        generation,
2813                        results: applied.results,
2814                        end_results: applied.end_results,
2815                        ended: applied.ended,
2816                    }
2817                } else {
2818                    ModuleControlResponseToModule::ScopeSync {
2819                        generation,
2820                        results: applied.results,
2821                        ended: applied.ended,
2822                    }
2823                };
2824                Ok(vec![control_response_body_frame(
2825                    &frame,
2826                    &response,
2827                    "scope change response",
2828                )?])
2829            }
2830            Err(refusal) => {
2831                info!(
2832                    owner = %owner,
2833                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2834                    generation,
2835                    code = refusal.code,
2836                    "scope change refused"
2837                );
2838                Ok(vec![control_error_frame(
2839                    &frame,
2840                    refusal.code,
2841                    refusal.message,
2842                )?])
2843            }
2844        }
2845    }
2846
2847    /// Tell both ends of each route a scope change closed. The module gets a
2848    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
2849    /// the client's route handle. The client also gets `route.closed` with the
2850    /// scope reason, one push per module and reason, so it can tell a revoked
2851    /// route from an ordinary close and not reopen it.
2852    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2853        if drained.is_empty() {
2854            return;
2855        }
2856        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2857            BTreeMap::new();
2858        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2859        for route in drained {
2860            warn!(
2861                module_id = %route.module_id,
2862                reason = ?route.reason,
2863                client_connection_id = route.client.connection_id.get(),
2864                route_channel = route.client.channel,
2865                "closing route because its scope changed"
2866            );
2867            pushes
2868                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2869                .or_insert_with(|| (route.reason, Vec::new()))
2870                .1
2871                .push(EndpointRoute {
2872                    goodbye_target: route.client.clone(),
2873                    principal: Principal::Unverified,
2874                    bound_at: Instant::now(),
2875                    draining: false,
2876                    drain_reason: None,
2877                });
2878            goodbyes.push(route.module);
2879            goodbyes.push(route.client);
2880        }
2881        for ((module_id, _), (reason, routes)) in pushes {
2882            send_route_control_pushes(
2883                &self.forwarding,
2884                routes,
2885                ClientControlPush::RouteClosed {
2886                    module_id,
2887                    channels: Vec::new(),
2888                    reason,
2889                    drained: false,
2890                    abandoned: 0,
2891                    excluded_subscriptions: 0,
2892                    terminal: Some(false),
2893                },
2894            );
2895        }
2896        self.emit_route_goodbyes(goodbyes);
2897    }
2898
2899    /// `scope.describe`: any registered module may read any scope, because a
2900    /// provider must read the scope a route it serves is stamped with.
2901    fn handle_scope_describe(
2902        &self,
2903        connection_id: ConnectionId,
2904        frame: Frame,
2905        owner: Principal,
2906        scope_ref: String,
2907    ) -> Result<Vec<Frame>, RouterError> {
2908        let registered = self
2909            .registry
2910            .get_module_by_connection(connection_id)
2911            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2912        if registered.is_none() {
2913            return Ok(vec![control_error_frame(
2914                &frame,
2915                "not_registered",
2916                "scope.describe requires an active module registration owned by this connection",
2917            )?]);
2918        }
2919        let description = self
2920            .scopes
2921            .read()
2922            .unwrap_or_else(|poisoned| poisoned.into_inner())
2923            .describe(&owner, &scope_ref);
2924        let owner_configured = match &owner {
2925            // Ask whether the owner is configured (`is_configured`), not
2926            // whether it is on the roster (`get(..).is_some()`): a supervised
2927            // module's process can register and describe a scope before the
2928            // supervisor has put it on the roster.
2929            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2930            _ => false,
2931        };
2932        let response = ModuleControlResponseToModule::ScopeDescribe {
2933            status: description.status,
2934            scope_epoch: description.scope_epoch,
2935            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2936            owner_synced: description.owner_synced,
2937            owner_configured,
2938            scope: description.stamp,
2939        };
2940        Ok(vec![control_response_body_frame(
2941            &frame,
2942            &response,
2943            "ModuleControlResponseToModule::ScopeDescribe",
2944        )?])
2945    }
2946
2947    fn handle_catalog_update(
2948        &self,
2949        connection_id: ConnectionId,
2950        frame: Frame,
2951        provides: Vec<ProviderRole>,
2952        capabilities: Option<CapabilityDeclarations>,
2953        ready: Option<bool>,
2954    ) -> Result<Vec<Frame>, RouterError> {
2955        self.refresh_capability_requirements();
2956        let Some(registration) = self
2957            .registry
2958            .get_module_by_connection(connection_id)
2959            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2960        else {
2961            return Ok(vec![control_error_frame(
2962                &frame,
2963                "not_registered",
2964                "catalog.update requires an active module registration owned by this connection",
2965            )?]);
2966        };
2967
2968        if let Some(message) =
2969            catalog_update_frozen_field_message(&registration.manifest, &provides)
2970        {
2971            return Ok(vec![control_error_frame(
2972                &frame,
2973                "catalog_update_frozen_field",
2974                message,
2975            )?]);
2976        }
2977
2978        let mut candidate = registration.manifest.clone();
2979        candidate.provides = provides.clone();
2980        candidate.capabilities = capabilities
2981            .clone()
2982            .or_else(|| registration.manifest.capabilities.clone());
2983        if let Err(err) = candidate.validate_capability_grammar() {
2984            return Ok(vec![control_error_frame(
2985                &frame,
2986                "invalid_capability_grammar",
2987                err.to_string(),
2988            )?]);
2989        }
2990
2991        // Updates must honor the same reserved owner as initial registration;
2992        // otherwise an empty HELLO could acquire the claim after admission.
2993        let mut conflicts = self
2994            .capability_evaluator
2995            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2996        if let Some(conflict) = conflicts.first() {
2997            let message = format!(
2998                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2999                conflict.capability, conflict.claimants[0], candidate.module_id
3000            );
3001            for conflict in &mut conflicts {
3002                conflict.source = DuplicateClaimSource::CatalogUpdate;
3003            }
3004            log_duplicate_claim_events(conflicts);
3005            return Ok(vec![control_error_frame(
3006                &frame,
3007                "reserved_capability",
3008                message,
3009            )?]);
3010        }
3011
3012        let updated = self
3013            .registry
3014            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3015            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3016        if updated.is_none() {
3017            return Ok(vec![control_error_frame(
3018                &frame,
3019                "not_registered",
3020                "catalog.update requires an active module registration owned by this connection",
3021            )?]);
3022        }
3023        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3024            log_duplicate_claim_events(
3025                self.capability_evaluator
3026                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3027            );
3028        }
3029        if capability_census_trigger(
3030            registration.manifest.capabilities.as_ref(),
3031            updated
3032                .as_ref()
3033                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3034        ) {
3035            self.enforce_capability_denies();
3036        }
3037        self.refresh_capability_requirements();
3038
3039        let response = ModuleControlResponseToModule::CatalogUpdate {};
3040        control_response_body_frame(
3041            &frame,
3042            &response,
3043            "ModuleControlResponseToModule::CatalogUpdate",
3044        )
3045        .map(|frame| vec![frame])
3046    }
3047
3048    async fn handle_activity_record(
3049        &self,
3050        ctx: &RouteCtx,
3051        frame: Frame,
3052    ) -> Result<Vec<Frame>, RouterError> {
3053        use subc_control::{
3054            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3055            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3056        };
3057        let result = async {
3058            let module = self
3059                .forwarding
3060                .module_id_for_connection(ctx.connection_id)
3061                .ok()
3062                .flatten()
3063                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3064            let request: ActivityRecordRequest =
3065                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3066            crate::activity::validate(&module, &request)?;
3067            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3068            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3069            let at_ms = std::time::SystemTime::now()
3070                .duration_since(std::time::UNIX_EPOCH)
3071                .unwrap_or_default()
3072                .as_millis() as u64;
3073            ledger.record(module, request, actor, at_ms).await
3074        }
3075        .await;
3076        match result {
3077            Ok(entry) => Ok(vec![control_response_body_frame(
3078                &frame,
3079                &ActivityRecordResponse { seq: entry.seq },
3080                "activity.record",
3081            )?]),
3082            Err(code) => Ok(vec![control_error_frame(
3083                &frame,
3084                code,
3085                "activity recording refused",
3086            )?]),
3087        }
3088    }
3089
3090    fn activity_actor(
3091        &self,
3092        connection: ConnectionId,
3093        module: &str,
3094        on: Option<&subc_control::ActivityOn>,
3095    ) -> Result<subc_control::ActivityActor, &'static str> {
3096        use crate::forwarding::{DataRoute, DataRouteState};
3097        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3098        let principal = |id: &str| Principal::Reserved {
3099            module_id: id.to_owned(),
3100        };
3101        let mut actor = ActivityActor {
3102            principal: principal(module),
3103            scope_owner: None,
3104            agent_id: None,
3105            run_id: None,
3106            flow_id: None,
3107            project_root: None,
3108        };
3109        let scope = match on {
3110            None => return Ok(actor),
3111            Some(ActivityOn::Scope(scope)) => {
3112                let selector = subc_protocol::scope::ScopeSelector {
3113                    owner: principal(module),
3114                    scope_ref: scope.scope_ref.clone(),
3115                    scope_epoch: Some(scope.epoch),
3116                };
3117                self.scopes
3118                    .read()
3119                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3120                    .admit(&principal(module), module, &selector, true)
3121                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3122            }
3123            Some(ActivityOn::Route(route)) => {
3124                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3125                    .forwarding
3126                    .lookup_data_route(connection, route.channel, route.epoch)
3127                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3128                else {
3129                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3130                };
3131                if binding.module_id != module {
3132                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3133                }
3134                actor.principal = binding.principal.clone();
3135                actor.project_root = binding
3136                    .project_root
3137                    .as_ref()
3138                    .map(|root| root.as_path().to_path_buf());
3139                let Some(bound) = &binding.scope else {
3140                    return Ok(actor);
3141                };
3142                let owner = principal(&bound.owner);
3143                let selector = subc_protocol::scope::ScopeSelector {
3144                    owner: owner.clone(),
3145                    scope_ref: bound.scope_ref.clone(),
3146                    scope_epoch: Some(bound.tag.scope_epoch),
3147                };
3148                let admitted = self
3149                    .scopes
3150                    .read()
3151                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3152                    .admit(&owner, module, &selector, true)
3153                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3154                if admitted.tag != bound.tag {
3155                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3156                }
3157                admitted
3158            }
3159        };
3160        actor.scope_owner = Some(scope.stamp.owner);
3161        actor.agent_id = scope.stamp.attributes.agent_id;
3162        actor.run_id = scope.stamp.attributes.run_id;
3163        actor.flow_id = scope.stamp.attributes.flow_id;
3164        Ok(actor)
3165    }
3166
3167    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3168        self.refresh_capability_requirements();
3169        // A bare connection count is ambiguous between many clients holding a
3170        // route each and one client accumulating hundreds, so publish the
3171        // concentration alongside it. Route state is best-effort here: a
3172        // diagnostic endpoint must still answer if the forwarding lock is
3173        // contended.
3174        let mut counters = self.counters.snapshot();
3175        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3176            obj.insert(
3177                "activity_unparseable_lines".into(),
3178                ledger.read_errors().into(),
3179            );
3180        }
3181        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3182            self.forwarding.client_route_concentration(),
3183            counters.as_object_mut(),
3184        ) {
3185            obj.insert(
3186                "client_connections_with_routes".into(),
3187                connections_with_routes.into(),
3188            );
3189            obj.insert("max_routes_on_one_connection".into(), max.into());
3190        }
3191        // A module that is being fast-refused and a module that is fine look
3192        // identical from a client that retries and succeeds, so name the open
3193        // breakers here. This rides the existing free-form counters object
3194        // rather than a new wire field, so no sibling that deserializes
3195        // `ServerDescribe` has to be rebuilt to keep reading it.
3196        if let (Some(open_breakers), Some(obj)) = (
3197            self.route_bind_breakers.open_snapshot(),
3198            counters.as_object_mut(),
3199        ) {
3200            obj.insert("route_bind_breakers_open".into(), open_breakers);
3201        }
3202        let response = ClientControlResponse::ServerDescribe {
3203            protocol_ver: PROTOCOL_VERSION,
3204            subc_ops: subc_ops(),
3205            capabilities: self.subc_capabilities.as_ref().to_vec(),
3206            connected_clients: self.connected_clients.count(),
3207            counters: Some(counters),
3208            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3209            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3210            capability_requirements: self.capability_requirement_statuses(),
3211            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3212        };
3213        Ok(vec![control_response_body_frame(
3214            &frame,
3215            &response,
3216            "ClientControlResponse::ServerDescribe",
3217        )?])
3218    }
3219
3220    fn handle_catalog_list(
3221        &self,
3222        frame: Frame,
3223        module_id: Option<String>,
3224    ) -> Result<Vec<Frame>, RouterError> {
3225        let (generation, modules) = self.registry.list_modules().map_err(|err| {
3226            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3227        })?;
3228        let entries = modules
3229            .into_iter()
3230            .filter(|registration| {
3231                module_id
3232                    .as_deref()
3233                    .map(|wanted| registration.manifest.module_id == wanted)
3234                    .unwrap_or(true)
3235            })
3236            .map(|registration| {
3237                let not_ready = self.not_ready_reason(&registration);
3238                let roles = registration.manifest.provides;
3239                CatalogEntry::new(
3240                    registration.manifest.module_id,
3241                    roles,
3242                    registration.control_ops,
3243                )
3244                .with_ready(not_ready.is_none())
3245                .with_not_ready(not_ready)
3246                .with_module_version(Some(registration.manifest.module_version))
3247                .with_capabilities(registration.manifest.capabilities)
3248                .with_self_signals(registration.manifest.self_signals)
3249                .with_events(registration.manifest.events)
3250            })
3251            .collect();
3252        let response = ClientControlResponse::CatalogList {
3253            generation,
3254            modules: entries,
3255            subc_ops: subc_ops(),
3256        };
3257        Ok(vec![control_response_body_frame(
3258            &frame,
3259            &response,
3260            "ClientControlResponse::CatalogList",
3261        )?])
3262    }
3263
3264    fn route_open_principal(
3265        &self,
3266        frame: &Frame,
3267        consumer_identity: Option<ConsumerIdentity>,
3268    ) -> Result<Result<Principal, Frame>, RouterError> {
3269        let Some(consumer_identity) = consumer_identity else {
3270            return Ok(Ok(Principal::Direct));
3271        };
3272
3273        if self.supervisor.spawned_consumer_authorized(
3274            &consumer_identity.module_id,
3275            &consumer_identity.launch_nonce,
3276        ) {
3277            return Ok(Ok(Principal::Reserved {
3278                module_id: consumer_identity.module_id,
3279            }));
3280        }
3281
3282        Ok(Err(control_error_frame(
3283            frame,
3284            "bad_consumer_identity",
3285            format!(
3286                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3287                consumer_identity.module_id
3288            ),
3289        )?))
3290    }
3291
3292    /// Ordinary `route.open` refusals go through here; admission and breaker
3293    /// refusals log separately with their capacity or breaker state. The daemon can
3294    /// attest which code it sent: without the event, a client's "the daemon
3295    /// refused me" and the daemon's own view could only be reconciled by
3296    /// argument. Malformed input (`invalid_project_root`) does not come here;
3297    /// rejecting a request that was never a valid open is not a refusal of one.
3298    fn route_open_refusal_frame(
3299        &self,
3300        ctx: &RouteCtx,
3301        frame: &Frame,
3302        module_id: &str,
3303        reason: &'static str,
3304        code: &'static str,
3305        message: impl Into<String>,
3306    ) -> Result<Frame, RouterError> {
3307        self.observe_route_open_refusal(ctx, module_id, reason, code);
3308        control_error_frame(frame, code, message.into())
3309    }
3310
3311    /// Refuse a `route.open` because the target module's bind-relay breaker is
3312    /// open, without attempting the relay.
3313    ///
3314    /// The wire code is `module_timeout`, which is the truth (the module has
3315    /// not been answering binds) and which both SDKs already classify as
3316    /// retryable with capped backoff. Reusing it is what keeps this change out
3317    /// of both SDKs; the daemon-side distinction lives in the counter key
3318    /// instead.
3319    ///
3320    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
3321    /// While a breaker is open this fires on every open to that module, and the
3322    /// stall written up in `docs/designs/route-open-head-of-line.md` already
3323    /// produced 261 lines about a single module inside 3000 lines of daemon
3324    /// log. The rare transitions are logged at warn/info instead and the volume
3325    /// is carried by the counter, so the evidence survives without the flood.
3326    /// The debug line keeps a per-refusal record reachable for whoever turns
3327    /// the level up.
3328    fn route_open_breaker_refusal_frame(
3329        &self,
3330        ctx: &RouteCtx,
3331        frame: &Frame,
3332        module_id: &str,
3333        consecutive_timeouts: u32,
3334        retry_in: Duration,
3335        probe_in_flight: bool,
3336    ) -> Result<Frame, RouterError> {
3337        self.counters
3338            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3339        debug!(
3340            target: "control",
3341            code = "module_timeout",
3342            module_id = ?module_id,
3343            connection_id = ctx.connection_id.get(),
3344            consecutive_timeouts,
3345            retry_in_ms = retry_in.as_millis() as u64,
3346            probe_in_flight,
3347            "route.open refused by open bind-relay breaker"
3348        );
3349        // Say what a caller can act on. An open bind-relay breaker means the
3350        // module timed out accepting several new routes in a row. The module
3351        // is still running and its established routes keep working; only new
3352        // route.open requests are refused until the cooldown ends and one
3353        // test route (the probe) gets through. A message that only counts
3354        // failed relays reads as "the module is down" to a worker that sees it.
3355        let detail = if probe_in_flight {
3356            "one test route is already being tried; retry once it settles".to_string()
3357        } else {
3358            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3359        };
3360        control_error_frame(
3361            frame,
3362            "module_timeout",
3363            format!(
3364                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3365                 timed out in a row); {detail}; its established routes are unaffected"
3366            ),
3367        )
3368    }
3369
3370    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
3371    /// requester's bytes (an unknown target is whatever the client sent) and
3372    /// is Debug-formatted so control characters land in the log escaped
3373    /// rather than as terminal sequences for whoever tails it.
3374    ///
3375    /// `reason` names the check that refused, because one wire code has
3376    /// several senders: after a module registers, `target_unavailable` can
3377    /// come from a missing role, an inactive registration, a supervisor that
3378    /// has not marked the process live, a missing forwarding connection, or a
3379    /// failed relay, and a log that records only the code cannot say which of
3380    /// them fired. It is a static, daemon-chosen label per branch, so it is
3381    /// safe to print plainly and stays a closed set.
3382    fn observe_route_open_refusal(
3383        &self,
3384        ctx: &RouteCtx,
3385        module_id: &str,
3386        reason: &'static str,
3387        code: &'static str,
3388    ) {
3389        self.counters.increment_route_open_refused(code);
3390        info!(
3391            target: "control",
3392            code,
3393            reason,
3394            module_id = ?module_id,
3395            connection_id = ctx.connection_id.get(),
3396            "route.open refused"
3397        );
3398        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3399            self.route_outages.record_not_serving(module_id, reason);
3400        }
3401    }
3402
3403    /// Record an ACCEPTED route.open.
3404    ///
3405    /// Refusals have been logged and counted since the attestation work; accepts
3406    /// were invisible, so the daemon knew every principal it stamped and wrote
3407    /// none of them down. The party that attests the identity was the only party
3408    /// not recording it, which left a credential vault unable to name the sender
3409    /// of a call that reached it (claustrum #43) and left the launch-nonce
3410    /// concurrency question unanswerable from the outside.
3411    ///
3412    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
3413    /// `code`/`module_id`/`connection_id` returns both directions of the same
3414    /// decision rather than two shapes a reader has to join by hand.
3415    ///
3416    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
3417    /// and the difference carries information rather than being an
3418    /// inconsistency. This line is only reachable after a successful bind to a
3419    /// REGISTERED module, so the value has already passed HELLO validation
3420    /// including the path-hazard refusal and cannot contain control bytes. A
3421    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
3422    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
3423    ///
3424    /// Bare is also what every other daemon line already emits (`module
3425    /// registered`, `configured module supervised`). Shipping `?module_id` here
3426    /// made this instrument the only one in the file whose ids did not answer
3427    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
3428    /// line whose whole purpose is being grepped beside its sibling.
3429    ///
3430    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
3431    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
3432    /// forwards every field type through it and a bare `&str` and a `?`-escaped
3433    /// one are recorded identically. A test written against that harness passes
3434    /// either way -- I wrote one, measured it, and deleted it rather than ship a
3435    /// green assertion that cannot fail. The same limit applies to the escaping
3436    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
3437    /// it reads as a guard on the Debug escaping and cannot detect its removal.
3438    /// Fencing either needs the real formatter, not the capture layer.
3439    ///
3440    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
3441    /// unix-socket options and subc is loopback TCP, so there is no peer identity
3442    /// to record. The ephemeral port would decay within minutes and answer only a
3443    /// live question. The identity question is instead answered by counting
3444    /// distinct live connections presenting one module's `consumer_identity` --
3445    /// "is anyone else holding this secret" rather than "is this the right
3446    /// process".
3447    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3448        self.route_outages.record_accepted(module_id);
3449        self.counters.increment_route_open_accepted(principal);
3450        info!(
3451            target: "control",
3452            principal,
3453            module_id,
3454            connection_id = ctx.connection_id.get(),
3455            "route.open accepted"
3456        );
3457    }
3458
3459    fn supervised_absent_route_open_refusal_frame(
3460        &self,
3461        ctx: &RouteCtx,
3462        frame: &Frame,
3463        module_id: &str,
3464        code: &'static str,
3465        status: &crate::supervise::ModuleStatus,
3466    ) -> Result<Frame, RouterError> {
3467        self.counters.increment_route_open_refused(code);
3468        info!(
3469            target: "control",
3470            code,
3471            reason = "supervised_not_registered",
3472            module_id = ?module_id,
3473            connection_id = ctx.connection_id.get(),
3474            state = %status.state,
3475            enabled = status.enabled,
3476            live = status.live,
3477            "route.open refused"
3478        );
3479        // A supervised module whose process has not registered is not
3480        // serving, whatever the reason; the supervisor knows this id, so it is
3481        // safe to track.
3482        self.route_outages
3483            .record_not_serving(module_id, "supervised_not_registered");
3484        let message = if let Some(pid) = status.waiting_for_previous_pid {
3485            format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3486        } else {
3487            format!(
3488                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3489                status.state, status.enabled, status.live
3490            )
3491        };
3492        control_error_frame(frame, code, message)
3493    }
3494
3495    async fn handle_route_open(
3496        &self,
3497        ctx: &RouteCtx,
3498        frame: Frame,
3499        request: RouteOpenRequest,
3500    ) -> Result<Vec<Frame>, RouterError> {
3501        let RouteOpenRequest {
3502            target,
3503            mut identity,
3504            consumer_identity,
3505            consumer_capabilities,
3506            role_versions,
3507            admission_facts,
3508            scope,
3509        } = request;
3510        let target_module_id = target_module_id(&target).to_string();
3511        if self
3512            .registry
3513            .get_module_by_connection(ctx.connection_id)
3514            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3515            .is_some()
3516        {
3517            return Ok(vec![control_error_frame(
3518                &frame,
3519                "invalid_request",
3520                "module connections cannot open client routes",
3521            )?]);
3522        }
3523        debug!(
3524            connection_id = ctx.connection_id.get(),
3525            corr = frame.header.corr,
3526            module_id = %target_module_id,
3527            "handling route.open"
3528        );
3529
3530        // A malformed declaration is refused first, before anything about the
3531        // target is looked up: the same body would be refused against any
3532        // module, so the caller learns nothing by retrying or waiting. An empty
3533        // map declares nothing and travels as no field at all, so a provider
3534        // only ever sees a missing field or a non-empty one.
3535        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3536        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3537            self.observe_route_open_refusal(
3538                ctx,
3539                &target_module_id,
3540                "invalid_role_versions",
3541                error_codes::INVALID_REQUEST,
3542            );
3543            return Ok(vec![control_error_body_frame(
3544                &frame,
3545                ErrorBody {
3546                    code: error_codes::INVALID_REQUEST.to_string(),
3547                    message: error.to_string(),
3548                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3549                },
3550            )?]);
3551        }
3552
3553        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3554            if status.waiting_for_previous_pid.is_some() {
3555                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3556                    ctx,
3557                    &frame,
3558                    &target_module_id,
3559                    "module_warming",
3560                    &status,
3561                )?]);
3562            }
3563        }
3564
3565        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
3566        // opposite. Below, a caller learns whether a module is unregistered,
3567        // supervised-but-down (with state/enabled/live), or registered without the
3568        // requested role. Elsewhere that is an enumeration leak: a probe learning
3569        // the shape of a fleet it cannot otherwise see.
3570        //
3571        // It is not one here, and the reason is the ACCESS MODEL rather than
3572        // anything about these errors. Reaching route.open requires the
3573        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
3574        // connection file, so any caller who completes it already runs as this
3575        // user -- and can read subc.jsonc for the module list and `ck module
3576        // status` for live state. The reply discloses nothing the caller cannot
3577        // read more easily from disk, while the precision is load-bearing:
3578        // `unknown_module` is retryable and a missing role is not.
3579        //
3580        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
3581        // remote transport, a sandboxed caller, a shared-host mode -- THAT
3582        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
3583        let Some(registration) = self
3584            .registry
3585            .get_module(&target_module_id)
3586            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3587        else {
3588            if let Some((status, warming)) =
3589                self.supervisor_status(&target_module_id, frame.header.corr)?
3590            {
3591                // BEFORE the two availability codes below, because for a module
3592                // that speaks no subc wire both of them are false comfort: they
3593                // say "not right now" and are retried, and this module will
3594                // never register no matter how long the caller waits. The
3595                // absence here is the declaration being honoured, not a module
3596                // that is late.
3597                if status.protocol == ModuleProtocol::None {
3598                    return Ok(vec![self.route_open_refusal_frame(
3599                        ctx,
3600                        &frame,
3601                        &target_module_id,
3602                        "protocol_none",
3603                        error_codes::MODULE_NO_PROTOCOL,
3604                        format!(
3605                            "module_id '{target_module_id}' is declared protocol: none; \
3606                             it speaks no subc wire and serves no routes"
3607                        ),
3608                    )?]);
3609                }
3610                let code = if warming {
3611                    "module_warming"
3612                } else {
3613                    "target_unavailable"
3614                };
3615                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3616                    ctx,
3617                    &frame,
3618                    &target_module_id,
3619                    code,
3620                    &status,
3621                )?]);
3622            }
3623            if let Some(removed_ago_ms) =
3624                self.supervisor.removal_tombstone_age_ms(&target_module_id)
3625            {
3626                return Ok(vec![self.route_open_refusal_frame(
3627                    ctx,
3628                    &frame,
3629                    &target_module_id,
3630                    "removed",
3631                    error_codes::MODULE_REMOVED,
3632                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3633                )?]);
3634            }
3635            return Ok(vec![self.route_open_refusal_frame(
3636                ctx,
3637                &frame,
3638                &target_module_id,
3639                "not_registered",
3640                error_codes::UNKNOWN_MODULE,
3641                format!("module_id '{target_module_id}' is not registered"),
3642            )?]);
3643        };
3644
3645        // Best-effort only: registry readiness and forwarding reservation use
3646        // different locks, so a module can flip readiness between this read and
3647        // the relay. Modules must still tolerate an `on_bind` while not ready.
3648        if !registration.ready {
3649            self.counters
3650                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3651            info!(
3652                target: "control",
3653                code = error_codes::MODULE_WARMING,
3654                module_id = ?target_module_id,
3655                connection_id = ctx.connection_id.get(),
3656                reason = "declared_not_ready",
3657                "route.open refused"
3658            );
3659            // The module is registered but says it cannot take work, which is
3660            // an outage from the caller's side even though its process is up.
3661            self.route_outages
3662                .record_not_serving(&target_module_id, "declared_not_ready");
3663            return Ok(vec![control_error_body_frame(
3664                &frame,
3665                ErrorBody {
3666                    code: error_codes::MODULE_WARMING.to_string(),
3667                    message: format!(
3668                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3669                    ),
3670                    detail: Some(serde_json::json!({
3671                        "reason": "declared_not_ready"
3672                    })),
3673                },
3674            )?]);
3675        }
3676
3677        // Effective readiness, second half: a module that declares a capability
3678        // `need: required` is not routable while that capability has no
3679        // registered provider. It is enforced HERE, as a retryable routing
3680        // refusal, and deliberately not as spawn ordering or a boot block. The
3681        // module is still started and registered and can make its own calls;
3682        // spawn ordering is a promise that cannot be kept once a provider
3683        // crashes at runtime, and refusing to boot would stop the whole
3684        // machine, including the tools needed to fix its configuration.
3685        //
3686        // "Provided" is the evaluator's verdict, which counts a provider as
3687        // soon as it has REGISTERED, not once it is ready. Two modules that
3688        // require each other's capabilities are therefore both routable once
3689        // both register; counting readiness instead would deadlock them.
3690        //
3691        // Only new opens are refused. Routes already bound when a provider
3692        // goes away stay bound: nothing here tears them down, and the module
3693        // answers them as it can. Like the readiness read above this is
3694        // best-effort against a provider registering or leaving concurrently.
3695        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
3696            self.counters
3697                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3698            info!(
3699                target: "control",
3700                code = error_codes::MODULE_WARMING,
3701                module_id = ?target_module_id,
3702                connection_id = ctx.connection_id.get(),
3703                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3704                capability = %capability,
3705                "route.open refused"
3706            );
3707            return Ok(vec![control_error_body_frame(
3708                &frame,
3709                ErrorBody {
3710                    code: error_codes::MODULE_WARMING.to_string(),
3711                    message: format!(
3712                        "module_id '{target_module_id}' requires capability '{capability}', \
3713                         which no registered module provides; retry"
3714                    ),
3715                    detail: Some(serde_json::json!({
3716                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3717                        "capability": capability,
3718                    })),
3719                },
3720            )?]);
3721        }
3722
3723        if !target_has_required_role(&target, &registration.manifest.provides) {
3724            return Ok(vec![self.route_open_refusal_frame(
3725                ctx,
3726                &frame,
3727                &target_module_id,
3728                "role_not_provided",
3729                "target_unavailable",
3730                format!("module_id '{target_module_id}' does not provide the requested target"),
3731            )?]);
3732        }
3733
3734        if registration.state != ChannelState::Active {
3735            return Ok(vec![self.route_open_refusal_frame(
3736                ctx,
3737                &frame,
3738                &target_module_id,
3739                "registration_not_active",
3740                "target_unavailable",
3741                format!("module_id '{target_module_id}' is not active"),
3742            )?]);
3743        }
3744
3745        if self
3746            .forwarding
3747            .module_is_draining(&target_module_id)
3748            .map_err(RouterError::Forwarding)?
3749        {
3750            return Ok(vec![self.route_open_refusal_frame(
3751                ctx,
3752                &frame,
3753                &target_module_id,
3754                "reloading",
3755                "module_reloading",
3756                format!("module_id '{target_module_id}' is reloading"),
3757            )?]);
3758        }
3759
3760        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3761            process_liveness.process_live(&target_module_id) == Some(false)
3762        }) {
3763            // A module the supervisor is restarting or reloading can still hold
3764            // a registration: the old process before its connection closes, or
3765            // a new one that registered while the supervisor was draining. The
3766            // forwarding table does not see that as draining, but the consumer
3767            // should still be told to retry soon, exactly as for the drain
3768            // above, rather than that the target is unavailable.
3769            if process_liveness.process_replacing(&target_module_id) {
3770                return Ok(vec![self.route_open_refusal_frame(
3771                    ctx,
3772                    &frame,
3773                    &target_module_id,
3774                    "reloading",
3775                    "module_reloading",
3776                    format!("module_id '{target_module_id}' is reloading"),
3777                )?]);
3778            }
3779            return Ok(vec![self.route_open_refusal_frame(
3780                ctx,
3781                &frame,
3782                &target_module_id,
3783                "supervisor_not_live",
3784                "target_unavailable",
3785                format!("module_id '{target_module_id}' is not live"),
3786            )?]);
3787        }
3788
3789        if !self
3790            .forwarding
3791            .has_live_module_connection(&target_module_id)
3792            .map_err(RouterError::Forwarding)?
3793        {
3794            return Ok(vec![self.route_open_refusal_frame(
3795                ctx,
3796                &frame,
3797                &target_module_id,
3798                "no_forwarding_connection",
3799                "target_unavailable",
3800                format!("module_id '{target_module_id}' has no live forwarding connection"),
3801            )?]);
3802        }
3803
3804        if let Some(error) =
3805            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3806        {
3807            self.observe_route_open_refusal(
3808                ctx,
3809                &target_module_id,
3810                "op_not_allowed",
3811                "op_not_allowed",
3812            );
3813            return Ok(vec![error]);
3814        }
3815
3816        let principal = match self.route_open_principal(&frame, consumer_identity)? {
3817            Ok(principal) => principal,
3818            Err(error) => {
3819                self.observe_route_open_refusal(
3820                    ctx,
3821                    &target_module_id,
3822                    "bad_consumer_identity",
3823                    "bad_consumer_identity",
3824                );
3825                return Ok(vec![error]);
3826            }
3827        };
3828
3829        // This is attested, control-plane policy for supervised module origins.
3830        // Keep it before route reservation and out of the opaque forwarding hot
3831        // path: data frames must never acquire a per-frame capability check.
3832        if let Principal::Reserved {
3833            module_id: opening_module_id,
3834        } = &principal
3835        {
3836            if let Some(opening_registration) = self
3837                .registry
3838                .get_module(opening_module_id)
3839                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3840            {
3841                if let Some(capability) =
3842                    denied_capability(&opening_registration.manifest, &registration.manifest)
3843                {
3844                    warn!(
3845                        opening_module_id,
3846                        target_module_id,
3847                        capability,
3848                        "refusing route.open because an attested capability deny edge matches"
3849                    );
3850                    return Ok(vec![self.route_open_refusal_frame(
3851                        ctx,
3852                        &frame,
3853                        &target_module_id,
3854                        "capability_deny_edge",
3855                        "capability_forbidden",
3856                        format!(
3857                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3858                        ),
3859                    )?]);
3860                }
3861            }
3862        }
3863
3864        if admission_facts.is_some() {
3865            let carrier_matches = matches!(
3866                &principal,
3867                Principal::Reserved { module_id }
3868                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3869            );
3870            if !carrier_matches {
3871                return Ok(vec![self.route_open_refusal_frame(
3872                    ctx,
3873                    &frame,
3874                    &target_module_id,
3875                    "admission_facts_carrier_not_permitted",
3876                    "admission_facts_not_permitted",
3877                    "admission facts may only be carried by the configured reserved module",
3878                )?]);
3879            }
3880
3881            let target_allowed = self
3882                .admission_facts_targets
3883                .as_ref()
3884                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3885            if !target_allowed {
3886                return Ok(vec![self.route_open_refusal_frame(
3887                    ctx,
3888                    &frame,
3889                    &target_module_id,
3890                    "admission_facts_target_not_listed",
3891                    "admission_facts_target_not_allowed",
3892                    format!(
3893                        "admission facts are not permitted for target module_id '{target_module_id}'"
3894                    ),
3895                )?]);
3896            }
3897
3898            // Keep the value opaque to subc. The downstream admission validator owns
3899            // schema and semantic checks; this daemon only enforces carrier authority
3900            // and the configured destination allowlist.
3901        }
3902
3903        // Scope admission, on the attested principal above and never on the
3904        // request body. The tag read here travels with the pending bind and is
3905        // compared with the published one at commit, so a sync between here
3906        // and the module's ack refuses the open instead of binding a stamp
3907        // that is no longer true.
3908        let (bound_scope, scope_stamp) = match scope {
3909            None => (None, None),
3910            Some(selector) => {
3911                let owner_configured = match &selector.owner {
3912                    // A reserved owner counts as configured from before its
3913                    // process is spawned (see `SupervisorHandle::is_configured`).
3914                    // So an owner that has not synced its scopes yet is refused
3915                    // as retryable (`scope_not_synced`), not as one that will
3916                    // never sync.
3917                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3918                    _ => false,
3919                };
3920                let admitted = self
3921                    .scopes
3922                    .read()
3923                    .unwrap_or_else(|poisoned| poisoned.into_inner())
3924                    .admit(&principal, &target_module_id, &selector, owner_configured)
3925                    .and_then(|admission| {
3926                        crate::scopes::check_target_flow_support(
3927                            &admission.stamp,
3928                            &target_module_id,
3929                            registration.manifest.capabilities.as_ref(),
3930                        )?;
3931                        crate::scopes::check_target_agent_run_support(
3932                            &admission.stamp,
3933                            &target_module_id,
3934                            registration.manifest.capabilities.as_ref(),
3935                        )?;
3936                        Ok(admission)
3937                    });
3938                match admitted {
3939                    Ok(admission) => (
3940                        Some(BoundScope {
3941                            owner: admission.owner,
3942                            scope_ref: admission.stamp.scope_ref.clone(),
3943                            tag: admission.tag,
3944                        }),
3945                        Some(admission.stamp),
3946                    ),
3947                    Err(refusal) => {
3948                        return Ok(vec![self.route_open_refusal_frame(
3949                            ctx,
3950                            &frame,
3951                            &target_module_id,
3952                            refusal.code,
3953                            refusal.code,
3954                            refusal.message,
3955                        )?]);
3956                    }
3957                }
3958            }
3959        };
3960
3961        // Bind admits a root that no longer exists on disk, because refusing here
3962        // closes the only exit from a paused run: cancel needs a bound route, and a
3963        // renamed or reclaimed directory makes that route unopenable forever. The
3964        // run itself is intact and still addressable by its recorded identity.
3965        //
3966        // This does NOT relax the rule the strict constructor protects. That rule is
3967        // that no root is ever aliased into NEW durable state -- a missing component
3968        // can reappear as a symlink elsewhere, which would move the identity and
3969        // split a session's history across two of them. The engine now refuses the
3970        // two operations that create such state (send and import) at admission,
3971        // which is a narrower way to hold the same invariant: reads and terminations
3972        // are admitted, writes are not. That refusal had to ship before this line
3973        // changed, or there is an interval where a send commits under a provisional
3974        // identity -- the exact failure the original policy existed to prevent.
3975        //
3976        // Resolution follows realpath rather than lexical cleanup: the longest
3977        // existing ancestor is canonicalized and the missing tail re-appended, so a
3978        // live root is unchanged and a vanished leaf keeps the identity it was
3979        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
3980        // same caller the moment the directory vanished, which strands the run more
3981        // quietly than refusing it.
3982        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3983            Ok(project_root) => project_root,
3984            Err(err) => {
3985                return Ok(vec![control_error_frame(
3986                    &frame,
3987                    "invalid_project_root",
3988                    err.to_string(),
3989                )?])
3990            }
3991        };
3992        identity.project_root = project_root.as_path().to_path_buf();
3993
3994        // Last gate before any relay work, and deliberately after the cheap
3995        // registry and availability checks above: those name a more precise
3996        // condition (unknown, removed, reloading) and a caller is better served
3997        // by the precise code than by this one.
3998        //
3999        // Everything below this point costs an egress permit, a reserved handle
4000        // pair and, if the module does not answer, the whole relay budget. The
4001        // reader no longer waits for that budget, so cap each target explicitly;
4002        // serial dispatch used to provide the accidental cap of one relay per
4003        // connection. Admission is a mutex-protected count and never waits.
4004        let _concurrency_guard = match self
4005            .route_bind_concurrency
4006            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4007        {
4008            Ok(guard) => guard,
4009            Err(in_flight) => {
4010                return Ok(vec![self.route_open_target_capacity_refusal(
4011                    ctx,
4012                    &frame,
4013                    &target_module_id,
4014                    in_flight,
4015                )?]);
4016            }
4017        };
4018
4019        // A module that has already burned the whole budget `threshold` times
4020        // in a row does not get to charge it again until a probe says it recovered.
4021        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4022            RouteBindAdmission::Admitted { guard, probe } => {
4023                if probe {
4024                    info!(
4025                        module_id = %target_module_id,
4026                        connection_id = ctx.connection_id.get(),
4027                        "route.bind breaker half-open: admitting one probe"
4028                    );
4029                }
4030                guard
4031            }
4032            RouteBindAdmission::Refused {
4033                consecutive_timeouts,
4034                retry_in,
4035                probe_in_flight,
4036            } => {
4037                return Ok(vec![self.route_open_breaker_refusal_frame(
4038                    ctx,
4039                    &frame,
4040                    &target_module_id,
4041                    consecutive_timeouts,
4042                    retry_in,
4043                    probe_in_flight,
4044                )?]);
4045            }
4046        };
4047
4048        // Resolve the per-module budget here so the wait matches the operator's
4049        // intent for this specific target. A per-module override in
4050        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4051        // daemons) wins over the daemon-wide default.
4052        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4053        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4054        let pending = match self
4055            .forwarding
4056            .begin_route_bind_relay_for(
4057                ctx.connection_id,
4058                ctx.egress.clone(),
4059                response_version(&frame),
4060                frame.header.corr,
4061                &target_module_id,
4062                principal.clone(),
4063                bound_scope,
4064                Some(project_root),
4065                relay_deadline,
4066            )
4067            .await
4068        {
4069            Ok(pending) => pending,
4070            Err(err) => {
4071                return Ok(vec![self.route_open_refusal_frame(
4072                    ctx,
4073                    &frame,
4074                    &target_module_id,
4075                    "relay_reservation_failed",
4076                    forwarding_error_code(&err),
4077                    err.to_string(),
4078                )?])
4079            }
4080        };
4081        let crate::forwarding::PendingRouteBindRelay {
4082            endpoint,
4083            module_sink,
4084            negotiated_ver,
4085            client_channel,
4086            client_epoch,
4087            module_channel,
4088            module_epoch,
4089            corr: relay_corr,
4090            receiver,
4091        } = pending;
4092        let mut reservation =
4093            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4094
4095        // Reserving egress can wait while a module reconnects or a swap cuts
4096        // over. Check the relay's captured connection, not the earlier by-id
4097        // lookup: the original module's flow/run capabilities cannot authorize
4098        // its replacement. The captured sink stays bound to that connection.
4099        if let Some(stamp) = scope_stamp
4100            .as_ref()
4101            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4102        {
4103            let relay_registration = self
4104                .registry
4105                .get_module_by_connection(endpoint.connection_id)
4106                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4107            if let Err(refusal) = crate::scopes::check_target_flow_support(
4108                stamp,
4109                &target_module_id,
4110                relay_registration
4111                    .as_ref()
4112                    .and_then(|registration| registration.manifest.capabilities.as_ref()),
4113            )
4114            .and_then(|()| {
4115                crate::scopes::check_target_agent_run_support(
4116                    stamp,
4117                    &target_module_id,
4118                    relay_registration
4119                        .as_ref()
4120                        .and_then(|registration| registration.manifest.capabilities.as_ref()),
4121                )
4122            }) {
4123                reservation.release_and_disarm();
4124                return Ok(vec![self.route_open_refusal_frame(
4125                    ctx,
4126                    &frame,
4127                    &target_module_id,
4128                    refusal.code,
4129                    refusal.code,
4130                    refusal.message,
4131                )?]);
4132            }
4133        }
4134
4135        debug!(
4136            connection_id = ctx.connection_id.get(),
4137            client_channel,
4138            client_epoch,
4139            module_channel,
4140            module_epoch,
4141            "reserved route handle pair"
4142        );
4143        // Rendered BEFORE the move into the relay, because the accept arm below
4144        // is where it is logged and the principal is gone by then.
4145        let principal_label = principal_label(&principal);
4146        let relay = ModuleControlRequest::RouteBind {
4147            route_channel: module_channel,
4148            epoch: module_epoch,
4149            target,
4150            identity,
4151            principal: Some(principal),
4152            consumer_capabilities,
4153            role_versions,
4154            admission_facts,
4155            scope: scope_stamp,
4156        };
4157        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4158            RouterError::backend(
4159                0,
4160                frame.header.corr,
4161                format!("failed to encode route.bind request: {err}"),
4162            )
4163        })?;
4164        let relay_frame = Frame::build_with_version(
4165            negotiated_ver,
4166            FrameType::Request,
4167            control_flags(),
4168            0,
4169            0,
4170            relay_corr,
4171            relay_body,
4172        )
4173        .map_err(RouterError::FrameBuild)?;
4174
4175        if let Err(err) = module_sink.send(relay_frame).await {
4176            reservation.release_and_disarm();
4177            return Ok(vec![self.route_open_refusal_frame(
4178                ctx,
4179                &frame,
4180                &target_module_id,
4181                "relay_send_failed",
4182                "target_unavailable",
4183                err.to_string(),
4184            )?]);
4185        }
4186
4187        #[cfg(test)]
4188        if let Some(hold) = &self.route_bind_relay_sent_hold {
4189            // First wait: tell the test the bind is queued. Second wait: stay
4190            // here until the test has driven whatever it wants to happen
4191            // between queueing the bind and marking it enqueued.
4192            hold.wait().await;
4193            hold.wait().await;
4194        }
4195
4196        // A missing relay is not by itself an abandoned bind: the module may
4197        // already have acked it and the route may be committed. Only a relay
4198        // dropped without any module reply needs this GOODBYE; one a reply
4199        // settled has already had its GOODBYE decided by the reply path.
4200        if self
4201            .forwarding
4202            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4203            .map_err(RouterError::Forwarding)?
4204            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4205        {
4206            self.send_abandoned_route_bind_goodbye(
4207                &module_sink,
4208                negotiated_ver,
4209                module_channel,
4210                module_epoch,
4211            );
4212        }
4213
4214        match timeout_at(relay_deadline, receiver).await {
4215            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4216                reservation.disarm();
4217                if breaker.record_accepted() {
4218                    info!(
4219                        module_id = %target_module_id,
4220                        "route.bind breaker closed: the probe was accepted"
4221                    );
4222                }
4223                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4224                Ok(Vec::new())
4225            }
4226            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4227                reservation.release_and_disarm();
4228                // A module that says no in microseconds is healthy. Rejection
4229                // is a different condition with its own refusal and must not
4230                // move the breaker.
4231                breaker.record_inconclusive();
4232                // The daemon's own commit re-check refused the bind because the
4233                // scope ended or changed after admission. The module accepted;
4234                // counting it as a module rejection would blame the module.
4235                let scope_code = match body.code.as_str() {
4236                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4237                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4238                    _ => None,
4239                };
4240                if let Some(code) = scope_code {
4241                    self.observe_route_open_refusal(
4242                        ctx,
4243                        &target_module_id,
4244                        "scope_changed_before_commit",
4245                        code,
4246                    );
4247                    return Ok(vec![control_error_body_frame(&frame, body)?]);
4248                }
4249                self.counters
4250                    .increment_route_open_refused("module_rejected");
4251                info!(
4252                    target: "control",
4253                    code = "module_rejected",
4254                    module_code = ?body.code,
4255                    module_id = ?target_module_id,
4256                    connection_id = ctx.connection_id.get(),
4257                    "route.open refused"
4258                );
4259                Ok(vec![control_error_body_frame(&frame, body)?])
4260            }
4261            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4262                reservation.release_and_disarm();
4263                breaker.record_inconclusive();
4264                // Fires when the module's connection closes while a relayed
4265                // bind is pending -- typically a caller racing a module restart
4266                // whose bind was relayed BEFORE the drain mark went up. Logged
4267                // because the caller sees only its own error and the fleet has
4268                // already spent one diagnosis round unable to tell this arm
4269                // from a relay timeout without daemon-side evidence.
4270                tracing::warn!(
4271                    module_id = %target_module_id,
4272                    "route.bind relay abandoned: {message}"
4273                );
4274                Ok(vec![self.route_open_refusal_frame(
4275                    ctx,
4276                    &frame,
4277                    &target_module_id,
4278                    "relay_abandoned",
4279                    "target_unavailable",
4280                    message,
4281                )?])
4282            }
4283            Ok(Err(_)) => {
4284                reservation.release_and_disarm();
4285                breaker.record_inconclusive();
4286                Ok(vec![self.route_open_refusal_frame(
4287                    ctx,
4288                    &frame,
4289                    &target_module_id,
4290                    "relay_waiter_canceled",
4291                    "target_unavailable",
4292                    "route.bind relay waiter was canceled before the module responded",
4293                )?])
4294            }
4295            Err(_) => {
4296                reservation.release_and_disarm();
4297                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
4298                // answer at all is the one condition a fast refusal can
4299                // usefully stand in for; every other arm already answered.
4300                if let Some(opened) = breaker.record_timeout(
4301                    self.route_bind_breaker_threshold,
4302                    self.route_bind_breaker_cooldown,
4303                ) {
4304                    warn!(
4305                        module_id = %target_module_id,
4306                        consecutive_timeouts = opened.consecutive_timeouts,
4307                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4308                        reopened_after_probe = opened.reopened_after_probe,
4309                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4310                    );
4311                }
4312                // The generous budget just burned to no answer: the module is
4313                // registered and its connection is up, but its bind handler sat
4314                // on the ack for the full budget (warm-on-bind, cold configure,
4315                // or a wedged handler). Every earlier unavailability shape
4316                // fast-refuses BEFORE the relay, so this arm firing means the
4317                // slowness is module-side -- log it so the per-module timeline
4318                // is reconstructable without client audit rows.
4319                tracing::warn!(
4320                    module_id = %target_module_id,
4321                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4322                    "route.bind relay timed out: module did not ack within budget"
4323                );
4324                Ok(vec![self.route_open_refusal_frame(
4325                    ctx,
4326                    &frame,
4327                    &target_module_id,
4328                    "relay_timed_out",
4329                    "module_timeout",
4330                    format!(
4331                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
4332                        route_bind_relay_timeout
4333                    ),
4334                )?])
4335            }
4336        }
4337    }
4338
4339    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4340        let response = ClientControlResponse::SupervisorSpawnSnapshot {
4341            snapshot: self.supervisor.spawn_snapshot(),
4342        };
4343        Ok(vec![control_response_body_frame(
4344            &frame,
4345            &response,
4346            "ClientControlResponse::SupervisorSpawnSnapshot",
4347        )?])
4348    }
4349
4350    fn handle_supervisor_spawn_subscribe(
4351        &self,
4352        ctx: &RouteCtx,
4353        frame: Frame,
4354        since: Option<SpawnCursor>,
4355    ) -> Result<Vec<Frame>, RouterError> {
4356        match self.supervisor.subscribe_spawns(
4357            ctx.connection_id,
4358            frame.header.corr,
4359            response_version(&frame),
4360            since,
4361            ctx.egress.clone(),
4362        ) {
4363            Ok(()) => Ok(Vec::new()),
4364            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4365                Ok(vec![control_error_body_frame(
4366                    &frame,
4367                    ErrorBody {
4368                        code: "spawn_cursor_incarnation_mismatch".to_string(),
4369                        message: "spawn cursor belongs to a different daemon incarnation"
4370                            .to_string(),
4371                        detail: Some(serde_json::json!({
4372                            "current_daemon_incarnation": current
4373                        })),
4374                    },
4375                )?])
4376            }
4377            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4378                &frame,
4379                ErrorBody {
4380                    code: "spawn_cursor_too_old".to_string(),
4381                    message: "spawn cursor predates the retained event ring".to_string(),
4382                    detail: Some(serde_json::json!({
4383                        "oldest_retained_cursor": oldest
4384                    })),
4385                },
4386            )?]),
4387            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4388                0,
4389                frame.header.corr,
4390                format!("failed to open supervisor spawn subscription: {error}"),
4391            )),
4392        }
4393    }
4394
4395    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4396        let generation = self
4397            .registry
4398            .generation()
4399            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4400        let mut modules = Vec::new();
4401        for module in self.supervisor.list() {
4402            let status = module.status_for_control("list").map_err(|err| {
4403                RouterError::backend(
4404                    0,
4405                    frame.header.corr,
4406                    format!("failed to read supervisor status: {err}"),
4407                )
4408            })?;
4409            let (configured, _) = module.configuration().map_err(|err| {
4410                RouterError::backend(
4411                    0,
4412                    frame.header.corr,
4413                    format!("failed to read module configuration: {err}"),
4414                )
4415            })?;
4416            // Status and configuration snapshots release their locks before the image probe awaits.
4417            let image = module.running_image_agreement().await;
4418            // Read per request so the figure is current when the operator asks;
4419            // the daemon samples nothing in between.
4420            let resources = Some(module.child_resource_usage());
4421            let pending_reload = Some(reload_verdict(
4422                &configured.program,
4423                status.spawned_from.as_deref(),
4424                image,
4425            ));
4426            // Keep the retired policy field on the wire for one release so
4427            // existing status consumers still receive the platform policy.
4428            modules.push(
4429                SupervisorEntry::new(
4430                    status.module_id,
4431                    status.state.to_string(),
4432                    status.enabled,
4433                    status.live,
4434                    status.health.status,
4435                )
4436                .with_launch_nonce_env(Some(!cfg!(unix)))
4437                .with_launch_nonce_source(status.launch_nonce_source)
4438                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4439                .with_protocol(status.protocol)
4440                .with_pending_reload(pending_reload)
4441                .with_last_probe_ms(status.health.last_probe_ms)
4442                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4443                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4444                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4445                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4446                .with_restart_count(Some(status.restart_count))
4447                .with_max_restarts(Some(status.max_restarts))
4448                .with_lifetime_restarts(Some(status.lifetime_restarts))
4449                .with_spawn_generation(Some(status.spawn_generation))
4450                .with_restart_window_secs(Some(status.restart_window.as_secs()))
4451                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4452                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4453                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4454                .with_resources(resources),
4455            );
4456        }
4457        let response = ClientControlResponse::SupervisorList {
4458            generation,
4459            modules,
4460        };
4461        Ok(vec![control_response_body_frame(
4462            &frame,
4463            &response,
4464            "ClientControlResponse::SupervisorList",
4465        )?])
4466    }
4467
4468    fn handle_supervisor_stderr_tail(
4469        &self,
4470        frame: Frame,
4471        module_id: String,
4472        max_lines: Option<u32>,
4473        max_bytes: Option<u32>,
4474    ) -> Result<Vec<Frame>, RouterError> {
4475        let Some(module) = self.supervisor.get(&module_id) else {
4476            return Ok(vec![control_error_frame(
4477                &frame,
4478                "unknown_module",
4479                format!("module_id '{module_id}' is not supervised"),
4480            )?]);
4481        };
4482
4483        let snapshot = module.stderr_tail(
4484            max_lines.map(|value| value as usize),
4485            max_bytes.map(|value| value as usize),
4486        );
4487
4488        let response = ClientControlResponse::SupervisorStderrTail {
4489            module_id,
4490            tail: StderrTail {
4491                capture: match snapshot.capture {
4492                    CaptureState::Captured => StderrCaptureState::Captured,
4493                    CaptureState::Incomplete { reason } => {
4494                        StderrCaptureState::Incomplete { reason }
4495                    }
4496                    CaptureState::NotCaptured { reason } => {
4497                        StderrCaptureState::NotCaptured { reason }
4498                    }
4499                },
4500                entries: snapshot
4501                    .entries
4502                    .into_iter()
4503                    .map(|entry| match entry {
4504                        TailEntry::Line {
4505                            text,
4506                            truncated,
4507                            at_ms,
4508                        } => StderrTailEntry::Line {
4509                            text,
4510                            truncated,
4511                            at_ms,
4512                        },
4513                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4514                    })
4515                    .collect(),
4516                dropped_lines: snapshot.dropped_lines,
4517            },
4518        };
4519        Ok(vec![control_response_body_frame(
4520            &frame,
4521            &response,
4522            "ClientControlResponse::SupervisorStderrTail",
4523        )?])
4524    }
4525
4526    async fn handle_supervisor_terminals(
4527        &self,
4528        frame: Frame,
4529        module_id: String,
4530    ) -> Result<Vec<Frame>, RouterError> {
4531        let Some(module) = self.supervisor.get(&module_id) else {
4532            return Ok(vec![control_error_frame(
4533                &frame,
4534                "unknown_module",
4535                format!("module_id '{module_id}' is not supervised"),
4536            )?]);
4537        };
4538
4539        // The journal read runs on a blocking thread: it can be megabytes of
4540        // file I/O and must not occupy a runtime worker.
4541        let terminals = module
4542            .read_durable_terminal_history()
4543            .await
4544            .map_err(|error| {
4545                RouterError::backend(
4546                    0,
4547                    frame.header.corr,
4548                    format!("failed to read terminal history: {error}"),
4549                )
4550            })?;
4551        let response = ClientControlResponse::SupervisorTerminals {
4552            module_id,
4553            terminals,
4554        };
4555        Ok(vec![control_response_body_frame(
4556            &frame,
4557            &response,
4558            "ClientControlResponse::SupervisorTerminals",
4559        )?])
4560    }
4561
4562    fn handle_supervisor_routes(
4563        &self,
4564        frame: Frame,
4565        module_id: Option<String>,
4566    ) -> Result<Vec<Frame>, RouterError> {
4567        let modules = self
4568            .forwarding
4569            .route_census(module_id.as_deref())
4570            .map_err(RouterError::Forwarding)?
4571            .into_iter()
4572            .map(|(module_id, routes)| SupervisorRouteModule {
4573                module_id,
4574                routes: routes
4575                    .into_iter()
4576                    .map(|route| SupervisorRoute {
4577                        consumer: match route.principal {
4578                            Principal::Reserved { module_id } => {
4579                                SupervisorRouteConsumer::Reserved { module_id }
4580                            }
4581                            Principal::Direct | Principal::Unverified => {
4582                                SupervisorRouteConsumer::Direct {
4583                                    connection_id: route.goodbye_target.connection_id.get(),
4584                                }
4585                            }
4586                        },
4587                        age_ms: Instant::now()
4588                            .saturating_duration_since(route.bound_at)
4589                            .as_millis()
4590                            .try_into()
4591                            .unwrap_or(u64::MAX),
4592                        draining: route.draining,
4593                        drain_reason: route.drain_reason,
4594                    })
4595                    .collect(),
4596            })
4597            .collect();
4598        let response = ClientControlResponse::SupervisorRoutes { modules };
4599        Ok(vec![control_response_body_frame(
4600            &frame,
4601            &response,
4602            "ClientControlResponse::SupervisorRoutes",
4603        )?])
4604    }
4605
4606    async fn handle_supervisor_provenance(
4607        &self,
4608        frame: Frame,
4609        module_id: Option<String>,
4610    ) -> Result<Vec<Frame>, RouterError> {
4611        let mut selected = if let Some(module_id) = module_id {
4612            let Some(module) = self.supervisor.get(&module_id) else {
4613                return Ok(vec![control_error_frame(
4614                    &frame,
4615                    "unknown_module",
4616                    format!("module_id '{module_id}' is not supervised"),
4617                )?]);
4618            };
4619            vec![module]
4620        } else {
4621            self.supervisor.list()
4622        };
4623
4624        let mut modules = Vec::with_capacity(selected.len());
4625        for module in selected.drain(..) {
4626            let (status, observed_image) = module
4627                .status_and_running_image_agreement()
4628                .await
4629                .map_err(|err| {
4630                    RouterError::backend(
4631                        0,
4632                        frame.header.corr,
4633                        format!("failed to read supervisor status: {err}"),
4634                    )
4635                })?;
4636            let module_declared = self
4637                .registry
4638                .get_module(&status.module_id)
4639                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4640                .and_then(|registration| registration.manifest.provenance)
4641                .map(|build| ModuleDeclaredProvenance::Reported { build })
4642                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4643            #[cfg(test)]
4644            let running_image = match &self.provenance_probe_override {
4645                Some(result) => result.clone(),
4646                None => observed_image,
4647            };
4648            #[cfg(not(test))]
4649            let running_image = observed_image;
4650            modules.push(SupervisorModuleProvenance {
4651                module_id: status.module_id,
4652                module_declared,
4653                daemon_observed: SupervisorObservedProcess {
4654                    launch_nonce_source: status.launch_nonce_source,
4655                    pid: status.pid,
4656                    spawned_at_ms: status.spawned_at_ms,
4657                    spawned_from: status.spawned_from,
4658                    running_image,
4659                },
4660            });
4661        }
4662        let daemon = SupervisorDaemonProvenance {
4663            daemon_build: self.daemon_provenance.build.clone(),
4664            daemon_observed: DaemonObservedProcess {
4665                pid: self.daemon_provenance.pid,
4666                started_at_ms: self
4667                    .daemon_provenance
4668                    .start_clock
4669                    .map(|clock| clock.started_at_ms())
4670                    .or(self.daemon_provenance.started_at_ms),
4671                running_image: self
4672                    .daemon_provenance
4673                    .probe
4674                    .observe(
4675                        self.daemon_provenance.pid,
4676                        self.daemon_provenance.executable_path.as_deref(),
4677                        self.daemon_provenance.executable_identity.clone(),
4678                        self.daemon_provenance.process_start_time,
4679                    )
4680                    .await,
4681            },
4682        };
4683        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4684        Ok(vec![control_response_body_frame(
4685            &frame,
4686            &response,
4687            "ClientControlResponse::SupervisorProvenance",
4688        )?])
4689    }
4690
4691    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4692        self.refresh_capability_requirements();
4693        let generation = self
4694            .registry
4695            .generation()
4696            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4697        let modules = self
4698            .supervisor
4699            .list()
4700            .into_iter()
4701            .map(|module| {
4702                let status = module.status_for_control("health").map_err(|err| {
4703                    RouterError::backend(
4704                        0,
4705                        frame.header.corr,
4706                        format!("failed to read supervisor health: {err}"),
4707                    )
4708                })?;
4709                let module_id = status.module_id;
4710                let capability_detail = self
4711                    .capability_evaluator
4712                    .required_problem_detail(&module_id);
4713                Ok(SupervisorHealthEntry {
4714                    module_id,
4715                    status: status.health.status,
4716                    detail: append_capability_problem_detail(
4717                        status.health.detail,
4718                        capability_detail,
4719                    ),
4720                    metrics: status.health.metrics,
4721                    consecutive_failures: status.health.consecutive_failures,
4722                    late_answer_count: status.health.late_answer_count,
4723                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4724                    last_action: status.health.last_action,
4725                    last_action_ms: status.health.last_action_ms,
4726                    last_probe_ms: status.health.last_probe_ms,
4727                })
4728            })
4729            .collect::<Result<Vec<_>, RouterError>>()?;
4730        let response = ClientControlResponse::SupervisorHealth {
4731            generation,
4732            modules,
4733        };
4734        Ok(vec![control_response_body_frame(
4735            &frame,
4736            &response,
4737            "ClientControlResponse::SupervisorHealth",
4738        )?])
4739    }
4740
4741    async fn handle_supervisor_restart(
4742        &self,
4743        frame: Frame,
4744        module_id: String,
4745        drain_timeout_ms: Option<u64>,
4746    ) -> Result<Vec<Frame>, RouterError> {
4747        let operation_lock = self.supervisor.operation_lock();
4748        let _operation_guard = operation_lock.lock().await;
4749        let Some(module) = self.supervisor.get(&module_id) else {
4750            return Ok(vec![control_error_frame(
4751                &frame,
4752                "unknown_module",
4753                format!("module_id '{module_id}' is not supervised"),
4754            )?]);
4755        };
4756
4757        self.route_outages.mark_operator_action(&module_id);
4758        if let Err(err) = module.restart(drain_timeout_ms).await {
4759            self.route_outages
4760                .operator_action_ended_unrefused(&module_id);
4761            let (code, message) = match err {
4762                crate::supervise::SuperviseError::Disabled { .. } => {
4763                    ("module_disabled", err.to_string())
4764                }
4765                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4766                    ("swap_in_progress", err.to_string())
4767                }
4768                _ => (
4769                    "target_unavailable",
4770                    format!("failed to restart module_id '{module_id}': {err}"),
4771                ),
4772            };
4773            return Ok(vec![control_error_frame(&frame, code, message)?]);
4774        }
4775
4776        let response = ClientControlResponse::SupervisorAck {
4777            module_id,
4778            applied: true,
4779        };
4780        Ok(vec![control_response_body_frame(
4781            &frame,
4782            &response,
4783            "ClientControlResponse::SupervisorAck",
4784        )?])
4785    }
4786
4787    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
4788    /// when the old process has finished draining: a caller whose own lane
4789    /// rides the old process must get its reply before that drain waits on it.
4790    async fn handle_supervisor_swap(
4791        &self,
4792        frame: Frame,
4793        module_id: String,
4794        ready_timeout_ms: Option<u64>,
4795    ) -> Result<Vec<Frame>, RouterError> {
4796        // The daemon-wide operation lock is held only to resolve the handle,
4797        // not across the swap. The swap can take its whole readiness budget,
4798        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
4799        // holding it here would park an operator's stop behind the swap it is
4800        // meant to abort. A rescan or stop that reaches the module during the
4801        // swap is served by the swap itself (see `supervise_swap`).
4802        let module = {
4803            let operation_lock = self.supervisor.operation_lock();
4804            let _operation_guard = operation_lock.lock().await;
4805            self.supervisor.get(&module_id)
4806        };
4807        let Some(module) = module else {
4808            return Ok(vec![control_error_frame(
4809                &frame,
4810                "unknown_module",
4811                format!("module_id '{module_id}' is not supervised"),
4812            )?]);
4813        };
4814
4815        self.route_outages.mark_operator_action(&module_id);
4816        if let Err(err) = module
4817            .swap(ready_timeout_ms.map(Duration::from_millis))
4818            .await
4819        {
4820            self.route_outages
4821                .operator_action_ended_unrefused(&module_id);
4822            use crate::supervise::SuperviseError;
4823            let message = err.to_string();
4824            let error = match err {
4825                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4826                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4827                    code: "swap_refused".to_string(),
4828                    message,
4829                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4830                },
4831                SuperviseError::SwapFailed {
4832                    arm,
4833                    candidate_exit,
4834                    ..
4835                } => ErrorBody {
4836                    code: "swap_failed".to_string(),
4837                    message,
4838                    detail: Some(serde_json::json!({
4839                        "arm": arm.as_str(),
4840                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4841                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4842                    })),
4843                },
4844                _ => ErrorBody::new(
4845                    "target_unavailable",
4846                    format!("failed to swap module_id '{module_id}': {message}"),
4847                ),
4848            };
4849            return Ok(vec![control_error_body_frame(&frame, error)?]);
4850        }
4851        // A completed swap kept the incumbent serving until cutover, so it
4852        // usually opened no outage; a mark left behind would make the next,
4853        // unrelated outage read as requested.
4854        self.route_outages
4855            .operator_action_ended_unrefused(&module_id);
4856
4857        let response = ClientControlResponse::SupervisorAck {
4858            module_id,
4859            applied: true,
4860        };
4861        Ok(vec![control_response_body_frame(
4862            &frame,
4863            &response,
4864            "ClientControlResponse::SupervisorAck",
4865        )?])
4866    }
4867
4868    async fn handle_supervisor_reload(
4869        &self,
4870        frame: Frame,
4871        module_id: String,
4872    ) -> Result<Vec<Frame>, RouterError> {
4873        let operation_lock = self.supervisor.operation_lock();
4874        let _operation_guard = operation_lock.lock().await;
4875        let Some(module) = self.supervisor.get(&module_id) else {
4876            return Ok(vec![control_error_frame(
4877                &frame,
4878                "unknown_module",
4879                format!("module_id '{module_id}' is not supervised"),
4880            )?]);
4881        };
4882
4883        self.route_outages.mark_operator_action(&module_id);
4884        if let Err(err) = module.reload().await {
4885            self.route_outages
4886                .operator_action_ended_unrefused(&module_id);
4887            let (code, message) = match err {
4888                crate::supervise::SuperviseError::Disabled { .. } => {
4889                    ("module_disabled", err.to_string())
4890                }
4891                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4892                    ("swap_in_progress", err.to_string())
4893                }
4894                _ => (
4895                    "reload_failed",
4896                    format!("failed to reload module_id '{module_id}': {err}"),
4897                ),
4898            };
4899            return Ok(vec![control_error_frame(&frame, code, message)?]);
4900        }
4901
4902        let response = ClientControlResponse::SupervisorAck {
4903            module_id,
4904            applied: true,
4905        };
4906        Ok(vec![control_response_body_frame(
4907            &frame,
4908            &response,
4909            "ClientControlResponse::SupervisorAck",
4910        )?])
4911    }
4912
4913    async fn handle_supervisor_rescan(
4914        &self,
4915        frame: Frame,
4916        preview: bool,
4917    ) -> Result<Vec<Frame>, RouterError> {
4918        let Some(context) = self.rescan.clone() else {
4919            return Ok(vec![control_error_frame(
4920                &frame,
4921                "rescan_unavailable",
4922                "the daemon was not started with a reloadable config path".to_string(),
4923            )?]);
4924        };
4925
4926        let operation_lock = self.supervisor.operation_lock();
4927        let _operation_guard = operation_lock.lock().await;
4928        let loaded = match crate::daemon_config::load(&context.config_path) {
4929            Ok(config) => config,
4930            Err(err) => {
4931                return Ok(vec![control_error_frame(
4932                    &frame,
4933                    "invalid_daemon_config",
4934                    format!("supervisor rescan rejected daemon config: {err}"),
4935                )?])
4936            }
4937        };
4938        // `load` reports a missing file as Ok(None), which is correct at boot
4939        // (no config, nothing to supervise) and catastrophic here: rescan treats
4940        // "not in the config" as "remove it", so an absent file would read as an
4941        // empty module list and retire the entire running fleet. An editor
4942        // writing via write-new-then-rename, or a half-finished edit, is enough
4943        // to open that window. Refuse instead: a config that cannot be read
4944        // carries no instruction to remove anything.
4945        let Some(config) = loaded else {
4946            return Ok(vec![control_error_frame(
4947                &frame,
4948                "invalid_daemon_config",
4949                format!(
4950                    "daemon config not found at {}; refusing to rescan (an absent config would \
4951                     retire every supervised module)",
4952                    context.config_path.display()
4953                ),
4954            )?]);
4955        };
4956        let (
4957            configured_port,
4958            storage_config,
4959            admission_facts_carrier_module_id,
4960            admission_facts_targets,
4961            scope_authority_owners,
4962            modules,
4963            reserved_capabilities,
4964        ) = (
4965            config.port,
4966            config.storage,
4967            config.admission_facts_carrier_module_id,
4968            config.admission_facts_targets,
4969            config.scope_authority_owners,
4970            config.modules,
4971            config.reserved_capabilities,
4972        );
4973
4974        // Collect the sections rescan cannot apply, so the REPLY carries them.
4975        //
4976        // The warning below has always been correct and has always gone only to
4977        // the journal -- addressed to whoever reads logs, while the person who
4978        // just edited the config is looking at the CLI. Naming each section
4979        // individually rather than setting a flag: "something outside modules
4980        // changed" sends the operator back to diffing their own file, which is
4981        // the work this is meant to save.
4982        let mut restart_required = Vec::new();
4983        for section in RestartRequiredSection::ALL {
4984            let changed = match section {
4985                RestartRequiredSection::Port => configured_port != context.configured_port,
4986                RestartRequiredSection::Storage => storage_config != context.storage_config,
4987                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4988                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4989                }
4990                RestartRequiredSection::AdmissionFactsTargets => {
4991                    admission_facts_targets != context.admission_facts_targets
4992                }
4993                RestartRequiredSection::ScopeAuthorityOwners => {
4994                    scope_authority_owners != context.scope_authority_owners
4995                }
4996            };
4997            if changed {
4998                restart_required.push(section.label().to_string());
4999            }
5000        }
5001        if !restart_required.is_empty() {
5002            warn!(
5003                config_path = %context.config_path.display(),
5004                sections = %restart_required.join(", "),
5005                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5006            );
5007        }
5008
5009        for configured in &modules {
5010            if let Err(err) = validate_spec(&configured.module_spec()) {
5011                return Ok(vec![control_error_frame(
5012                    &frame,
5013                    "invalid_daemon_config",
5014                    format!("supervisor rescan rejected daemon config: {err}"),
5015                )?]);
5016            }
5017        }
5018
5019        let configured_capabilities = modules
5020            .iter()
5021            .map(|module| (module.module_id.clone(), module.enabled))
5022            .collect::<Vec<_>>();
5023        let preview_capability_warnings = if preview {
5024            let (_, registrations) = self.runtime_capability_snapshot()?;
5025            let current_modules = self
5026                .supervisor
5027                .list()
5028                .into_iter()
5029                .map(|module| module.module_id().to_string())
5030                .collect::<BTreeSet<_>>();
5031            let resulting_modules = configured_capabilities.clone();
5032            let removed = current_modules
5033                .into_iter()
5034                .filter(|module_id| {
5035                    !resulting_modules
5036                        .iter()
5037                        .any(|(configured_id, _)| configured_id == module_id)
5038                })
5039                .collect::<Vec<_>>();
5040            self.capability_evaluator.preview_removal_warnings(
5041                resulting_modules,
5042                &removed,
5043                &registrations,
5044            )
5045        } else {
5046            Vec::new()
5047        };
5048        let result = match self
5049            .reconcile_supervised_modules(&context.supervisor, modules, preview)
5050            .await
5051        {
5052            Ok(result) => result,
5053            Err(message) => {
5054                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5055            }
5056        };
5057        if !preview {
5058            self.capability_evaluator
5059                .configure(configured_capabilities, reserved_capabilities);
5060            self.capability_evaluator.wake_deadline_loop();
5061            self.refresh_capability_requirements();
5062        }
5063        let mut result = result;
5064        result.restart_required = restart_required;
5065        result.capability_warnings = preview_capability_warnings;
5066        let response = ClientControlResponse::SupervisorRescan { result };
5067        Ok(vec![control_response_body_frame(
5068            &frame,
5069            &response,
5070            "ClientControlResponse::SupervisorRescan",
5071        )?])
5072    }
5073
5074    async fn handle_supervisor_release_reserved(
5075        &self,
5076        frame: Frame,
5077        module_id: String,
5078    ) -> Result<Vec<Frame>, RouterError> {
5079        let Some(context) = self.rescan.clone() else {
5080            return Ok(vec![control_error_frame(
5081                &frame,
5082                "release_unavailable",
5083                "reserved-id release requires a daemon started with a reloadable config path",
5084            )?]);
5085        };
5086        let operation_lock = self.supervisor.operation_lock();
5087        let _operation_guard = operation_lock.lock().await;
5088        let loaded = match crate::daemon_config::load(&context.config_path) {
5089            Ok(Some(config)) => config,
5090            Ok(None) => {
5091                return Ok(vec![control_error_frame(
5092                    &frame,
5093                    "invalid_daemon_config",
5094                    format!(
5095                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5096                        context.config_path.display()
5097                    ),
5098                )?])
5099            }
5100            Err(err) => {
5101                return Ok(vec![control_error_frame(
5102                    &frame,
5103                    "invalid_daemon_config",
5104                    format!("unable to verify reserved-id release against daemon config: {err}"),
5105                )?])
5106            }
5107        };
5108        if loaded
5109            .modules
5110            .iter()
5111            .any(|configured| configured.module_id == module_id)
5112        {
5113            return Ok(vec![control_error_frame(
5114                &frame,
5115                "reserved_module_configured",
5116                format!(
5117                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5118                ),
5119            )?]);
5120        }
5121        if !self.supervisor.release_retained_reserved_gate(&module_id) {
5122            return Ok(vec![control_error_frame(
5123                &frame,
5124                "reserved_gate_not_retained",
5125                format!(
5126                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5127                ),
5128            )?]);
5129        }
5130
5131        let response = ClientControlResponse::SupervisorAck {
5132            module_id,
5133            applied: true,
5134        };
5135        Ok(vec![control_response_body_frame(
5136            &frame,
5137            &response,
5138            "ClientControlResponse::SupervisorAck",
5139        )?])
5140    }
5141
5142    /// Reconcile the running module set against the configured one.
5143    ///
5144    /// With `preview` set, the diff is computed and returned WITHOUT applying any
5145    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
5146    /// deliberately shares this function with the executing path rather than
5147    /// computing the same diff somewhere else -- two implementations of one
5148    /// decision agree until they do not, and the whole value of a preview is that
5149    /// it describes the operation that will actually run.
5150    async fn reconcile_supervised_modules(
5151        &self,
5152        supervisor: &Supervisor,
5153        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5154        preview: bool,
5155    ) -> Result<SupervisorRescanResult, String> {
5156        let mut current = BTreeMap::new();
5157        for module in self.supervisor.list() {
5158            let (spec, health) = module.configuration().map_err(|err| {
5159                format!(
5160                    "failed to read configuration for module_id '{}': {err}",
5161                    module.module_id()
5162                )
5163            })?;
5164            let enabled = module
5165                .status()
5166                .map_err(|err| {
5167                    format!(
5168                        "failed to read status for module_id '{}': {err}",
5169                        module.module_id()
5170                    )
5171                })?
5172                .enabled;
5173            current.insert(
5174                module.module_id().to_string(),
5175                (module, spec, health, enabled),
5176            );
5177        }
5178        let configured = configured_modules
5179            .into_iter()
5180            .map(|module| (module.module_id.clone(), module))
5181            .collect::<BTreeMap<_, _>>();
5182
5183        let added = configured
5184            .keys()
5185            .filter(|module_id| !current.contains_key(*module_id))
5186            .cloned()
5187            .collect::<Vec<_>>();
5188        let removed = current
5189            .keys()
5190            .filter(|module_id| !configured.contains_key(*module_id))
5191            .cloned()
5192            .collect::<Vec<_>>();
5193        let mut changed_pending_reload = Vec::new();
5194        let mut configuration_changes = BTreeSet::new();
5195        let mut enabled_changes = BTreeSet::new();
5196        let mut unchanged = 0_u32;
5197
5198        for (module_id, configured_module) in &configured {
5199            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5200            else {
5201                continue;
5202            };
5203            // Compare the whole launch spec so a future launch field cannot
5204            // accidentally become a live-only policy change. Health is stored
5205            // separately and applies live without replacing the process.
5206            let launch_changed = *current_spec != configured_module.module_spec();
5207            let configuration_changed =
5208                launch_changed || *current_health != configured_module.health;
5209            let enabled_changed = *current_enabled != configured_module.enabled;
5210            if configuration_changed {
5211                configuration_changes.insert(module_id.clone());
5212            }
5213            if launch_changed {
5214                changed_pending_reload.push(module_id.clone());
5215            }
5216            if enabled_changed {
5217                enabled_changes.insert(module_id.clone());
5218            }
5219            if !configuration_changed && !enabled_changed {
5220                unchanged = unchanged.saturating_add(1);
5221            }
5222        }
5223
5224        // Everything above this point is pure computation over two snapshots.
5225        // Everything below MUTATES. The preview returns here so the boundary is a
5226        // single early return rather than a condition repeated at each mutation
5227        // site, where one missed guard would apply part of a change the caller was
5228        // told would not happen.
5229        if preview {
5230            return Ok(SupervisorRescanResult {
5231                added,
5232                removed,
5233                changed_pending_reload,
5234                enabled_changes: enabled_changes.iter().cloned().collect(),
5235                unchanged,
5236                preview: true,
5237                // Filled by the caller on both paths, so the preview reports
5238                // restart-required sections identically to an executed rescan --
5239                // the preview is where an operator is most likely to be looking.
5240                restart_required: Vec::new(),
5241                capability_warnings: Vec::new(),
5242            });
5243        }
5244
5245        for module_id in &removed {
5246            let module = &current
5247                .get(module_id)
5248                .expect("removed module came from current supervisor state")
5249                .0;
5250            module.retire().await.map_err(|err| {
5251                format!("failed to retire module_id '{module_id}' during rescan: {err}")
5252            })?;
5253            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
5254            //
5255            // `handle_route_open` resolves an absent module in three steps:
5256            // registry, then supervisor status, then tombstone. Retiring first
5257            // opens a window where ALL THREE ARE ABSENT -- the registry entry
5258            // went with the teardown above, the supervisor entry went with
5259            // `retire`, and the tombstone does not exist yet -- so a route.open
5260            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
5261            // it") for a module that was deliberately removed and whose caller
5262            // should get `module_removed` (TERMINAL, carrying a removal age).
5263            //
5264            // Writing the tombstone first closes it: during the window the
5265            // supervisor entry still answers, so the caller gets
5266            // `target_unavailable` -- retryable, and TRUE, because the module
5267            // is mid-teardown. After both statements it is `module_removed`.
5268            // No instant remains where a removed module reads as one that
5269            // never existed.
5270            //
5271            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
5272            // the absence of a test beside a fix invites deletion: these are
5273            // two sync statements with no await between them, so reaching the
5274            // window needs a second worker thread to land exactly between them
5275            // and there is no hook to force it. MEASURED: the 25 daemon_config
5276            // tests pass identically with the old order and the new one, so
5277            // the existing suite cannot see this and a green run is not
5278            // evidence either way. What the suite does hold is the
5279            // post-condition -- a removed module answers `module_removed` --
5280            // which this preserves.
5281            //
5282            // Found by an Athena panel reading the shipped tree against a
5283            // design note (2026-09-19), as the one concrete instance of that
5284            // note's class that survived contact with source. Direction is
5285            // benign: retryable where terminal was intended, never the reverse.
5286            self.supervisor.record_rescan_removal(module_id);
5287            self.supervisor.retire(module_id);
5288            self.route_outages.forget(module_id);
5289        }
5290
5291        for module_id in configured.keys() {
5292            let Some((module, _, _, _)) = current.get(module_id) else {
5293                continue;
5294            };
5295            let configured_module = configured
5296                .get(module_id)
5297                .expect("configured module id came from configured map");
5298            if configuration_changes.contains(module_id) {
5299                module
5300                    .update_configuration(
5301                        configured_module.module_spec(),
5302                        configured_module.health.clone(),
5303                        configured_module.drain_timeout_ms,
5304                    )
5305                    .await
5306                    .map_err(|err| {
5307                        format!(
5308                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
5309                        )
5310                    })?;
5311            }
5312            if enabled_changes.contains(module_id) {
5313                // A rescan that starts or stops a module applies an operator's
5314                // edit to the config, so the resulting outage was asked for.
5315                self.route_outages.mark_operator_action(module_id);
5316                module
5317                    .set_enabled(configured_module.enabled)
5318                    .await
5319                    .map_err(|err| {
5320                        self.route_outages.operator_action_ended_unrefused(module_id);
5321                        format!(
5322                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5323                            configured_module.enabled
5324                        )
5325                    })?;
5326            }
5327        }
5328
5329        for module_id in &added {
5330            let configured_module = configured
5331                .get(module_id)
5332                .expect("added module id came from configured map");
5333            supervisor
5334                .supervise_configured_with_health(
5335                    configured_module.module_spec(),
5336                    configured_module.enabled,
5337                    configured_module.health.clone(),
5338                    configured_module.drain_timeout_ms,
5339                    configured_module.restart,
5340                )
5341                .map_err(|err| {
5342                    format!("failed to add module_id '{module_id}' during rescan: {err}")
5343                })?;
5344        }
5345
5346        Ok(SupervisorRescanResult {
5347            added,
5348            removed,
5349            changed_pending_reload,
5350            enabled_changes: enabled_changes.iter().cloned().collect(),
5351            unchanged,
5352            preview: false,
5353            // Filled by the caller, which is the only layer that can see the
5354            // previous config to diff against.
5355            restart_required: Vec::new(),
5356            capability_warnings: Vec::new(),
5357        })
5358    }
5359
5360    async fn handle_supervisor_set_enabled(
5361        &self,
5362        frame: Frame,
5363        module_id: String,
5364        enabled: bool,
5365    ) -> Result<Vec<Frame>, RouterError> {
5366        let operation_lock = self.supervisor.operation_lock();
5367        let _operation_guard = operation_lock.lock().await;
5368        let Some(module) = self.supervisor.get(&module_id) else {
5369            return Ok(vec![control_error_frame(
5370                &frame,
5371                "unknown_module",
5372                format!("module_id '{module_id}' is not supervised"),
5373            )?]);
5374        };
5375
5376        // Enabling counts as well as disabling: a module an operator starts
5377        // is refused until it registers, and that wait was asked for.
5378        self.route_outages.mark_operator_action(&module_id);
5379        let applied = match module.set_enabled(enabled).await {
5380            Ok(applied) => applied,
5381            Err(err) => {
5382                self.route_outages
5383                    .operator_action_ended_unrefused(&module_id);
5384                return Ok(vec![control_error_frame(
5385                    &frame,
5386                    "target_unavailable",
5387                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5388                )?]);
5389            }
5390        };
5391        if !applied {
5392            // Already in the requested state: nothing was made unavailable,
5393            // so the mark must not outlive this request.
5394            self.route_outages
5395                .operator_action_ended_unrefused(&module_id);
5396        }
5397
5398        self.capability_evaluator.wake_deadline_loop();
5399        self.refresh_capability_requirements();
5400        let response = ClientControlResponse::SupervisorAck { module_id, applied };
5401        Ok(vec![control_response_body_frame(
5402            &frame,
5403            &response,
5404            "ClientControlResponse::SupervisorAck",
5405        )?])
5406    }
5407
5408    async fn handle_supervisor_health_probe(
5409        &self,
5410        frame: Frame,
5411        module_id: String,
5412    ) -> Result<Vec<Frame>, RouterError> {
5413        self.refresh_capability_requirements();
5414        let Some(registration) = self
5415            .registry
5416            .get_module(&module_id)
5417            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5418        else {
5419            return Ok(vec![control_error_frame(
5420                &frame,
5421                "unknown_module",
5422                format!("module_id '{module_id}' is not registered"),
5423            )?]);
5424        };
5425
5426        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
5427        // named for it. Making `module_registration_grants_op` return false
5428        // unconditionally reddens five tests, and every one is named for something
5429        // else -- capability relay, probe/bind demultiplexing, supervision-only
5430        // probing. They exercise a successful advertisement check on the way to their
5431        // own subject.
5432        //
5433        // Real protection, fragile in a specific way: narrowing any of those tests to
5434        // focus on its stated subject would silently remove coverage nobody knows
5435        // they are carrying. Recorded here rather than as a sixth test, because the
5436        // useful fact is WHICH tests hold the guard up -- a new test would add
5437        // coverage without telling the next person what the existing ones quietly do.
5438        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5439        {
5440            return Ok(vec![control_error_frame(
5441                &frame,
5442                "health_not_advertised",
5443                format!("module_id '{module_id}' did not advertise health.check"),
5444            )?]);
5445        }
5446
5447        let deadline = Instant::now() + self.health_probe_timeout;
5448        let pending = match self.forwarding.begin_module_control_rpc_for(
5449            &module_id,
5450            MODULE_CONTROL_OP_HEALTH_CHECK,
5451            deadline,
5452        ) {
5453            Ok(pending) => pending,
5454            Err(err) => {
5455                return Ok(vec![control_error_frame(
5456                    &frame,
5457                    forwarding_error_code(&err),
5458                    err.to_string(),
5459                )?])
5460            }
5461        };
5462
5463        let PendingModuleControlRpc {
5464            endpoint,
5465            module_sink,
5466            negotiated_ver,
5467            corr: probe_corr,
5468            receiver,
5469        } = pending;
5470        let mut guard =
5471            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5472        let probe_body =
5473            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5474                RouterError::backend(
5475                    0,
5476                    frame.header.corr,
5477                    format!("failed to encode health.check request: {err}"),
5478                )
5479            })?;
5480        let probe_frame = Frame::build_with_version(
5481            negotiated_ver,
5482            FrameType::Request,
5483            control_flags(),
5484            0,
5485            0,
5486            probe_corr,
5487            probe_body,
5488        )
5489        .map_err(RouterError::FrameBuild)?;
5490
5491        if let Err(err) = module_sink.send(probe_frame).await {
5492            return Ok(vec![control_error_frame(
5493                &frame,
5494                "target_unavailable",
5495                err.to_string(),
5496            )?]);
5497        }
5498
5499        match timeout_at(deadline, receiver).await {
5500            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5501                guard.disarm();
5502                let Some(report) = response.health_report() else {
5503                    return Ok(vec![control_error_frame(
5504                        &frame,
5505                        "invalid_control_body",
5506                        "health.check RPC returned a non-health response",
5507                    )?]);
5508                };
5509                // Metrics go out whole here. The supervisor's cached snapshot
5510                // caps this blob (see truncate_health_metrics), and this path
5511                // exists precisely to answer without that cap -- so applying it
5512                // here would leave no way to see what the cached view drops.
5513                let HealthReport {
5514                    status,
5515                    detail,
5516                    metrics,
5517                } = report;
5518                let capability_detail = self
5519                    .capability_evaluator
5520                    .required_problem_detail(&module_id);
5521                let response = ClientControlResponse::SupervisorHealthProbe {
5522                    module_id,
5523                    status,
5524                    detail: append_capability_problem_detail(detail, capability_detail),
5525                    metrics,
5526                };
5527                Ok(vec![control_response_body_frame(
5528                    &frame,
5529                    &response,
5530                    "ClientControlResponse::SupervisorHealthProbe",
5531                )?])
5532            }
5533            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5534                guard.disarm();
5535                Ok(vec![control_error_body_frame(&frame, body)?])
5536            }
5537            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5538                guard.disarm();
5539                Ok(vec![control_error_frame(
5540                    &frame,
5541                    "target_unavailable",
5542                    message,
5543                )?])
5544            }
5545            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5546                guard.disarm();
5547                Ok(vec![control_error_frame(
5548                    &frame,
5549                    "invalid_control_body",
5550                    message,
5551                )?])
5552            }
5553            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5554                guard.disarm();
5555                Ok(vec![control_error_frame(
5556                    &frame,
5557                    "invalid_control_body",
5558                    format!("expected module-control op '{expected}', got '{actual}'"),
5559                )?])
5560            }
5561            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5562                guard.disarm();
5563                Ok(vec![control_error_frame(
5564                    &frame,
5565                    "module_timeout",
5566                    format!(
5567                        "module_id '{module_id}' answered health.check after {:?}",
5568                        self.health_probe_timeout
5569                    ),
5570                )?])
5571            }
5572            Ok(Err(_)) => Ok(vec![control_error_frame(
5573                &frame,
5574                "target_unavailable",
5575                "health.check waiter was canceled before the module responded",
5576            )?]),
5577            Err(_) => Ok(vec![control_error_frame(
5578                &frame,
5579                "module_timeout",
5580                format!(
5581                    "module_id '{module_id}' did not answer health.check within {:?}",
5582                    self.health_probe_timeout
5583                ),
5584            )?]),
5585        }
5586    }
5587
5588    fn supervisor_status(
5589        &self,
5590        module_id: &str,
5591        corr: u64,
5592    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5593        self.supervisor
5594            .get(module_id)
5595            .map(|module| {
5596                let warming = module.is_warming_for_control("status").map_err(|err| {
5597                    RouterError::backend(
5598                        0,
5599                        corr,
5600                        format!(
5601                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
5602                        ),
5603                    )
5604                })?;
5605                module.status_for_control("status").map_err(|err| {
5606                    RouterError::backend(
5607                        0,
5608                        corr,
5609                        format!(
5610                            "failed to read supervisor status for module_id '{module_id}': {err}"
5611                        ),
5612                    )
5613                }).map(|status| (status, warming))
5614            })
5615            .transpose()
5616    }
5617
5618    fn guard_module_control_op(
5619        &self,
5620        frame: &Frame,
5621        module_id: &str,
5622        op: &str,
5623    ) -> Result<Option<Frame>, RouterError> {
5624        if self.module_grants_op(module_id, op, frame.header.corr)? {
5625            return Ok(None);
5626        }
5627
5628        Ok(Some(control_error_frame(
5629            frame,
5630            "op_not_allowed",
5631            format!("module_id '{module_id}' did not grant control op '{op}'"),
5632        )?))
5633    }
5634
5635    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5636        let Some(registration) = self
5637            .registry
5638            .get_module(module_id)
5639            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5640        else {
5641            return Ok(false);
5642        };
5643        Ok(module_registration_grants_op(&registration.control_ops, op))
5644    }
5645
5646    fn handle_status_update(
5647        &self,
5648        endpoint: ModuleEndpointId,
5649        frame: Frame,
5650    ) -> Result<Vec<Frame>, RouterError> {
5651        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5652            Ok(update) => update,
5653            Err(err) => {
5654                // Forward-compat: a newer module may push a channel-0 op this subc
5655                // version doesn't know. The control contract says unknown push ops
5656                // are IGNORED, never answered with an error. Only a malformed body
5657                // for an op we DO know is a real error worth surfacing.
5658                if is_known_module_push_op(&frame.body) {
5659                    return Ok(vec![control_error_frame(
5660                        &frame,
5661                        "invalid_control_body",
5662                        format!("malformed module control push body: {err}"),
5663                    )?]);
5664                }
5665                return Ok(Vec::new());
5666            }
5667        };
5668
5669        match update {
5670            ModuleControlPush::RouteStatus {
5671                route_channel,
5672                route_epoch,
5673                status,
5674            } => {
5675                self.forwarding
5676                    .cache_status(endpoint, route_channel, route_epoch, status)
5677                    .map_err(RouterError::Forwarding)?;
5678            }
5679        }
5680        Ok(Vec::new())
5681    }
5682
5683    fn handle_route_poll(
5684        &self,
5685        ctx: &RouteCtx,
5686        frame: Frame,
5687        route_channel: u16,
5688        route_epoch: u32,
5689        kind: PollKind,
5690    ) -> Result<Vec<Frame>, RouterError> {
5691        let snapshot = self
5692            .forwarding
5693            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5694            .map_err(RouterError::Forwarding)?;
5695        let response = match (kind, snapshot) {
5696            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5697                ClientControlResponse::RoutePoll {
5698                    route_channel,
5699                    route_epoch,
5700                    status,
5701                    live: None,
5702                }
5703            }
5704            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5705                route_channel,
5706                route_epoch,
5707                status: None,
5708                live: None,
5709            },
5710            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5711                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
5712                // is what makes reporting `true` correct rather than a
5713                // confident guess. `process_live` returns None only when the
5714                // module id has no supervisor snapshot at all -- an
5715                // externally-started module the daemon did not spawn -- and
5716                // for those the supervisor has no opinion to offer, ever. It
5717                // is never None for a supervised module in an unknown state:
5718                // a supervised module always has a snapshot, and the answer
5719                // comes from `state == Running && process_alive`.
5720                //
5721                // The route is Bound, so the module completed a HELLO on a
5722                // live connection; "the process this route points at is
5723                // running" is therefore attested by the binding rather than
5724                // assumed. Reporting `false` for an unsupervised module would
5725                // be the actual lie -- it would tell a client its healthy
5726                // route is dead because the daemon does not manage the
5727                // process.
5728                //
5729                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
5730                // module whose liveness is genuinely unknown, e.g. a snapshot
5731                // that has not been populated yet -- THIS DEFAULT BECOMES
5732                // WRONG and must split: unsupervised stays true, unknown
5733                // becomes null so the client can tell the two apart. The
5734                // response field is already `Option<bool>`, so the wire can
5735                // carry that distinction today.
5736                let live = self
5737                    .process_liveness
5738                    .as_ref()
5739                    .and_then(|source| source.process_live(&module_id))
5740                    .unwrap_or(true);
5741                ClientControlResponse::RoutePoll {
5742                    route_channel,
5743                    route_epoch,
5744                    status: None,
5745                    live: Some(live),
5746                }
5747            }
5748            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5749                route_channel,
5750                route_epoch,
5751                status: None,
5752                live: Some(false),
5753            },
5754        };
5755
5756        Ok(vec![control_response_body_frame(
5757            &frame,
5758            &response,
5759            "ClientControlResponse::RoutePoll",
5760        )?])
5761    }
5762
5763    pub(crate) fn observe_module_control_completion(
5764        &self,
5765        completion: ModuleControlRpcCompletion,
5766    ) -> bool {
5767        match completion {
5768            ModuleControlRpcCompletion::Unknown => false,
5769            ModuleControlRpcCompletion::Settled => true,
5770            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5771                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5772                info!(
5773                    module_id = %module_id,
5774                    latency_ms,
5775                    "late health.check answer proves the module is alive"
5776                );
5777                match self
5778                    .supervisor
5779                    .record_late_health_answer(&module_id, latency_ms)
5780                {
5781                    Ok(true) => {}
5782                    Ok(false) => debug!(
5783                        module_id = %module_id,
5784                        latency_ms,
5785                        "late health.check answer has no active supervisor snapshot"
5786                    ),
5787                    Err(err) => warn!(
5788                        module_id = %module_id,
5789                        latency_ms,
5790                        error = %err,
5791                        "failed to record late health.check answer"
5792                    ),
5793                }
5794                true
5795            }
5796        }
5797    }
5798
5799    /// Decide whether a failure while settling a relayed `route.bind` belongs to
5800    /// the module connection whose frame is being handled, or to the client that
5801    /// relay was opened for.
5802    ///
5803    /// This runs on the MODULE connection's frame handler, where returning `Err`
5804    /// ends that connection -- and a module connection carries every client's
5805    /// routes to that module, so ending it costs the whole fleet its tools.
5806    /// `ConnectionClosing` carries the id of the connection that is closing, and
5807    /// when that id is a CLIENT's, the condition is entirely about that one
5808    /// client's route.open. A client-scoped condition has no authority over a
5809    /// shared module connection, so it is logged and the single relay is dropped:
5810    /// the client is going away, and `complete_pending_relay` already removed the
5811    /// relay before failing, so there is nothing left to settle. Anything that
5812    /// relay still reserved is released by that client's own connection teardown,
5813    /// which is already under way -- that is what "closing" means.
5814    ///
5815    /// Every other failure is a statement about THIS connection and stays fatal:
5816    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
5817    /// id in `ConnectionClosing` all mean this connection cannot keep serving
5818    /// frames correctly.
5819    fn refuse_to_end_module_connection_for_a_client(
5820        &self,
5821        module_connection_id: ConnectionId,
5822        corr: u64,
5823        err: ForwardingError,
5824    ) -> Result<(), RouterError> {
5825        if let ForwardingError::ConnectionClosing { connection_id } = err {
5826            if connection_id != module_connection_id {
5827                warn!(
5828                    module_connection_id = module_connection_id.get(),
5829                    client_connection_id = connection_id.get(),
5830                    corr,
5831                    "dropping a route.bind response for a closing client; the module connection keeps serving"
5832                );
5833                return Ok(());
5834            }
5835        }
5836        Err(RouterError::Forwarding(err))
5837    }
5838
5839    fn handle_module_relay_response(
5840        &self,
5841        connection_id: ConnectionId,
5842        frame: Frame,
5843    ) -> Result<Vec<Frame>, RouterError> {
5844        let mut secondary_error = None;
5845        let outcome = match frame.header.ty {
5846            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5847                Ok(probe) if probe.op == "route.bind" => {
5848                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5849                        Ok(ModuleControlResponse::RouteBindAck {}) => {
5850                            RouteBindRelayOutcome::Accepted
5851                        }
5852                        Ok(other) => {
5853                            let message =
5854                                format!("route.bind response carried unexpected body: {other:?}");
5855                            secondary_error = Some(control_error_frame(
5856                                &frame,
5857                                "invalid_control_body",
5858                                message.clone(),
5859                            )?);
5860                            RouteBindRelayOutcome::ModuleGone(message)
5861                        }
5862                        Err(err) => {
5863                            let message = format!("malformed route.bind response body: {err}");
5864                            secondary_error = Some(control_error_frame(
5865                                &frame,
5866                                "invalid_control_body",
5867                                message.clone(),
5868                            )?);
5869                            RouteBindRelayOutcome::ModuleGone(message)
5870                        }
5871                    }
5872                }
5873                Ok(probe) => {
5874                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5875                    {
5876                        Ok(response) => ModuleControlRpcOutcome::Response(response),
5877                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5878                            "malformed {} response body: {err}",
5879                            probe.op
5880                        )),
5881                    };
5882                    let completion = self
5883                        .forwarding
5884                        .complete_module_control_rpc(
5885                            connection_id,
5886                            frame.header.corr,
5887                            Some(&probe.op),
5888                            outcome,
5889                        )
5890                        .map_err(RouterError::Forwarding)?;
5891                    if !self.observe_module_control_completion(completion) {
5892                        debug!(
5893                            connection_id = connection_id.get(),
5894                            corr = frame.header.corr,
5895                            op = %probe.op,
5896                            "dropping late or unknown module-control RPC response"
5897                        );
5898                    }
5899                    return Ok(Vec::new());
5900                }
5901                Err(err) => {
5902                    if let Some(expected_op) = self
5903                        .forwarding
5904                        .pending_module_control_op(connection_id, frame.header.corr)
5905                        .map_err(RouterError::Forwarding)?
5906                    {
5907                        let completion = self
5908                            .forwarding
5909                            .complete_module_control_rpc(
5910                                connection_id,
5911                                frame.header.corr,
5912                                None,
5913                                ModuleControlRpcOutcome::MalformedResponse(format!(
5914                                    "malformed {expected_op} response body: {err}"
5915                                )),
5916                            )
5917                            .map_err(RouterError::Forwarding)?;
5918                        if !self.observe_module_control_completion(completion) {
5919                            debug!(
5920                                connection_id = connection_id.get(),
5921                                corr = frame.header.corr,
5922                                "dropping late malformed module-control RPC response"
5923                            );
5924                        }
5925                        return Ok(Vec::new());
5926                    }
5927                    let message = format!("malformed route.bind response body: {err}");
5928                    secondary_error = Some(control_error_frame(
5929                        &frame,
5930                        "invalid_control_body",
5931                        message.clone(),
5932                    )?);
5933                    RouteBindRelayOutcome::ModuleGone(message)
5934                }
5935            },
5936            FrameType::Error => {
5937                if self
5938                    .forwarding
5939                    .pending_module_control_op(connection_id, frame.header.corr)
5940                    .map_err(RouterError::Forwarding)?
5941                    .is_some()
5942                {
5943                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5944                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5945                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5946                            "malformed module-control ERROR body: {err}"
5947                        )),
5948                    };
5949                    let completion = self
5950                        .forwarding
5951                        .complete_module_control_rpc(
5952                            connection_id,
5953                            frame.header.corr,
5954                            None,
5955                            outcome,
5956                        )
5957                        .map_err(RouterError::Forwarding)?;
5958                    if !self.observe_module_control_completion(completion) {
5959                        debug!(
5960                            connection_id = connection_id.get(),
5961                            corr = frame.header.corr,
5962                            "dropping late or unknown module-control RPC error"
5963                        );
5964                    }
5965                    return Ok(Vec::new());
5966                }
5967                match serde_json::from_slice::<ErrorBody>(&frame.body) {
5968                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
5969                    Err(err) => {
5970                        let message = format!("malformed route.bind ERROR body: {err}");
5971                        secondary_error = Some(control_error_frame(
5972                            &frame,
5973                            "invalid_control_body",
5974                            message.clone(),
5975                        )?);
5976                        RouteBindRelayOutcome::ModuleGone(message)
5977                    }
5978                }
5979            }
5980            ty => {
5981                return Ok(vec![control_error_frame(
5982                    &frame,
5983                    "unsupported_control_frame",
5984                    format!("unsupported module channel-0 frame {ty:?}"),
5985                )?])
5986            }
5987        };
5988
5989        let settled =
5990            self.forwarding
5991                .complete_pending_relay(connection_id, frame.header.corr, outcome);
5992        let completion = match settled {
5993            Ok(completion) => completion,
5994            Err(err) => {
5995                self.refuse_to_end_module_connection_for_a_client(
5996                    connection_id,
5997                    frame.header.corr,
5998                    err,
5999                )?;
6000                return Ok(secondary_error.into_iter().collect());
6001            }
6002        };
6003        if let Some(target) = completion.abandoned.as_ref() {
6004            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6005        }
6006        if !completion.settled {
6007            debug!(
6008                connection_id = connection_id.get(),
6009                corr = frame.header.corr,
6010                frame_type = ?frame.header.ty,
6011                "dropping late or unknown route.bind relay response"
6012            );
6013        }
6014        Ok(secondary_error.into_iter().collect())
6015    }
6016
6017    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6018        debug!(connection_id = connection_id.get(), "handling GOODBYE");
6019        // GOODBYE ends the connection's logical session even when its socket
6020        // stays open. Use disconnect teardown so verdicts, client notices and
6021        // scope authority are released at the same lifecycle boundary.
6022        self.cleanup_connection_with_end_reason(
6023            connection_id,
6024            RegistrationEndReason::ExplicitGoodbye,
6025        )
6026        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6027        Ok(Vec::new())
6028    }
6029}
6030
6031impl Default for ControlHandler {
6032    fn default() -> Self {
6033        Self::new(Arc::new(Registry::default()))
6034    }
6035}
6036
6037impl crate::supervise::SwapPromotionObserver for ControlHandler {
6038    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6039        self.apply_registration_capabilities(registration);
6040    }
6041}
6042
6043fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6044    CapabilityRequirementStatus {
6045        consumer: status.consumer,
6046        capability: status.capability,
6047        need: match status.need {
6048            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6049            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6050        },
6051        verdict: status.verdict.as_str().to_string(),
6052        episode_seq: status.episode_seq,
6053        config_satisfiable: status.config_satisfiable,
6054        runtime_available: status.runtime_available,
6055        detail: status.detail,
6056    }
6057}
6058
6059fn append_capability_problem_detail(
6060    detail: Option<String>,
6061    capability_detail: Option<String>,
6062) -> Option<String> {
6063    match (detail, capability_detail) {
6064        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6065        (Some(detail), None) => Some(detail),
6066        (None, Some(capability_detail)) => Some(capability_detail),
6067        (None, None) => None,
6068    }
6069}
6070
6071fn subc_ops() -> Vec<String> {
6072    SUBC_CONTROL_OPS
6073        .iter()
6074        .map(|op| (*op).to_string())
6075        .collect()
6076}
6077
6078fn module_subc_ops() -> Vec<String> {
6079    SUBC_CONTROL_OPS
6080        .iter()
6081        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6082        .map(|op| (*op).to_string())
6083        .collect()
6084}
6085
6086#[cfg(test)]
6087fn module_baseline_control_ops() -> Vec<String> {
6088    MODULE_BASELINE_CONTROL_OPS
6089        .iter()
6090        .map(|op| (*op).to_string())
6091        .collect()
6092}
6093
6094fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6095    let mut seen = HashSet::new();
6096    let mut effective = Vec::new();
6097    for op in MODULE_BASELINE_CONTROL_OPS {
6098        if seen.insert((*op).to_string()) {
6099            effective.push((*op).to_string());
6100        }
6101    }
6102    for op in declared.unwrap_or_default() {
6103        if seen.insert(op.clone()) {
6104            effective.push(op);
6105        }
6106    }
6107    effective
6108}
6109
6110fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6111    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6112}
6113
6114fn target_module_id(target: &RouteTarget) -> &str {
6115    match target {
6116        RouteTarget::ToolProvider { module_id }
6117        | RouteTarget::ManagementSurface { module_id }
6118        | RouteTarget::InternalService { module_id, .. } => module_id,
6119    }
6120}
6121
6122fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6123    roles.iter().any(|role| match (target, role) {
6124        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6125        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6126        (
6127            RouteTarget::InternalService { service_id, .. },
6128            ProviderRole::InternalService {
6129                service_id: provided,
6130                ..
6131            },
6132        ) => service_id == provided,
6133        _ => false,
6134    })
6135}
6136
6137fn is_routable_role(role: &ProviderRole) -> bool {
6138    matches!(
6139        role,
6140        ProviderRole::ToolProvider { .. }
6141            | ProviderRole::ManagementSurface { .. }
6142            | ProviderRole::InternalService { .. }
6143    )
6144}
6145
6146#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6147enum ControlRequestBodyError {
6148    UnknownOp,
6149    InvalidBody,
6150}
6151
6152#[derive(Debug, Deserialize)]
6153struct ControlOpProbe {
6154    op: String,
6155}
6156
6157/// Channel-0 push ops this subc version understands. A push whose `op` is not in
6158/// this set is treated as a forward-compat unknown and ignored rather than errored.
6159const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6160
6161fn is_known_module_push_op(body: &[u8]) -> bool {
6162    serde_json::from_slice::<ControlOpProbe>(body)
6163        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6164        .unwrap_or(false)
6165}
6166
6167fn is_known_module_request_op(body: &[u8]) -> bool {
6168    serde_json::from_slice::<ControlOpProbe>(body)
6169        .map(|probe| is_module_to_subc_op(&probe.op))
6170        .unwrap_or(false)
6171}
6172
6173fn is_module_to_subc_op(op: &str) -> bool {
6174    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6175}
6176
6177fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6178    debug!(
6179        op = %op,
6180        connection_id = connection_id.get(),
6181        corr,
6182        "control dispatch"
6183    );
6184}
6185
6186fn log_slow_control_dispatch(
6187    dispatch_started_at: Option<StdInstant>,
6188    op: &'static str,
6189    connection_id: ConnectionId,
6190    corr: u64,
6191) {
6192    let Some(dispatch_started_at) = dispatch_started_at else {
6193        return;
6194    };
6195    let elapsed = dispatch_started_at.elapsed();
6196    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6197        warn!(
6198            op = %op,
6199            connection_id = connection_id.get(),
6200            corr,
6201            elapsed_ms = elapsed.as_millis() as u64,
6202            "slow control dispatch"
6203        );
6204    }
6205}
6206
6207fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6208    match request {
6209        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6210        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6211        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6212        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6213        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6214        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6215        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6216        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6217        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6218        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6219        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6220        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6221        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6222        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6223        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6224        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6225        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6226        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6227        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6228    }
6229}
6230
6231fn principal_label(principal: &Principal) -> String {
6232    match principal {
6233        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6234        Principal::Direct => "direct".to_string(),
6235        other => format!("{other:?}"),
6236    }
6237}
6238
6239fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6240    match request {
6241        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6242        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6243        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6244        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6245        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6246    }
6247}
6248
6249fn parse_client_control_request(
6250    body: &[u8],
6251) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6252    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6253        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6254            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6255                ControlRequestBodyError::InvalidBody
6256            }
6257            Ok(_) => ControlRequestBodyError::UnknownOp,
6258            Err(_) => ControlRequestBodyError::InvalidBody,
6259        };
6260        (err, classification)
6261    })
6262}
6263
6264fn parse_module_control_request_from_module(
6265    body: &[u8],
6266) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6267    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6268        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6269            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6270            Ok(_) => ControlRequestBodyError::UnknownOp,
6271            Err(_) => ControlRequestBodyError::InvalidBody,
6272        };
6273        (err, classification)
6274    })
6275}
6276
6277#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6278enum ProviderRoleKind {
6279    ToolProvider,
6280    PipelineStage,
6281    ManagementSurface,
6282    InternalService,
6283}
6284
6285fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6286    match role {
6287        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6288        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6289        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6290        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6291    }
6292}
6293
6294fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6295    roles.iter().map(provider_role_kind).collect()
6296}
6297
6298/// Most refused scope records named individually in the log per sync; the
6299/// `refused` count on the accepted line is always complete.
6300const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6301
6302/// Per-outcome counts of one accepted `scope.sync`, for its log line.
6303#[derive(Debug, Default, PartialEq, Eq)]
6304struct ScopeOutcomeCounts {
6305    created: usize,
6306    replaced: usize,
6307    updated: usize,
6308    unchanged: usize,
6309    refused: usize,
6310}
6311
6312impl ScopeOutcomeCounts {
6313    fn of(results: &[ScopeRecordResult]) -> Self {
6314        let mut counts = Self::default();
6315        for result in results {
6316            let slot = match result.outcome {
6317                ScopeRecordOutcome::Created => &mut counts.created,
6318                ScopeRecordOutcome::Replaced => &mut counts.replaced,
6319                ScopeRecordOutcome::Updated => &mut counts.updated,
6320                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6321                ScopeRecordOutcome::Refused => &mut counts.refused,
6322            };
6323            *slot += 1;
6324        }
6325        counts
6326    }
6327}
6328
6329#[cfg(test)]
6330mod scope_outcome_count_tests {
6331    use super::*;
6332
6333    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6334        ScopeRecordResult {
6335            scope_ref: "r".to_string(),
6336            scope_epoch: 1,
6337            outcome,
6338            code: None,
6339            message: None,
6340            version: None,
6341            parent_state: None,
6342        }
6343    }
6344
6345    /// Each outcome lands in its own count, so a refused record can never be
6346    /// hidden inside the total the log already printed.
6347    #[test]
6348    fn every_outcome_is_counted_in_its_own_field() {
6349        let results = [
6350            result(ScopeRecordOutcome::Created),
6351            result(ScopeRecordOutcome::Created),
6352            result(ScopeRecordOutcome::Replaced),
6353            result(ScopeRecordOutcome::Updated),
6354            result(ScopeRecordOutcome::Unchanged),
6355            result(ScopeRecordOutcome::Refused),
6356            result(ScopeRecordOutcome::Refused),
6357            result(ScopeRecordOutcome::Refused),
6358        ];
6359        assert_eq!(
6360            ScopeOutcomeCounts::of(&results),
6361            ScopeOutcomeCounts {
6362                created: 2,
6363                replaced: 1,
6364                updated: 1,
6365                unchanged: 1,
6366                refused: 3,
6367            }
6368        );
6369    }
6370}
6371
6372/// Return whether a catalog change can create a newly violating live route.
6373/// Removing an attested claim is intentionally excluded: it makes fewer routes
6374/// forbidden and therefore must leave the existing route census untouched.
6375fn capability_census_trigger(
6376    old: Option<&CapabilityDeclarations>,
6377    new: Option<&CapabilityDeclarations>,
6378) -> bool {
6379    let old_provides = old
6380        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6381        .unwrap_or_default();
6382    let old_denies = old
6383        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6384        .unwrap_or_default();
6385    let new = new.cloned().unwrap_or(CapabilityDeclarations {
6386        provides: Vec::new(),
6387        requires: Vec::new(),
6388        must_never_reach: Vec::new(),
6389    });
6390
6391    new.provides
6392        .iter()
6393        .any(|capability| !old_provides.contains(capability))
6394        || new
6395            .must_never_reach
6396            .iter()
6397            .any(|capability| !old_denies.contains(capability))
6398}
6399
6400/// Find the first capability an attested opener denies that an attested target
6401/// claims. Both manifests are live registry records, never cached or client data.
6402fn denied_capability<'a>(
6403    opening_manifest: &'a ModuleManifest,
6404    target_manifest: &ModuleManifest,
6405) -> Option<&'a str> {
6406    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6407    let target_capabilities = target_manifest.capabilities.as_ref()?;
6408    opening_capabilities
6409        .must_never_reach
6410        .iter()
6411        .find(|denied| {
6412            target_capabilities
6413                .provides
6414                .iter()
6415                .any(|provided| provided == *denied)
6416        })
6417        .map(String::as_str)
6418}
6419
6420fn catalog_update_frozen_field_message(
6421    registered: &ModuleManifest,
6422    provides: &[ProviderRole],
6423) -> Option<String> {
6424    let old_has_provides = !registered.provides.is_empty();
6425    let new_has_provides = !provides.is_empty();
6426    if old_has_provides != new_has_provides {
6427        return Some(format!(
6428            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6429            registered.module_id
6430        ));
6431    }
6432
6433    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
6434        return Some(format!(
6435            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6436            registered.module_id
6437        ));
6438    }
6439
6440    let registered_concurrency = manifest_concurrency(registered);
6441    let mut candidate = registered.clone();
6442    candidate.provides = provides.to_vec();
6443    let candidate_concurrency = manifest_concurrency(&candidate);
6444    if candidate_concurrency != registered_concurrency {
6445        return Some(format!(
6446            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6447            registered.module_id, registered_concurrency, candidate_concurrency
6448        ));
6449    }
6450
6451    // control_ops live beside the manifest in the HELLO body, not inside
6452    // ModuleManifest, so a provides-only catalog.update cannot change them.
6453    None
6454}
6455
6456fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6457    manifest.provides.iter().any(is_routable_role)
6458}
6459
6460/// Returns the routable-provider concurrency subc should enforce for this manifest.
6461///
6462/// ToolProvider and ManagementSurface store their delivery concurrency directly.
6463/// InternalService has no role-specific concurrency field, so it retains the
6464/// existing ModuleManaged default for backward compatibility.
6465fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6466    manifest
6467        .provides
6468        .iter()
6469        .find_map(|provider| match provider {
6470            ProviderRole::ToolProvider { concurrency, .. }
6471            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6472            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6473        })
6474        .unwrap_or(Concurrency::ModuleManaged)
6475}
6476
6477/// True when the manifest carries a ManagementSurface role whose concurrency
6478/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
6479/// bytes because the typed manifest deliberately erases that distinction: the
6480/// default exists for wire compatibility, and this probe exists so the default
6481/// stays observable. Any parse irregularity returns false -- the caller only
6482/// logs, and a malformed body already failed registration upstream.
6483fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6484    let has_management_surface = manifest
6485        .provides
6486        .iter()
6487        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6488    if !has_management_surface {
6489        return false;
6490    }
6491    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6492        return false;
6493    };
6494    let Some(provides) = raw
6495        .get("manifest")
6496        .and_then(|manifest| manifest.get("provides"))
6497        .and_then(serde_json::Value::as_array)
6498    else {
6499        return false;
6500    };
6501    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
6502    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
6503    // against the management_surface_manifest_without_concurrency golden, not
6504    // recalled (the externally-tagged guess was this function's first bug).
6505    provides.iter().any(|role| {
6506        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6507            && role.get("concurrency").is_none()
6508    })
6509}
6510
6511fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6512    if peer_version != PROTOCOL_VERSION {
6513        return Err(format!(
6514            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6515        ));
6516    }
6517    Ok(PROTOCOL_VERSION)
6518}
6519
6520fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6521    Frame::build_with_version(
6522        response_version(frame),
6523        FrameType::Pong,
6524        frame.header.flags,
6525        0,
6526        0,
6527        frame.header.corr,
6528        Vec::new(),
6529    )
6530    .map_err(RouterError::FrameBuild)
6531}
6532
6533pub(crate) fn control_error_frame(
6534    frame: &Frame,
6535    code: &'static str,
6536    message: impl Into<String>,
6537) -> Result<Frame, RouterError> {
6538    control_error_body_frame(
6539        frame,
6540        ErrorBody {
6541            code: code.to_string(),
6542            message: message.into(),
6543            detail: None,
6544        },
6545    )
6546}
6547
6548fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6549    let body = serde_json::to_vec(&error).map_err(|err| {
6550        RouterError::backend(
6551            0,
6552            frame.header.corr,
6553            format!("failed to encode control ERROR: {err}"),
6554        )
6555    })?;
6556
6557    Frame::build_with_version(
6558        response_version(frame),
6559        FrameType::Error,
6560        control_flags(),
6561        0,
6562        0,
6563        frame.header.corr,
6564        body,
6565    )
6566    .map_err(RouterError::FrameBuild)
6567}
6568
6569fn control_response_body_frame<T: Serialize>(
6570    frame: &Frame,
6571    reply: &T,
6572    label: &'static str,
6573) -> Result<Frame, RouterError> {
6574    let body = serde_json::to_vec(reply).map_err(|err| {
6575        RouterError::backend(
6576            0,
6577            frame.header.corr,
6578            format!("failed to encode {label}: {err}"),
6579        )
6580    })?;
6581
6582    Frame::build_with_version(
6583        response_version(frame),
6584        FrameType::Response,
6585        control_flags(),
6586        0,
6587        0,
6588        frame.header.corr,
6589        body,
6590    )
6591    .map_err(RouterError::FrameBuild)
6592}
6593
6594/// Map a forwarding failure to the wire code a client sees.
6595///
6596/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
6597/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
6598/// chosen here decides whether a caller retries or gives up.
6599///
6600/// That makes attribution the load-bearing property, not merely having a code. A
6601/// permanent fault published as a retryable one produces a fleet-wide retry storm
6602/// against something that can never recover; a transient fault published as
6603/// permanent gives up on work that would have succeeded. Both look correct in a
6604/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
6605/// the mapping per variant rather than merely asserting that some code exists.
6606///
6607/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
6608/// two codes on the same side of the boundary passes it. Measured rather than
6609/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
6610/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
6611/// a test named for something else that happens to assert the string.
6612///
6613/// That accidental coverage is deliberately left alone rather than promoted to a
6614/// named test, because it guards a property this function does not promise.
6615/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
6616/// specific code within a class, so identity is free to change and only the
6617/// partition is a contract. Splitting it out would assert a guarantee nothing
6618/// depends on — and a suite that promises more than the code does is the harder
6619/// thing to correct later, because the next reader cannot tell which assertions
6620/// are load-bearing.
6621///
6622/// Pin identity here the moment a consumer branches on a specific code.
6623fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6624    match err {
6625        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6626        ForwardingError::NoModuleConnection => "target_unavailable",
6627        ForwardingError::ModuleReloading { .. } => "module_reloading",
6628        ForwardingError::ClientRouteChannelExhausted { .. }
6629        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6630        ForwardingError::StaleModuleEndpoint
6631        | ForwardingError::UnknownReservation { .. }
6632        | ForwardingError::ConnectionClosing { .. }
6633        | ForwardingError::ClientEgressClosed { .. }
6634        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6635        // Only a swap candidate's registration can produce this, and it means
6636        // exactly what a second active HELLO for a live id means.
6637        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6638        ForwardingError::RelayCorrelationExhausted
6639        | ForwardingError::RouteOpenBuild(_)
6640        | ForwardingError::Poisoned => "forwarding_error",
6641    }
6642}
6643
6644fn response_version(frame: &Frame) -> u8 {
6645    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6646        frame.header.ver
6647    } else {
6648        PROTOCOL_VERSION
6649    }
6650}
6651
6652fn control_flags() -> Flags {
6653    Flags::new(false, Priority::Passive, false)
6654}
6655
6656/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
6657/// channel. The target is the module (a client never saw the route), so this
6658/// takes the module path: delivered late rather than dropped when the module's
6659/// queue is momentarily full, and never closing its connection.
6660fn send_goodbye_target_best_effort(
6661    counters: &DaemonCounters,
6662    target: &GoodbyeTarget,
6663    context: &'static str,
6664) {
6665    let Ok(frame) = Frame::build_with_version(
6666        target.negotiated_ver,
6667        FrameType::Goodbye,
6668        control_flags(),
6669        target.channel,
6670        target.epoch,
6671        0,
6672        Vec::new(),
6673    ) else {
6674        return;
6675    };
6676    crate::forwarding::send_module_route_goodbye(
6677        counters,
6678        &target.sink,
6679        frame,
6680        target.module_id.as_deref(),
6681        context,
6682    );
6683}
6684
6685pub(crate) fn send_route_control_pushes(
6686    forwarding: &ForwardingTable,
6687    routes: Vec<EndpointRoute>,
6688    push: ClientControlPush,
6689) {
6690    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6691    for route in routes {
6692        let target = route.goodbye_target;
6693        if let Some((existing, channels)) = targets
6694            .iter_mut()
6695            .find(|(existing, _)| existing.connection_id == target.connection_id)
6696        {
6697            debug_assert_eq!(
6698                existing.negotiated_ver, target.negotiated_ver,
6699                "one connection cannot negotiate multiple frame versions"
6700            );
6701            if !channels.contains(&target.channel) {
6702                channels.push(target.channel);
6703            }
6704            continue;
6705        }
6706        let channel = target.channel;
6707        targets.push((target, vec![channel]));
6708    }
6709    for (target, mut channels) in targets {
6710        channels.sort_unstable();
6711        let mut push = push.clone();
6712        match &mut push {
6713            ClientControlPush::RouteClosing {
6714                channels: covered, ..
6715            }
6716            | ClientControlPush::RouteClosed {
6717                channels: covered, ..
6718            } => *covered = channels,
6719        }
6720        let body = match serde_json::to_vec(&push) {
6721            Ok(body) => body,
6722            Err(err) => {
6723                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6724                continue;
6725            }
6726        };
6727        let frame = match Frame::build_with_version(
6728            target.negotiated_ver,
6729            FrameType::Push,
6730            control_flags(),
6731            0,
6732            0,
6733            0,
6734            body.clone(),
6735        ) {
6736            Ok(frame) => frame,
6737            Err(err) => {
6738                warn!(
6739                    route_channel = target.channel,
6740                    error = %err,
6741                    "failed to build route lifecycle control PUSH frame"
6742                );
6743                continue;
6744            }
6745        };
6746        if let Err(err) = target.sink.try_send(frame) {
6747            if target.close_on_delivery_failure() {
6748                warn!(
6749                    target_connection_id = target.connection_id.get(),
6750                    route_channel = target.channel,
6751                    error = %err,
6752                    "route lifecycle control PUSH was not delivered to client; closing target connection"
6753                );
6754                let _ = forwarding.escalate_client_delivery_failure(
6755                    target.connection_id,
6756                    target.channel,
6757                    target.epoch,
6758                    CloseReason::new(
6759                        "route_lifecycle_push_delivery_failed",
6760                        format!(
6761                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6762                            target.channel
6763                        ),
6764                    ),
6765                    crate::forwarding::UndeliveredFrame {
6766                        module_id: target.module_id.as_deref(),
6767                        sink: &target.sink,
6768                    },
6769                );
6770            }
6771        }
6772    }
6773}
6774
6775#[cfg(test)]
6776mod tests {
6777    #[cfg(unix)]
6778    #[tokio::test]
6779    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6780        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
6781        let path = dir.join("subc.jsonc");
6782        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6783            "program":"/bin/sleep","args":["60"],"protocol":"none",
6784            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6785        }}}).to_string()).unwrap();
6786        let mut configured = crate::daemon_config::load(&path)
6787            .unwrap()
6788            .unwrap()
6789            .modules
6790            .pop()
6791            .unwrap();
6792        let registry = std::sync::Arc::new(crate::Registry::default());
6793        let handle = crate::SupervisorHandle::new();
6794        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6795            .with_handle(handle.clone());
6796        let module = supervisor
6797            .supervise_configured_with_health(
6798                configured.module_spec(),
6799                true,
6800                configured.health.clone(),
6801                None,
6802                configured.restart,
6803            )
6804            .unwrap();
6805        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6806        let before = module.status().unwrap().pid;
6807        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6808        configured.health.cadence = std::time::Duration::from_secs(3600);
6809        let health_only = handler
6810            .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6811            .await
6812            .unwrap();
6813        assert!(
6814            health_only.changed_pending_reload.is_empty(),
6815            "health policy is already applied live"
6816        );
6817        assert_eq!(module.status().unwrap().pid, before);
6818        assert_eq!(
6819            module.configuration().unwrap().1.http,
6820            configured.health.http
6821        );
6822        configured.args = vec!["61".into()];
6823        let launch = handler
6824            .reconcile_supervised_modules(&supervisor, vec![configured], false)
6825            .await
6826            .unwrap();
6827        assert_eq!(launch.changed_pending_reload, ["stock"]);
6828        assert_eq!(
6829            module.status().unwrap().pid,
6830            before,
6831            "a launch edit is stored until reload"
6832        );
6833        module.drain().await.unwrap();
6834    }
6835    use cortexkit_test_support::ScratchDir;
6836    use std::{
6837        collections::BTreeMap,
6838        fmt,
6839        path::PathBuf,
6840        sync::{Arc, Mutex},
6841        time::Duration,
6842    };
6843
6844    use serde_json::{json, Value};
6845    use subc_protocol::{
6846        manifest::{
6847            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6848            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6849        },
6850        session::HealthStatus,
6851        FrameType,
6852    };
6853
6854    use super::*;
6855    use crate::{
6856        forwarding::{DataRoute, DataRouteState},
6857        registry::ChannelState,
6858        router::FrameSink,
6859        stderr_tail::DEFAULT_MAX_LINE_BYTES,
6860        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6861        RouteCtx, Router,
6862    };
6863    use tokio::{
6864        sync::mpsc,
6865        time::{sleep, Instant},
6866    };
6867    use tracing::{
6868        field::{Field, Visit},
6869        Event, Subscriber,
6870    };
6871    use tracing_subscriber::{layer::Context, prelude::*, Layer};
6872
6873    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
6874    ///
6875    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
6876    /// is only populated for `tests/*.rs` integration test binaries -- this file
6877    /// compiles as part of the library target, which gets neither. This test's
6878    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
6879    /// and the sibling binary lives two directories up at
6880    /// `<target-dir>/<profile>/fake-aft-stub`.
6881    ///
6882    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
6883    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
6884    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
6885    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
6886    /// `NotFound`, which reads as a broken test rather than an unbuilt
6887    /// dependency -- so state the cause and the remedy instead. Deliberately a
6888    /// panic and not a silent skip: a test that quietly passes when it could not
6889    /// run is worse than one that fails, because it reports health it never
6890    /// verified.
6891    fn fake_aft_stub_path() -> PathBuf {
6892        let mut path = std::env::current_exe().expect("current_exe available in tests");
6893        path.pop(); // .../deps/
6894        path.pop(); // .../<profile>/
6895        path.push(if cfg!(windows) {
6896            "fake-aft-stub.exe"
6897        } else {
6898            "fake-aft-stub"
6899        });
6900        assert!(
6901            path.exists(),
6902            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6903             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6904            path.display()
6905        );
6906        path
6907    }
6908
6909    /// Whether clients retry `code` in place: the predicate itself, never a copy
6910    /// of its set. A copied list breaks silently when a code is added to or
6911    /// removed from the real one, and a stale copy here would let exactly the
6912    /// failure this test exists to catch pass.
6913    fn client_retries(code: &str) -> bool {
6914        subc_protocol::error_codes::is_retryable_route_open(code)
6915    }
6916
6917    /// A code is not a label — clients branch on it, so publishing the wrong KIND
6918    /// of failure is worse than publishing none. A permanent fault dressed as
6919    /// retryable makes every client in the fleet retry forever against something
6920    /// that cannot recover; a transient fault dressed as permanent abandons work
6921    /// that would have succeeded.
6922    ///
6923    /// Asserting "a code exists" cannot catch either, because the string is free
6924    /// to say anything. This enumerates every variant and pins which side of the
6925    /// retry boundary it lands on, so a new variant must be classified here
6926    /// deliberately rather than inheriting whichever arm it was appended to.
6927    #[test]
6928    fn retryability_of_forwarding_codes_matches_the_failure() {
6929        // Transient by nature: the target is booting, reloading, or its endpoint
6930        // was swapped mid-flight. Retrying is how these resolve.
6931        let transient = [
6932            ForwardingError::NoModuleConnection,
6933            ForwardingError::ModuleReloading {
6934                module_id: "m".into(),
6935            },
6936            ForwardingError::StaleModuleEndpoint,
6937            ForwardingError::UnknownReservation {
6938                client_channel: 1,
6939                module_channel: 1,
6940            },
6941            ForwardingError::ConnectionClosing {
6942                connection_id: ConnectionId::new(1),
6943            },
6944            ForwardingError::ClientEgressClosed {
6945                connection_id: ConnectionId::new(1),
6946            },
6947            ForwardingError::ModuleEgressUnavailable {
6948                connection_id: ConnectionId::new(1),
6949            },
6950        ];
6951        for err in transient {
6952            let code = forwarding_error_code(&err);
6953            assert!(
6954                client_retries(code),
6955                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6956            );
6957        }
6958
6959        // Not fixed by retrying. Channel and correlation exhaustion need the
6960        // caller to close routes, and a poisoned lock is a daemon that cannot
6961        // recover at all — the worst thing to advertise as retryable, since every
6962        // client would storm a daemon that will never answer.
6963        let permanent = [
6964            ForwardingError::ConnectionRoleConflict {
6965                connection_id: ConnectionId::new(1),
6966            },
6967            ForwardingError::ClientRouteChannelExhausted {
6968                connection_id: ConnectionId::new(1),
6969            },
6970            ForwardingError::ModuleRouteChannelExhausted {
6971                endpoint: ModuleEndpointId {
6972                    connection_id: ConnectionId::new(1),
6973                    generation: 1,
6974                },
6975            },
6976            ForwardingError::RelayCorrelationExhausted,
6977            ForwardingError::RouteOpenBuild("x".into()),
6978            ForwardingError::Poisoned,
6979        ];
6980        for err in permanent {
6981            let code = forwarding_error_code(&err);
6982            assert!(
6983                !client_retries(code),
6984                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6985            );
6986        }
6987    }
6988
6989    /// The principal is the daemon's answer to "who is calling", and modules
6990    /// branch on it: aft gates bash on it, cerebellum gates browser control,
6991    /// plexus gates connector invocation. So a stamp is an authorization input in
6992    /// another process, not a label — and both possible answers SUCCEED, which is
6993    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
6994    /// first-party capability to something that never proved it; a supervised one
6995    /// stamped `Direct` silently strips a module of capability it is entitled to.
6996    ///
6997    /// Neither shows up in a test that only checks the bind succeeded. Before this
6998    /// test the only coverage was accidental —
6999    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
7000    /// stamped principal on its way past, so narrowing that wire-shape test to its
7001    /// stated subject would have deleted the last assertion on this value. It
7002    /// still asserts the stamp, which is now redundancy rather than the only
7003    /// guard: both fail under the same mutation, and this one names the reason.
7004    /// SCOPE: this handler's supervisor has spawned nothing, so
7005    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
7006    /// is unreachable here. Both assertions below are refusals, and a mutant that
7007    /// refuses everything would satisfy them.
7008    ///
7009    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
7010    /// spawns a supervised consumer, reads its live nonce, and asserts the module
7011    /// observed `principal.kind == "reserved"` carrying that module_id — verified
7012    /// at source rather than assumed, since a citation is a claim about another
7013    /// file and ages like one. Recorded because a harness that structurally
7014    /// cannot reach an arm reports "none" for that arm identically to one that
7015    /// covers it and found nothing.
7016    #[tokio::test]
7017    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7018        let handler = ControlHandler::default();
7019        let frame =
7020            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7021
7022        // Absent consumer_identity is the ordinary case: a human at a terminal, or
7023        // any process holding the connection file. Nothing was proved, so nothing
7024        // may be granted beyond the unattested floor.
7025        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7026        assert_eq!(
7027            stamped,
7028            Principal::Direct,
7029            "a caller that proved nothing must not be stamped as a supervised module"
7030        );
7031
7032        // A claimed module_id with a nonce no supervised child was given is a
7033        // forgery attempt, not a weaker caller: it must be REFUSED rather than
7034        // quietly demoted to Direct, or an impersonation attempt looks identical
7035        // to an ordinary unattested connection.
7036        let forged = handler
7037            .route_open_principal(
7038                &frame,
7039                Some(ConsumerIdentity {
7040                    module_id: "aft".to_string(),
7041                    launch_nonce: "not-a-real-nonce".to_string(),
7042                }),
7043            )
7044            .unwrap();
7045        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7046        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7047    }
7048
7049    /// The test above hands `route_open_principal` an identity it built itself,
7050    /// which proves the stamping rule and nothing about where the identity comes
7051    /// from. The real producer is a wire body, and the two are joined by a serde
7052    /// field name that nothing else asserts.
7053    ///
7054    /// That join fails quietly in one specific way: an unrecognised key is simply
7055    /// absent after parsing, so a renamed or misspelled `consumer_identity`
7056    /// yields `None` and every supervised module silently drops to `Direct`.
7057    /// Capability-wise that is the safe direction, but it surfaces far from its
7058    /// cause — as a module mysteriously refused bash — and it would pass every
7059    /// test that builds its own input.
7060    ///
7061    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
7062    /// would break every client the moment the daemon gains a field, trading a
7063    /// quiet demotion for a hard refusal on additive change. Asserting the join
7064    /// instead means a rename breaks a test here rather than the fleet.
7065    #[test]
7066    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7067        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7068        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7069        let ClientControlRequest::RouteOpen {
7070            consumer_identity, ..
7071        } = parsed
7072        else {
7073            panic!("route.open body must parse as RouteOpen");
7074        };
7075        assert_eq!(
7076            consumer_identity,
7077            Some(ConsumerIdentity {
7078                module_id: "aft".to_string(),
7079                launch_nonce: "n".to_string(),
7080            }),
7081            "the wire field name must reach the value route_open_principal reads"
7082        );
7083    }
7084
7085    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7086        ModuleManifest::builder(module_id, "0.1.0")
7087            .protocol_ver(protocol_ver)
7088            .provides(vec![ProviderRole::ToolProvider {
7089                tools: vec![Tool {
7090                    name: "read".to_string(),
7091                    description: None,
7092                    execution_mode: ExecutionMode::Pure,
7093                    schema: json!({"type": "object"}),
7094                }],
7095                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7096                concurrency: Concurrency::ModuleManaged,
7097                emits_push: true,
7098                sub_supervises: true,
7099            }])
7100            .build()
7101    }
7102
7103    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7104        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7105    }
7106
7107    fn hello_frame_with_control_ops(
7108        module_id: &str,
7109        protocol_ver: u8,
7110        corr: u64,
7111        control_ops: Option<Vec<String>>,
7112    ) -> Frame {
7113        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7114    }
7115
7116    fn hello_frame_with_nonce(
7117        module_id: &str,
7118        protocol_ver: u8,
7119        corr: u64,
7120        launch_nonce: Option<&str>,
7121    ) -> Frame {
7122        hello_frame_full(
7123            module_id,
7124            protocol_ver,
7125            corr,
7126            None,
7127            launch_nonce.map(ToOwned::to_owned),
7128        )
7129    }
7130
7131    fn hello_frame_full(
7132        module_id: &str,
7133        protocol_ver: u8,
7134        corr: u64,
7135        control_ops: Option<Vec<String>>,
7136        launch_nonce: Option<String>,
7137    ) -> Frame {
7138        let body = serde_json::to_vec(&ModuleHelloBody {
7139            manifest: manifest(module_id, protocol_ver),
7140            protocol_ver,
7141            control_ops,
7142            launch_nonce,
7143        })
7144        .unwrap();
7145        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7146    }
7147
7148    fn non_routable_hello_frame_with_control_ops(
7149        module_id: &str,
7150        corr: u64,
7151        control_ops: Option<Vec<String>>,
7152    ) -> Frame {
7153        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7154        manifest.provides.clear();
7155        let body = serde_json::to_vec(&ModuleHelloBody {
7156            manifest,
7157            protocol_ver: PROTOCOL_VERSION,
7158            control_ops,
7159            launch_nonce: None,
7160        })
7161        .unwrap();
7162        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7163    }
7164
7165    fn capability_grammar_hello_frame(
7166        capabilities: Value,
7167        runtime_computed: Option<Value>,
7168        corr: u64,
7169    ) -> Frame {
7170        let mut body = serde_json::to_value(ModuleHelloBody {
7171            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7172            protocol_ver: PROTOCOL_VERSION,
7173            control_ops: None,
7174            launch_nonce: None,
7175        })
7176        .expect("HELLO body serializes");
7177        body["manifest"]["capabilities"] = capabilities;
7178        if let Some(runtime_computed) = runtime_computed {
7179            body["runtime_computed"] = runtime_computed;
7180        }
7181        Frame::build(
7182            FrameType::Hello,
7183            control_flags(),
7184            0,
7185            0,
7186            corr,
7187            serde_json::to_vec(&body).expect("HELLO body reserializes"),
7188        )
7189        .expect("HELLO frame builds")
7190    }
7191
7192    fn channel_request(channel: u16, corr: u64) -> Frame {
7193        Frame::build(
7194            FrameType::Request,
7195            Flags::new(true, Priority::Interactive, false),
7196            channel,
7197            0,
7198            corr,
7199            b"opaque".to_vec(),
7200        )
7201        .unwrap()
7202    }
7203
7204    fn route_ctx(
7205        connection_id: ConnectionId,
7206    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7207        let (tx, rx) = mpsc::channel(8);
7208        (
7209            RouteCtx {
7210                connection_id,
7211                egress: FrameSink::new(tx),
7212            },
7213            rx,
7214        )
7215    }
7216
7217    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7218        serde_json::from_slice(&frame.body).unwrap()
7219    }
7220
7221    /// Register a module over a connection that has a sink and return the
7222    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
7223    /// module's own sink rather than returning it as a reply, so the ack is
7224    /// taken off `rx` here and whatever the test reads next is what followed it.
7225    async fn hello_via_sink(
7226        handler: &ControlHandler,
7227        ctx: &RouteCtx,
7228        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7229        hello: Frame,
7230    ) -> Frame {
7231        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7232        assert!(
7233            replies.is_empty(),
7234            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7235        );
7236        let ack = rx
7237            .try_recv()
7238            .expect("HELLO_ACK is queued on the module sink")
7239            .frame;
7240        assert_eq!(ack.header.ty, FrameType::HelloAck);
7241        ack
7242    }
7243
7244    fn parse_error(frame: &Frame) -> Value {
7245        serde_json::from_slice(&frame.body).unwrap()
7246    }
7247
7248    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7249        serde_json::from_slice(&frame.body).unwrap()
7250    }
7251
7252    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7253        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7254            route_channel,
7255            route_epoch: 0,
7256            kind,
7257        })
7258        .unwrap();
7259        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7260    }
7261
7262    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7263        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7264            module_id: module_id.to_string(),
7265        })
7266        .unwrap();
7267        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7268    }
7269
7270    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7271        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7272    }
7273
7274    fn route_open_frame_with_consumer_capabilities(
7275        corr: u64,
7276        module_id: &str,
7277        project_root: ScratchDir,
7278        consumer_capabilities: Option<Vec<String>>,
7279    ) -> Frame {
7280        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7281            target: RouteTarget::ToolProvider {
7282                module_id: module_id.to_string(),
7283            },
7284            identity: BindIdentity::new(
7285                project_root.path().to_path_buf(),
7286                "unit".to_string(),
7287                "session".to_string(),
7288            ),
7289            consumer_identity: None,
7290            consumer_capabilities,
7291            role_versions: None,
7292            admission_facts: None,
7293            scope: None,
7294        })
7295        .unwrap();
7296        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7297    }
7298
7299    fn route_open_frame_with_role_versions(
7300        corr: u64,
7301        module_id: &str,
7302        project_root: ScratchDir,
7303        role_versions: Option<BTreeMap<String, String>>,
7304    ) -> Frame {
7305        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7306            target: RouteTarget::ToolProvider {
7307                module_id: module_id.to_string(),
7308            },
7309            identity: BindIdentity::new(
7310                project_root.path().to_path_buf(),
7311                "unit".to_string(),
7312                format!("session-{corr}"),
7313            ),
7314            consumer_identity: None,
7315            consumer_capabilities: None,
7316            role_versions,
7317            admission_facts: None,
7318            scope: None,
7319        })
7320        .unwrap();
7321        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7322    }
7323
7324    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7325        entries
7326            .iter()
7327            .map(|(role, version)| (role.to_string(), version.to_string()))
7328            .collect()
7329    }
7330
7331    fn route_open_frame_with_admission_facts(
7332        corr: u64,
7333        module_id: &str,
7334        project_root: ScratchDir,
7335        consumer_identity: Option<subc_control::ConsumerIdentity>,
7336        facts: Option<Value>,
7337    ) -> Frame {
7338        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7339            target: RouteTarget::ToolProvider {
7340                module_id: module_id.to_string(),
7341            },
7342            identity: BindIdentity::new(
7343                project_root.path().to_path_buf(),
7344                "unit".to_string(),
7345                format!("session-{corr}"),
7346            ),
7347            consumer_identity,
7348            consumer_capabilities: None,
7349            role_versions: None,
7350            admission_facts: facts,
7351            scope: None,
7352        })
7353        .unwrap();
7354        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7355    }
7356
7357    #[derive(Clone, Default)]
7358    struct EventCapture {
7359        events: Arc<Mutex<Vec<CapturedEvent>>>,
7360    }
7361
7362    #[derive(Clone, Debug)]
7363    struct CapturedEvent {
7364        target: String,
7365        level: tracing::Level,
7366        fields: BTreeMap<String, String>,
7367    }
7368
7369    impl EventCapture {
7370        fn events(&self) -> Vec<CapturedEvent> {
7371            self.events.lock().unwrap().clone()
7372        }
7373    }
7374
7375    impl<S> Layer<S> for EventCapture
7376    where
7377        S: Subscriber,
7378    {
7379        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7380            let mut visitor = EventFieldVisitor::default();
7381            event.record(&mut visitor);
7382            self.events.lock().unwrap().push(CapturedEvent {
7383                target: event.metadata().target().to_string(),
7384                level: *event.metadata().level(),
7385                fields: visitor.fields,
7386            });
7387        }
7388    }
7389
7390    #[derive(Default)]
7391    struct EventFieldVisitor {
7392        fields: BTreeMap<String, String>,
7393    }
7394
7395    impl Visit for EventFieldVisitor {
7396        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7397            self.fields
7398                .insert(field.name().to_string(), format!("{value:?}"));
7399        }
7400    }
7401
7402    fn health_response(corr: u64, status: HealthStatus) -> Frame {
7403        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7404            status,
7405            detail: Some("warming".to_string()),
7406            metrics: Some(json!({"queue_depth": 3})),
7407        })
7408        .unwrap();
7409        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7410    }
7411
7412    fn route_bind_ack(corr: u64) -> Frame {
7413        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7414        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7415    }
7416
7417    fn unique_project_root(label: &str) -> ScratchDir {
7418        ScratchDir::new(label)
7419    }
7420
7421    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7422        match parse_route_poll(frame) {
7423            ClientControlResponse::RoutePoll {
7424                status: None,
7425                live: Some(live),
7426                ..
7427            } => assert_eq!(live, expected_live),
7428            other => panic!("unexpected route.poll response: {other:?}"),
7429        }
7430    }
7431
7432    fn bind_liveness_route(
7433        registry: &Registry,
7434        forwarding: &ForwardingTable,
7435        module_id: &str,
7436    ) -> (RouteCtx, u16, u32) {
7437        let module_connection = ConnectionId::new(101);
7438        let client_connection = ConnectionId::new(202);
7439        let registration = registry
7440            .register_with_control_ops(
7441                manifest(module_id, PROTOCOL_VERSION),
7442                PROTOCOL_VERSION,
7443                module_connection,
7444                module_baseline_control_ops(),
7445            )
7446            .unwrap();
7447        let (module_tx, _module_rx) = mpsc::channel(8);
7448        let endpoint = forwarding
7449            .register_module_connection(
7450                module_connection,
7451                module_id.to_string(),
7452                PROTOCOL_VERSION,
7453                manifest_concurrency(&registration.manifest),
7454                FrameSink::new(module_tx),
7455            )
7456            .unwrap();
7457        let (client_ctx, _client_rx) = route_ctx(client_connection);
7458        let pending = forwarding
7459            .begin_route_bind_relay_for_test(
7460                client_connection,
7461                client_ctx.egress.clone(),
7462                1,
7463                module_id,
7464            )
7465            .unwrap();
7466        assert_eq!(pending.endpoint, endpoint);
7467        let route_channel = pending.client_channel;
7468        let route_epoch = pending.client_epoch;
7469        forwarding
7470            .complete_pending_relay(
7471                module_connection,
7472                pending.corr,
7473                RouteBindRelayOutcome::Accepted,
7474            )
7475            .unwrap();
7476        (client_ctx, route_channel, route_epoch)
7477    }
7478
7479    struct FakeProcessLiveness {
7480        live: Option<bool>,
7481    }
7482
7483    impl ModuleProcessLiveness for FakeProcessLiveness {
7484        fn process_live(&self, _module_id: &str) -> Option<bool> {
7485            self.live
7486        }
7487    }
7488
7489    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7490    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7491    {
7492        let registry = Arc::new(Registry::default());
7493        let supervisor_handle = SupervisorHandle::new();
7494        let supervisor = Supervisor::new_for_test(
7495            Arc::clone(&registry),
7496            RestartPolicy::new(1, Duration::from_millis(10)),
7497        )
7498        .with_handle(supervisor_handle.clone());
7499        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7500        let module = supervisor
7501            .spawn(ModuleSpec {
7502                module_id: "stderr-tail-wire".to_string(),
7503                program: fake_aft_stub_path(),
7504                args: Vec::new(),
7505                env: vec![
7506                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7507                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7508                ],
7509                reserved: false,
7510                reserved_prefixes: Vec::new(),
7511                protocol: ModuleProtocol::Subc,
7512                overlap: Default::default(),
7513            })
7514            .unwrap();
7515
7516        let deadline = Instant::now() + Duration::from_secs(5);
7517        loop {
7518            let tail = module.stderr_tail(None, None);
7519            if tail
7520                .entries
7521                .iter()
7522                .any(|entry| matches!(entry, TailEntry::ProcessStart))
7523                && tail.entries.iter().any(|entry| {
7524                    matches!(
7525                        entry,
7526                        TailEntry::Line {
7527                            truncated: true,
7528                            ..
7529                        }
7530                    )
7531                })
7532            {
7533                break;
7534            }
7535            assert!(
7536                Instant::now() < deadline,
7537                "module did not produce a truncated line and restart boundary: {tail:?}"
7538            );
7539            sleep(Duration::from_millis(10)).await;
7540        }
7541
7542        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
7543        let request = ClientControlRequest::SupervisorStderrTail {
7544            module_id: "stderr-tail-wire".to_string(),
7545            max_lines: None,
7546            max_bytes: None,
7547        };
7548        let frame = Frame::build(
7549            FrameType::Request,
7550            control_flags(),
7551            0,
7552            0,
7553            1,
7554            serde_json::to_vec(&request).unwrap(),
7555        )
7556        .unwrap();
7557        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7558        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7559        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7560            serde_json::from_slice(&responses[0].body).unwrap()
7561        else {
7562            panic!("expected supervisor.stderr_tail response");
7563        };
7564
7565        assert!(
7566            tail.entries
7567                .iter()
7568                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7569            "the control response lost the restart boundary"
7570        );
7571        let Some(StderrTailEntry::Line {
7572            text,
7573            truncated,
7574            at_ms,
7575        }) = tail.entries.iter().find(|entry| {
7576            matches!(
7577                entry,
7578                StderrTailEntry::Line {
7579                    truncated: true,
7580                    ..
7581                }
7582            )
7583        })
7584        else {
7585            panic!("the control response lost the truncated line");
7586        };
7587        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7588        assert!(*truncated);
7589        assert!(
7590            at_ms.is_some(),
7591            "the control response lost the line's capture time"
7592        );
7593    }
7594
7595    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
7596    /// read done on the worker thread would stall every other task until it
7597    /// finished; the read must run off the worker so this test's own task keeps
7598    /// running while the read is paused.
7599    #[tokio::test(flavor = "current_thread")]
7600    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7601        let dir = ScratchDir::new("terminals-off-worker");
7602        let journal_path = dir.join("terminals.jsonl");
7603        let registry = Arc::new(Registry::default());
7604        let supervisor_handle = SupervisorHandle::new();
7605        let supervisor =
7606            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
7607                .with_handle(supervisor_handle.clone())
7608                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7609        let module = supervisor
7610            .spawn(ModuleSpec {
7611                module_id: "terminal-off-worker".to_string(),
7612                program: fake_aft_stub_path(),
7613                args: Vec::new(),
7614                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7615                reserved: false,
7616                reserved_prefixes: Vec::new(),
7617                protocol: ModuleProtocol::Subc,
7618                overlap: Default::default(),
7619            })
7620            .unwrap();
7621        let deadline = Instant::now() + Duration::from_secs(5);
7622        while module.terminal_history().entries.len() != 2 {
7623            assert!(Instant::now() < deadline, "module did not record two exits");
7624            sleep(Duration::from_millis(10)).await;
7625        }
7626
7627        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7628        let handler =
7629            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
7630        let frame = Frame::build(
7631            FrameType::Request,
7632            control_flags(),
7633            0,
7634            0,
7635            1,
7636            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7637                module_id: "terminal-off-worker".to_string(),
7638            })
7639            .unwrap(),
7640        )
7641        .unwrap();
7642        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7643        let spawned_at = std::time::Instant::now();
7644        let read = tokio::spawn({
7645            let handler = Arc::clone(&handler);
7646            async move { handler.handle_control_frame(&ctx, frame).await }
7647        });
7648        // Waiting for the pause from a blocking thread keeps this task pending,
7649        // so the runtime's single worker is free to run the read task.
7650        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7651            .await
7652            .unwrap()
7653            .expect("the history read reached its pause");
7654        let elapsed = spawned_at.elapsed();
7655        assert!(
7656            elapsed < Duration::from_secs(2) && !read.is_finished(),
7657            "this task could not run while the history read was paused \
7658             (resumed after {elapsed:?}, read finished: {})",
7659            read.is_finished()
7660        );
7661
7662        drop(release);
7663        let responses = read.await.unwrap().unwrap();
7664        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7665        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7666            panic!("expected supervisor.terminals response");
7667        };
7668        assert_eq!(terminals.entries.len(), 2);
7669        assert_eq!(terminals.journal_skipped_lines, 0);
7670        assert_eq!(terminals.journal_read_errors, 0);
7671    }
7672
7673    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7674    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7675        let registry = Arc::new(Registry::default());
7676        let supervisor_handle = SupervisorHandle::new();
7677        let supervisor =
7678            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
7679                .with_handle(supervisor_handle.clone());
7680        let module = supervisor
7681            .spawn(ModuleSpec {
7682                module_id: "terminal-golden".to_string(),
7683                program: fake_aft_stub_path(),
7684                args: Vec::new(),
7685                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7686                reserved: false,
7687                reserved_prefixes: Vec::new(),
7688                protocol: ModuleProtocol::Subc,
7689                overlap: Default::default(),
7690            })
7691            .unwrap();
7692
7693        let deadline = Instant::now() + Duration::from_secs(5);
7694        while module.terminal_history().entries.len() != 2 {
7695            assert!(
7696                Instant::now() < deadline,
7697                "module did not retain two terminal exits: {:?}",
7698                module.terminal_history()
7699            );
7700            sleep(Duration::from_millis(10)).await;
7701        }
7702
7703        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
7704        let request = ClientControlRequest::SupervisorTerminals {
7705            module_id: "terminal-golden".to_string(),
7706        };
7707        let frame = Frame::build(
7708            FrameType::Request,
7709            control_flags(),
7710            0,
7711            0,
7712            1,
7713            serde_json::to_vec(&request).unwrap(),
7714        )
7715        .unwrap();
7716        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7717        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7718        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7719        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7720            panic!("expected supervisor.terminals response");
7721        };
7722        assert_eq!(terminals.entries.len(), 2);
7723        assert_eq!(terminals.dropped, 0);
7724
7725        let mut rendered = serde_json::to_value(response).unwrap();
7726        // Wall-clock fields are the observation contract, but not stable fixture
7727        // bytes; normalize only them after the real handler has shaped the response.
7728        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7729        for (index, entry) in rendered["entries"]
7730            .as_array_mut()
7731            .expect("terminal response entries array")
7732            .iter_mut()
7733            .enumerate()
7734        {
7735            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7736        }
7737
7738        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7739            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7740        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7741        if std::env::var_os("UPDATE_GOLDEN").is_some() {
7742            std::fs::write(&golden_path, &serialized).unwrap();
7743        }
7744        let expected: Value =
7745            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7746        assert_eq!(rendered, expected);
7747    }
7748
7749    #[test]
7750    fn hello_registers_manifest_and_returns_ack() {
7751        let registry = Arc::new(Registry::default());
7752        let handler = ControlHandler::new(Arc::clone(&registry));
7753        let conn = ConnectionId::new(1);
7754
7755        let responses = handler
7756            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7757            .unwrap();
7758
7759        assert_eq!(responses.len(), 1);
7760        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7761        assert_eq!(responses[0].header.channel, 0);
7762        assert_eq!(responses[0].header.corr, 7);
7763        let ack = parse_ack(&responses[0]);
7764        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7765        assert!(ack
7766            .subc_capabilities
7767            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7768        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7769        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7770        assert!(ack
7771            .subc_ops
7772            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7773        assert!(ack
7774            .subc_ops
7775            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7776
7777        let registration = registry.get_module("aft").unwrap().unwrap();
7778        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7779        assert_eq!(registration.state, ChannelState::Active);
7780        assert_eq!(registration.connection_id, conn);
7781        assert_eq!(registration.control_ops, module_baseline_control_ops());
7782    }
7783
7784    #[test]
7785    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7786        let invalid_identifiers = [
7787            ("case_change", "credentials-Provider/v1"),
7788            ("leading_zero", "credentials-provider/v01"),
7789            ("trailing_hyphen", "credentials-provider-/v1"),
7790            ("consecutive_hyphens", "credentials--provider/v1"),
7791            ("uppercase", "Credentials-provider/v1"),
7792            ("missing_v", "credentials-provider/1"),
7793            ("whitespace", "credentials provider/v1"),
7794            ("zero_version", "credentials-provider/v0"),
7795            ("out_of_range_version", "credentials-provider/v4294967296"),
7796            (
7797                "overlength_name",
7798                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7799            ),
7800        ];
7801        let mut cases = invalid_identifiers
7802            .into_iter()
7803            .map(|(name, identifier)| {
7804                (
7805                    format!("identifier_{name}"),
7806                    "capabilities.provides[0]".to_string(),
7807                    identifier.to_string(),
7808                    json!({ "provides": [identifier] }),
7809                    None,
7810                )
7811            })
7812            .collect::<Vec<_>>();
7813        cases.extend([
7814            (
7815                "unknown_need".to_string(),
7816                "capabilities.requires[0].need".to_string(),
7817                "deferred".to_string(),
7818                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7819                None,
7820            ),
7821            (
7822                "duplicate_provides".to_string(),
7823                "capabilities.provides[1]".to_string(),
7824                "credentials-provider/v1".to_string(),
7825                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7826                None,
7827            ),
7828            (
7829                "duplicate_must_never_reach".to_string(),
7830                "capabilities.must_never_reach[1]".to_string(),
7831                "credentials-provider/v1".to_string(),
7832                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7833                None,
7834            ),
7835            (
7836                "duplicate_requires_same_need".to_string(),
7837                "capabilities.requires[1]".to_string(),
7838                "credentials-provider/v1".to_string(),
7839                json!({ "requires": [
7840                    { "capability": "credentials-provider/v1", "need": "required" },
7841                    { "capability": "credentials-provider/v1", "need": "required" }
7842                ] }),
7843                None,
7844            ),
7845            (
7846                "duplicate_requires_conflicting_need".to_string(),
7847                "capabilities.requires[1]".to_string(),
7848                "credentials-provider/v1".to_string(),
7849                json!({ "requires": [
7850                    { "capability": "credentials-provider/v1", "need": "required" },
7851                    { "capability": "credentials-provider/v1", "need": "optional" }
7852                ] }),
7853                None,
7854            ),
7855            (
7856                "capabilities_root_pointer".to_string(),
7857                "runtime_computed[0]".to_string(),
7858                "/capabilities".to_string(),
7859                json!({}),
7860                Some(json!(["/capabilities"])),
7861            ),
7862            (
7863                "capabilities_descendant_pointer".to_string(),
7864                "runtime_computed[0]".to_string(),
7865                "/capabilities/provides".to_string(),
7866                json!({}),
7867                Some(json!(["/capabilities/provides"])),
7868            ),
7869            (
7870                "malformed_pointer_without_leading_slash".to_string(),
7871                "runtime_computed[0]".to_string(),
7872                "capabilities".to_string(),
7873                json!({}),
7874                Some(json!(["capabilities"])),
7875            ),
7876            (
7877                "malformed_pointer_escape".to_string(),
7878                "runtime_computed[0]".to_string(),
7879                "/roles/~2/tools".to_string(),
7880                json!({}),
7881                Some(json!(["/roles/~2/tools"])),
7882            ),
7883            (
7884                "unknown_capabilities_field".to_string(),
7885                "capabilities.future".to_string(),
7886                "<array>".to_string(),
7887                json!({ "future": [] }),
7888                None,
7889            ),
7890        ]);
7891
7892        for (index, (name, field, value, capabilities, runtime_computed)) in
7893            cases.into_iter().enumerate()
7894        {
7895            let registry = Arc::new(Registry::default());
7896            let handler = ControlHandler::new(Arc::clone(&registry));
7897            let response = handler
7898                .handle_control(
7899                    ConnectionId::new((index + 1) as u64),
7900                    capability_grammar_hello_frame(
7901                        capabilities,
7902                        runtime_computed,
7903                        index as u64 + 1,
7904                    ),
7905                )
7906                .expect("invalid HELLO returns a refusal");
7907
7908            assert_eq!(response.len(), 1, "{name} must emit one refusal");
7909            let error = parse_error(&response[0]);
7910            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7911            let message = error["message"]
7912                .as_str()
7913                .expect("error message is a string");
7914            assert!(
7915                message.contains(&field),
7916                "{name}: field missing from {message}"
7917            );
7918            assert!(
7919                message.contains(&value),
7920                "{name}: value missing from {message}"
7921            );
7922            assert_eq!(
7923                registry
7924                    .active_registration_count()
7925                    .expect("registry reads"),
7926                0,
7927                "{name}: refused HELLO must not create a catalog entry"
7928            );
7929        }
7930    }
7931
7932    #[test]
7933    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7934        let registry = Arc::new(Registry::default());
7935        let handler = ControlHandler::new(Arc::clone(&registry));
7936        let capabilities = json!({
7937            "provides": ["credentials-provider/v1"],
7938            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7939            "must_never_reach": ["federation-transport/v1"]
7940        });
7941        let response = handler
7942            .handle_control(
7943                ConnectionId::new(99),
7944                capability_grammar_hello_frame(
7945                    capabilities.clone(),
7946                    Some(json!(["/roles/0/tools"])),
7947                    99,
7948                ),
7949            )
7950            .expect("valid HELLO registers");
7951        assert_eq!(response[0].header.ty, FrameType::HelloAck);
7952
7953        let request = Frame::build(
7954            FrameType::Request,
7955            control_flags(),
7956            0,
7957            0,
7958            100,
7959            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7960                .expect("catalog request serializes"),
7961        )
7962        .expect("catalog request frame builds");
7963        let response = handler
7964            .handle_catalog_list(request, None)
7965            .expect("catalog list succeeds");
7966        let ClientControlResponse::CatalogList { modules, .. } =
7967            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7968        else {
7969            panic!("catalog request must return catalog.list");
7970        };
7971        assert_eq!(modules.len(), 1);
7972        assert_eq!(
7973            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7974            capabilities
7975        );
7976    }
7977
7978    #[test]
7979    fn catalog_list_mirrors_management_operation_description() {
7980        let registry = Arc::new(Registry::default());
7981        let handler = ControlHandler::new(Arc::clone(&registry));
7982        let description = "List managed records and return their identifiers and metadata.";
7983        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7984        manifest.provides = vec![ProviderRole::ManagementSurface {
7985            operations: vec![ManagementOperation {
7986                name: "records.list".to_string(),
7987                kind: ManagementOperationKind::Query,
7988                description: Some(description.to_string()),
7989            }],
7990            config_schema: json!({"type": "object"}),
7991            observability: vec![ObservabilitySurface {
7992                name: "records.stats".to_string(),
7993                kind: ObservabilityKind::Snapshot,
7994            }],
7995            identity_scope: vec![IdentityScope::Project],
7996            concurrency: Concurrency::ModuleManaged,
7997        }];
7998        registry
7999            .register_with_control_ops(
8000                manifest,
8001                PROTOCOL_VERSION,
8002                ConnectionId::new(99),
8003                Vec::new(),
8004            )
8005            .expect("described management manifest registers");
8006
8007        let request = Frame::build(
8008            FrameType::Request,
8009            control_flags(),
8010            0,
8011            0,
8012            100,
8013            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8014                .expect("catalog request serializes"),
8015        )
8016        .expect("catalog request frame builds");
8017        let response = handler
8018            .handle_catalog_list(request, None)
8019            .expect("catalog list succeeds");
8020        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8021        assert_eq!(
8022            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8023            "catalog.list must preserve the declared operation description verbatim"
8024        );
8025    }
8026
8027    #[test]
8028    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8029        let registry = Arc::new(Registry::default());
8030        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8031            [("vault".to_string(), true), ("squatter".to_string(), true)],
8032            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8033        );
8034        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8035        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8036            provides: vec!["credentials-provider/v1".to_string()],
8037            requires: Vec::new(),
8038            must_never_reach: Vec::new(),
8039        });
8040        let frame = Frame::build(
8041            FrameType::Hello,
8042            control_flags(),
8043            0,
8044            0,
8045            77,
8046            serde_json::to_vec(&ModuleHelloBody {
8047                manifest: squatter,
8048                protocol_ver: PROTOCOL_VERSION,
8049                control_ops: None,
8050                launch_nonce: None,
8051            })
8052            .expect("HELLO serializes"),
8053        )
8054        .expect("HELLO frame builds");
8055        let response = handler
8056            .handle_control(ConnectionId::new(77), frame)
8057            .expect("reserved claim receives a typed refusal");
8058        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8059        assert_eq!(
8060            registry
8061                .active_registration_count()
8062                .expect("registry reads"),
8063            0,
8064            "a reserved capability refusal must not leave a catalog entry"
8065        );
8066    }
8067
8068    #[test]
8069    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8070        for settlement in ["timeout", "inconclusive", "drop"] {
8071            let breakers = RouteBindBreakers::default();
8072            let RouteBindAdmission::Admitted {
8073                guard: mut old,
8074                probe: false,
8075            } = breakers.admit("prov")
8076            else {
8077                panic!("ordinary relay admitted")
8078            };
8079            let RouteBindAdmission::Admitted {
8080                guard: mut opener, ..
8081            } = breakers.admit("prov")
8082            else {
8083                panic!("second relay admitted")
8084            };
8085            assert!(
8086                !opener
8087                    .record_timeout(1, Duration::ZERO)
8088                    .unwrap()
8089                    .reopened_after_probe
8090            );
8091            let RouteBindAdmission::Admitted {
8092                guard: mut probe,
8093                probe: true,
8094            } = breakers.admit("prov")
8095            else {
8096                panic!("one cooldown probe admitted")
8097            };
8098            match settlement {
8099                "timeout" => assert!(
8100                    !old.record_timeout(1, Duration::ZERO)
8101                        .unwrap()
8102                        .reopened_after_probe
8103                ),
8104                "inconclusive" => old.record_inconclusive(),
8105                "drop" => drop(old),
8106                _ => unreachable!(),
8107            }
8108            assert!(
8109                matches!(
8110                    breakers.admit("prov"),
8111                    RouteBindAdmission::Refused {
8112                        probe_in_flight: true,
8113                        ..
8114                    }
8115                ),
8116                "{settlement} of a pre-open relay cannot release the real probe"
8117            );
8118            assert!(
8119                probe
8120                    .record_timeout(1, Duration::ZERO)
8121                    .unwrap()
8122                    .reopened_after_probe
8123            );
8124            assert!(matches!(
8125                breakers.admit("prov"),
8126                RouteBindAdmission::Admitted { probe: true, .. }
8127            ));
8128        }
8129        let breakers = RouteBindBreakers::default();
8130        let admit = || match breakers.admit("prov") {
8131            RouteBindAdmission::Admitted { guard, .. } => guard,
8132            _ => panic!("relay admitted"),
8133        };
8134        admit().record_timeout(1, Duration::ZERO);
8135        let mut old_probe = admit();
8136        breakers.reset_for_new_module_connection("prov");
8137        admit().record_timeout(1, Duration::ZERO);
8138        let _new_probe = admit();
8139        old_probe.record_inconclusive();
8140        assert!(matches!(
8141            breakers.admit("prov"),
8142            RouteBindAdmission::Refused {
8143                probe_in_flight: true,
8144                ..
8145            }
8146        ));
8147    }
8148
8149    #[tokio::test]
8150    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8151        for candidate in [false, true] {
8152            let registry = Arc::new(Registry::default());
8153            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8154                [("vault".to_string(), true), ("squatter".to_string(), true)],
8155                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8156            );
8157            let conn = ConnectionId::new(77);
8158            let (ctx, mut rx) = route_ctx(conn);
8159            let initial = capability_manifest("squatter", &[], &[]);
8160            if candidate {
8161                registry
8162                    .register_candidate_with_control_ops(
8163                        initial.clone(),
8164                        PROTOCOL_VERSION,
8165                        conn,
8166                        module_baseline_control_ops(),
8167                    )
8168                    .unwrap();
8169                handler
8170                    .forwarding
8171                    .register_candidate_module_connection(
8172                        conn,
8173                        "squatter".to_string(),
8174                        PROTOCOL_VERSION,
8175                        manifest_concurrency(&initial),
8176                        ctx.egress.clone(),
8177                    )
8178                    .unwrap();
8179            } else {
8180                hello_via_sink(
8181                    &handler,
8182                    &ctx,
8183                    &mut rx,
8184                    hello_frame_with_manifest(initial.clone(), 1),
8185                )
8186                .await;
8187            }
8188            let response = handler
8189                .handle_control_frame(
8190                    &ctx,
8191                    catalog_update_with_capabilities_frame(
8192                        2,
8193                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
8194                            .capabilities
8195                            .unwrap(),
8196                    ),
8197                )
8198                .await
8199                .unwrap();
8200            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8201            assert_eq!(
8202                registry
8203                    .get_module_by_connection(conn)
8204                    .unwrap()
8205                    .unwrap()
8206                    .manifest,
8207                initial
8208            );
8209        }
8210    }
8211
8212    #[test]
8213    fn server_describe_surfaces_required_capability_verdict_fields() {
8214        let registry = Arc::new(Registry::default());
8215        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8216            [
8217                ("consumer".to_string(), true),
8218                ("provider".to_string(), false),
8219            ],
8220            BTreeMap::new(),
8221        );
8222        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8223        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8224            provides: Vec::new(),
8225            requires: vec![subc_protocol::manifest::CapabilityRequirement {
8226                capability: "credentials-provider/v1".to_string(),
8227                need: subc_protocol::manifest::CapabilityNeed::Required,
8228            }],
8229            must_never_reach: Vec::new(),
8230        });
8231        let hello = Frame::build(
8232            FrameType::Hello,
8233            control_flags(),
8234            0,
8235            0,
8236            78,
8237            serde_json::to_vec(&ModuleHelloBody {
8238                manifest: consumer,
8239                protocol_ver: PROTOCOL_VERSION,
8240                control_ops: None,
8241                launch_nonce: None,
8242            })
8243            .expect("HELLO serializes"),
8244        )
8245        .expect("HELLO frame builds");
8246        handler
8247            .handle_control(ConnectionId::new(78), hello)
8248            .expect("consumer registers");
8249        let describe = Frame::build(
8250            FrameType::Request,
8251            control_flags(),
8252            0,
8253            0,
8254            79,
8255            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8256                .expect("request serializes"),
8257        )
8258        .expect("describe frame builds");
8259        let response = handler
8260            .handle_server_describe(describe)
8261            .expect("server.describe succeeds");
8262        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8263        let requirement = &rendered["capability_requirements"][0];
8264        assert_eq!(requirement["consumer"], "consumer");
8265        assert_eq!(requirement["verdict"], "never_provided");
8266        assert_eq!(requirement["episode_seq"], 1);
8267        assert_eq!(requirement["config_satisfiable"], false);
8268        assert_eq!(requirement["runtime_available"], false);
8269        assert!(requirement["detail"]
8270            .as_str()
8271            .expect("detail string")
8272            .contains("credentials-provider/v1"));
8273    }
8274
8275    #[test]
8276    fn catalog_list_omits_capabilities_for_legacy_manifest() {
8277        let registry = Arc::new(Registry::default());
8278        let handler = ControlHandler::new(Arc::clone(&registry));
8279        let hello = handler
8280            .handle_control(
8281                ConnectionId::new(101),
8282                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8283            )
8284            .expect("legacy HELLO registers");
8285        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8286
8287        let request = Frame::build(
8288            FrameType::Request,
8289            control_flags(),
8290            0,
8291            0,
8292            102,
8293            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8294                .expect("catalog request serializes"),
8295        )
8296        .expect("catalog request frame builds");
8297        let response = handler
8298            .handle_catalog_list(request, None)
8299            .expect("catalog list succeeds");
8300        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8301        assert!(
8302            body["modules"][0].get("capabilities").is_none(),
8303            "legacy manifest must retain an absent capabilities field on catalog.list"
8304        );
8305    }
8306
8307    #[test]
8308    fn hello_ack_omits_storage_when_no_storage_config() {
8309        let registry = Arc::new(Registry::default());
8310        let handler = ControlHandler::new(Arc::clone(&registry));
8311        let responses = handler
8312            .handle_control(
8313                ConnectionId::new(1),
8314                hello_frame("aft", PROTOCOL_VERSION, 7),
8315            )
8316            .unwrap();
8317        let ack = parse_ack(&responses[0]);
8318        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8319        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8320    }
8321
8322    #[tokio::test]
8323    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8324        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8325        let registry = Arc::new(Registry::default());
8326        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
8327        let responses = handler
8328            .handle_control(
8329                ConnectionId::new(1),
8330                hello_frame("aft", PROTOCOL_VERSION, 7),
8331            )
8332            .unwrap();
8333        let ack = parse_ack(&responses[0]);
8334        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8335
8336        let described = handler
8337            .handle_control_frame(
8338                &route_ctx(ConnectionId::new(2)).0,
8339                Frame::build(
8340                    FrameType::Request,
8341                    control_flags(),
8342                    0,
8343                    0,
8344                    9,
8345                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8346                )
8347                .unwrap(),
8348            )
8349            .await
8350            .unwrap();
8351        let ClientControlResponse::ServerDescribe { machine_id, .. } =
8352            serde_json::from_slice(&described[0].body).unwrap()
8353        else {
8354            panic!("server.describe answered with another shape");
8355        };
8356        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8357    }
8358
8359    #[test]
8360    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8361        // With a central sqlite storage policy, each registering module gets its
8362        // own resolved descriptor in HELLO_ACK, keyed by its module id.
8363        let registry = Arc::new(Registry::default());
8364        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
8365            crate::daemon_config::StorageConfig::Sqlite {
8366                data_home: std::path::PathBuf::from("/data"),
8367            },
8368        ));
8369
8370        let responses = handler
8371            .handle_control(
8372                ConnectionId::new(1),
8373                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8374            )
8375            .unwrap();
8376        let ack = parse_ack(&responses[0]);
8377        assert_eq!(
8378            ack.storage,
8379            Some(serde_json::json!({
8380                "module_id": "alfonso-routing",
8381                "storage_namespace": "default",
8382                "isolation": { "kind": "module" },
8383                "backend": {
8384                    "backend": "sqlite",
8385                    "path": "/data/cortexkit/alfonso-routing/store.db"
8386                }
8387            })),
8388            "the delivered descriptor is the module's own sqlite store path"
8389        );
8390    }
8391
8392    #[test]
8393    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8394        let registry = Arc::new(Registry::default());
8395        let handler = ControlHandler::new(Arc::clone(&registry));
8396        let conn = ConnectionId::new(1);
8397        let responses = handler
8398            .handle_control(
8399                conn,
8400                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8401            )
8402            .unwrap();
8403        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8404        let registration = registry.get_module("aft").unwrap().unwrap();
8405        assert_eq!(registration.control_ops, module_baseline_control_ops());
8406
8407        let frame =
8408            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8409        assert!(handler
8410            .guard_module_control_op(&frame, "aft", "route.bind")
8411            .unwrap()
8412            .is_none());
8413        let error = handler
8414            .guard_module_control_op(&frame, "aft", "test.synthetic")
8415            .unwrap()
8416            .expect("synthetic ungranted op should be rejected");
8417        assert_eq!(error.header.ty, FrameType::Error);
8418        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8419    }
8420
8421    #[test]
8422    fn hello_control_ops_some_adds_optional_grants() {
8423        let registry = Arc::new(Registry::default());
8424        let handler = ControlHandler::new(Arc::clone(&registry));
8425        handler
8426            .handle_control(
8427                ConnectionId::new(1),
8428                hello_frame_with_control_ops(
8429                    "aft",
8430                    PROTOCOL_VERSION,
8431                    7,
8432                    Some(vec![
8433                        "future.synthetic".to_string(),
8434                        "route.bind".to_string(),
8435                    ]),
8436                ),
8437            )
8438            .unwrap();
8439        let registration = registry.get_module("aft").unwrap().unwrap();
8440        assert_eq!(
8441            registration.control_ops,
8442            vec![
8443                "route.bind".to_string(),
8444                "route.status".to_string(),
8445                "future.synthetic".to_string(),
8446            ]
8447        );
8448        let frame =
8449            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8450        assert!(handler
8451            .guard_module_control_op(&frame, "aft", "future.synthetic")
8452            .unwrap()
8453            .is_none());
8454    }
8455
8456    #[tokio::test]
8457    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8458        let registry = Arc::new(Registry::default());
8459        let forwarding = Arc::new(ForwardingTable::default());
8460        let handler =
8461            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8462        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8463        hello_via_sink(
8464            &handler,
8465            &module_ctx,
8466            &mut module_rx,
8467            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8468        )
8469        .await;
8470
8471        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8472        let responses = handler
8473            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8474            .await
8475            .unwrap();
8476        assert_eq!(responses.len(), 1);
8477        assert_eq!(responses[0].header.ty, FrameType::Error);
8478        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8479        assert!(module_rx.try_recv().is_err());
8480    }
8481
8482    #[tokio::test]
8483    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8484        let registry = Arc::new(Registry::default());
8485        let forwarding = Arc::new(ForwardingTable::default());
8486        let handler =
8487            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8488        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8489        hello_via_sink(
8490            &handler,
8491            &module_ctx,
8492            &mut module_rx,
8493            hello_frame_with_control_ops(
8494                "aft",
8495                PROTOCOL_VERSION,
8496                7,
8497                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8498            ),
8499        )
8500        .await;
8501
8502        let project_root = unique_project_root("demux");
8503        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8504        let route_handler = handler.clone();
8505        let route_task = tokio::spawn(async move {
8506            route_handler
8507                .handle_control_frame(
8508                    &route_client_ctx,
8509                    route_open_frame(100, "aft", project_root),
8510                )
8511                .await
8512                .unwrap()
8513        });
8514        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8515            .await
8516            .unwrap()
8517            .unwrap();
8518        assert!(matches!(
8519            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8520            ModuleControlRequest::RouteBind { .. }
8521        ));
8522
8523        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8524        let health_handler = handler.clone();
8525        let health_task = tokio::spawn(async move {
8526            health_handler
8527                .handle_control_frame(
8528                    &health_client_ctx,
8529                    supervisor_health_probe_frame(101, "aft"),
8530                )
8531                .await
8532                .unwrap()
8533        });
8534        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8535            .await
8536            .unwrap()
8537            .unwrap();
8538        assert_eq!(
8539            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8540            ModuleControlRequest::HealthCheck {}
8541        );
8542
8543        handler
8544            .handle_control_frame(
8545                &module_ctx,
8546                health_response(health_frame.header.corr, HealthStatus::Degraded),
8547            )
8548            .await
8549            .unwrap();
8550        let health_response = health_task.await.unwrap();
8551        assert_eq!(health_response.len(), 1);
8552        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8553            ClientControlResponse::SupervisorHealthProbe {
8554                module_id,
8555                status,
8556                detail,
8557                metrics,
8558            } => {
8559                assert_eq!(module_id, "aft");
8560                assert_eq!(status, HealthStatus::Degraded);
8561                assert_eq!(detail.as_deref(), Some("warming"));
8562                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8563            }
8564            other => panic!("unexpected health response: {other:?}"),
8565        }
8566
8567        handler
8568            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8569            .await
8570            .unwrap();
8571        let route_response = route_task.await.unwrap();
8572        assert!(route_response.is_empty());
8573        let published = route_client_rx.recv().await.unwrap();
8574        assert!(matches!(
8575            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8576            ClientControlResponse::RouteOpen { .. }
8577        ));
8578    }
8579
8580    /// Start one `route.open` on `client_connection` and return its still-running
8581    /// handler task together with the `route.bind` the module received for it.
8582    /// The handler blocks until the module answers, so it has to run as a task
8583    /// while the test drives the module side.
8584    async fn relay_route_open(
8585        handler: &ControlHandler,
8586        client_connection: ConnectionId,
8587        client_egress: &FrameSink,
8588        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8589        corr: u64,
8590        module_id: &str,
8591        project_root_label: &str,
8592    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8593        let ctx = RouteCtx {
8594            connection_id: client_connection,
8595            egress: client_egress.clone(),
8596        };
8597        let handler = handler.clone();
8598        let project_root = unique_project_root(project_root_label);
8599        let module_id = module_id.to_string();
8600        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8601        let task = tokio::spawn(async move {
8602            let _guard = tracing::dispatcher::set_default(&dispatch);
8603            handler
8604                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8605                .await
8606                .unwrap()
8607        });
8608        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8609            .await
8610            .expect("module receives the relayed route.bind")
8611            .expect("module egress is open");
8612        (task, bind.frame)
8613    }
8614
8615    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8616        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8617            ModuleControlRequest::RouteBind {
8618                route_channel,
8619                epoch,
8620                ..
8621            } => (route_channel, epoch),
8622            other => panic!("expected a route.bind request, got {other:?}"),
8623        }
8624    }
8625
8626    fn published_route(frame: &Frame) -> (u16, u32) {
8627        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8628            ClientControlResponse::RouteOpen {
8629                route_channel,
8630                route_epoch,
8631            } => (route_channel, route_epoch),
8632            other => panic!("expected a route.open response, got {other:?}"),
8633        }
8634    }
8635
8636    /// Reproduction of a production outage. A client had `route.open`s in
8637    /// flight to a module and was already marked closing -- its egress had refused a
8638    /// module frame, so the daemon asked its connection to end -- while its sink
8639    /// was still open. When the module acked those binds, the daemon refused to
8640    /// commit a route for a closing client, and that refusal was returned from
8641    /// the MODULE connection's frame handler, where a router error that has no
8642    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
8643    /// the supervisor correctly did not respawn a clean exit, and every seat lost
8644    /// its tools for hours -- one client's teardown took down a connection
8645    /// carrying ~170 other routes.
8646    ///
8647    /// The window is opened here by calling the production path that opens it
8648    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
8649    /// state that matters is "in `closing_connections`, sink still open, relay
8650    /// still pending", and it lasts only from the close request until the
8651    /// connection loop reacts to it; a socket-level test can flood a client into
8652    /// that escalation but cannot pin the module's ack inside the window. Closing
8653    /// the socket instead takes the other path entirely -- connection teardown
8654    /// removes the pending relay under the same lock, so the ack finds nothing.
8655    #[tokio::test]
8656    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8657        let registry = Arc::new(Registry::default());
8658        let forwarding = Arc::new(ForwardingTable::default());
8659        let handler =
8660            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8661
8662        let module_connection = ConnectionId::new(30);
8663        let (module_ctx, mut module_rx) = route_ctx(module_connection);
8664        hello_via_sink(
8665            &handler,
8666            &module_ctx,
8667            &mut module_rx,
8668            hello_frame("aft", PROTOCOL_VERSION, 7),
8669        )
8670        .await;
8671
8672        let dying_client = ConnectionId::new(31);
8673        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8674
8675        // A published route on the dying client. The escalation below only marks
8676        // a connection closing for a route it has already published.
8677        let (first_task, first_bind) = relay_route_open(
8678            &handler,
8679            dying_client,
8680            &dying_ctx.egress,
8681            &mut module_rx,
8682            100,
8683            "aft",
8684            "closing-first",
8685        )
8686        .await;
8687        handler
8688            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8689            .await
8690            .unwrap();
8691        assert!(first_task.await.unwrap().is_empty());
8692        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8693
8694        // A second route.open from the same client, relayed and awaiting its ack.
8695        let (second_task, second_bind) = relay_route_open(
8696            &handler,
8697            dying_client,
8698            &dying_ctx.egress,
8699            &mut module_rx,
8700            101,
8701            "aft",
8702            "closing-second",
8703        )
8704        .await;
8705        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8706
8707        // The window: the client is closing, its sink is still open, and its
8708        // second bind is still pending.
8709        assert!(forwarding
8710            .escalate_client_delivery_failure(
8711                dying_client,
8712                first_channel,
8713                first_epoch,
8714                CloseReason::new(
8715                    "module_to_client_delivery_failed",
8716                    "client egress refused a module frame",
8717                ),
8718                crate::forwarding::UndeliveredFrame {
8719                    module_id: None,
8720                    sink: &dying_ctx.egress,
8721                },
8722            )
8723            .unwrap());
8724        assert!(!dying_ctx.egress.is_closed());
8725
8726        // The frame that used to end the module connection.
8727        let ack = handler
8728            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8729            .await;
8730        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8731        if module_loop_error.is_some() {
8732            // What the server's connection loop does with a router error that has
8733            // no ERROR-frame translation: end the connection, which releases the
8734            // module's registration and every route on it.
8735            handler.cleanup_connection(module_connection).unwrap();
8736        }
8737        // Read the module's next frame before opening the co-tenant's route, so
8738        // the GOODBYE assertion below is about THIS ack and not about later
8739        // traffic. `None` means the module was told nothing.
8740        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8741            .await
8742            .ok()
8743            .flatten();
8744
8745        // 1. The module connection is still registered.
8746        assert!(
8747            registry
8748                .get_module_by_connection(module_connection)
8749                .unwrap()
8750                .is_some(),
8751            "one client's closing connection ended the shared module connection: \
8752             {module_loop_error:?}"
8753        );
8754        // ...and still serving: another client can open and use a route on it.
8755        let cotenant = ConnectionId::new(32);
8756        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8757        let (cotenant_task, cotenant_bind) = relay_route_open(
8758            &handler,
8759            cotenant,
8760            &cotenant_ctx.egress,
8761            &mut module_rx,
8762            102,
8763            "aft",
8764            "closing-cotenant",
8765        )
8766        .await;
8767        handler
8768            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8769            .await
8770            .unwrap();
8771        assert!(cotenant_task.await.unwrap().is_empty());
8772        let (cotenant_channel, cotenant_epoch) =
8773            published_route(&cotenant_rx.recv().await.unwrap());
8774        assert!(matches!(
8775            forwarding
8776                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8777                .unwrap(),
8778            DataRoute::Client(DataRouteState::Bound(_))
8779        ));
8780
8781        // 2. The module was told to drop the binding it created for the route
8782        //    that will never be published.
8783        let goodbye = post_ack_module_frame
8784            .expect("module receives a GOODBYE for the abandoned route channel");
8785        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8786        assert_eq!(goodbye.header.channel, abandoned_channel);
8787        assert_eq!(goodbye.header.epoch, abandoned_epoch);
8788
8789        // 3. The dying client received nothing: no route was ever published to
8790        //    it. Its route.open is answered as unavailable, which the connection
8791        //    loop would write to a socket that is already going away.
8792        assert!(dying_rx.try_recv().is_err());
8793        let second_response = second_task.await.unwrap();
8794        assert_eq!(second_response.len(), 1);
8795        assert_eq!(
8796            parse_error(&second_response[0])["code"],
8797            "target_unavailable"
8798        );
8799    }
8800
8801    /// A handler whose `route.open` parks after queueing the route.bind and
8802    /// before marking it enqueued, with module "aft" registered on connection
8803    /// 40. Returns the two-party barrier that parks and releases it.
8804    async fn route_open_held_after_bind_send() -> (
8805        ControlHandler,
8806        Arc<tokio::sync::Barrier>,
8807        RouteCtx,
8808        mpsc::Receiver<crate::router::OutboundFrame>,
8809    ) {
8810        let hold = Arc::new(tokio::sync::Barrier::new(2));
8811        let handler = ControlHandler::with_forwarding(
8812            Arc::new(Registry::default()),
8813            Arc::new(ForwardingTable::default()),
8814        )
8815        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
8816        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8817        hello_via_sink(
8818            &handler,
8819            &module_ctx,
8820            &mut module_rx,
8821            hello_frame("aft", PROTOCOL_VERSION, 7),
8822        )
8823        .await;
8824        (handler, hold, module_ctx, module_rx)
8825    }
8826
8827    /// A fast module acks the route.bind, and the reader commits the route,
8828    /// before `route.open` has marked the relay enqueued. The daemon used to
8829    /// read the missing relay as an abandoned bind and send the module a
8830    /// GOODBYE for the route it had just committed: the client was told its
8831    /// route was open, the module unbound the channel, and the client's first
8832    /// request was dropped on the module side as unbound.
8833    #[tokio::test]
8834    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
8835        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8836        let client = ConnectionId::new(41);
8837        let (client_ctx, mut client_rx) = route_ctx(client);
8838
8839        let (open_task, bind) = relay_route_open(
8840            &handler,
8841            client,
8842            &client_ctx.egress,
8843            &mut module_rx,
8844            200,
8845            "aft",
8846            "acked-before-mark",
8847        )
8848        .await;
8849        let (module_channel, module_epoch) = route_bind_channel(&bind);
8850        // route.open is now parked between queueing the bind and marking it.
8851        hold.wait().await;
8852
8853        // The module answers inside that window, and the route commits.
8854        assert!(handler
8855            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8856            .await
8857            .unwrap()
8858            .is_empty());
8859        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
8860
8861        hold.wait().await;
8862        assert!(
8863            open_task.await.unwrap().is_empty(),
8864            "route.open answers accepted"
8865        );
8866
8867        // Every frame route.open could have sent is already queued, because it
8868        // has returned; the module must have been sent nothing.
8869        match module_rx.try_recv() {
8870            Err(mpsc::error::TryRecvError::Empty) => {}
8871            Ok(outbound) => panic!(
8872                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
8873                outbound.frame.header.ty,
8874                outbound.frame.header.channel,
8875                outbound.frame.header.epoch
8876            ),
8877            Err(err) => panic!("module egress closed: {err}"),
8878        }
8879
8880        // The client's first request on the route is the next thing the module
8881        // reads, on the binding it just created.
8882        let router = Router::with_control_handler(Arc::new(handler.clone()));
8883        let request = Frame::build(
8884            FrameType::Request,
8885            Flags::new(true, Priority::Interactive, false),
8886            client_channel,
8887            client_epoch,
8888            201,
8889            b"first request".to_vec(),
8890        )
8891        .unwrap();
8892        router
8893            .route_for_connection(&client_ctx, request)
8894            .await
8895            .unwrap();
8896        let delivered = module_rx.try_recv().unwrap().frame;
8897        assert_eq!(delivered.header.ty, FrameType::Request);
8898        assert_eq!(
8899            (delivered.header.channel, delivered.header.epoch),
8900            (module_channel, module_epoch)
8901        );
8902        assert_eq!(delivered.body, b"first request");
8903    }
8904
8905    /// The client goes away while `route.open` is still queueing the bind, so
8906    /// the relay is dropped with no module reply. The teardown cannot send the
8907    /// GOODBYE itself (it could reach the module ahead of the bind), so
8908    /// `route.open` must send it once the bind is queued; otherwise the module
8909    /// keeps a binding the daemon will never commit.
8910    #[tokio::test]
8911    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
8912        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8913        let client = ConnectionId::new(42);
8914        let (client_ctx, _client_rx) = route_ctx(client);
8915
8916        let (open_task, bind) = relay_route_open(
8917            &handler,
8918            client,
8919            &client_ctx.egress,
8920            &mut module_rx,
8921            300,
8922            "aft",
8923            "released-before-mark",
8924        )
8925        .await;
8926        let (module_channel, module_epoch) = route_bind_channel(&bind);
8927        hold.wait().await;
8928
8929        handler.cleanup_connection(client).unwrap();
8930        assert!(
8931            module_rx.try_recv().is_err(),
8932            "client teardown must not GOODBYE a bind route.open is still queueing"
8933        );
8934
8935        hold.wait().await;
8936        let response = open_task.await.unwrap();
8937        assert_eq!(response.len(), 1);
8938        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8939
8940        let goodbye = module_rx
8941            .try_recv()
8942            .expect("module is told to drop the binding the daemon abandoned")
8943            .frame;
8944        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8945        assert_eq!(
8946            (goodbye.header.channel, goodbye.header.epoch),
8947            (module_channel, module_epoch)
8948        );
8949        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8950    }
8951
8952    /// The module acks inside the same window, but the client's egress closed
8953    /// first, so the reader refuses to commit. The ack proves the module holds
8954    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
8955    /// send a second one.
8956    #[tokio::test]
8957    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
8958        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8959        let client = ConnectionId::new(43);
8960        let (client_ctx, client_rx) = route_ctx(client);
8961
8962        let (open_task, bind) = relay_route_open(
8963            &handler,
8964            client,
8965            &client_ctx.egress,
8966            &mut module_rx,
8967            400,
8968            "aft",
8969            "gone-client-before-mark",
8970        )
8971        .await;
8972        let (module_channel, module_epoch) = route_bind_channel(&bind);
8973        hold.wait().await;
8974
8975        drop(client_rx);
8976        assert!(client_ctx.egress.is_closed());
8977        handler
8978            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8979            .await
8980            .unwrap();
8981
8982        hold.wait().await;
8983        let response = open_task.await.unwrap();
8984        assert_eq!(response.len(), 1);
8985        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8986
8987        let goodbye = module_rx
8988            .try_recv()
8989            .expect("module is told to drop the binding it acked")
8990            .frame;
8991        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8992        assert_eq!(
8993            (goodbye.header.channel, goodbye.header.epoch),
8994            (module_channel, module_epoch)
8995        );
8996        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8997        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
8998    }
8999
9000    /// The fence at the module-loop boundary, stated as its own contract: which
9001    /// forwarding failures are allowed to end the module connection that is being
9002    /// served. A `ConnectionClosing` naming some client is about that client, and
9003    /// a module connection is shared; the same error naming the module's own
9004    /// connection is about this connection and must stay fatal, as must failures
9005    /// that are about the forwarding table itself.
9006    #[test]
9007    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9008        let handler = ControlHandler::default();
9009        let module_connection = ConnectionId::new(30);
9010        let client_connection = ConnectionId::new(31);
9011
9012        handler
9013            .refuse_to_end_module_connection_for_a_client(
9014                module_connection,
9015                77,
9016                ForwardingError::ConnectionClosing {
9017                    connection_id: client_connection,
9018                },
9019            )
9020            .expect("a closing client must never end the module connection");
9021
9022        assert!(matches!(
9023            handler.refuse_to_end_module_connection_for_a_client(
9024                module_connection,
9025                78,
9026                ForwardingError::ConnectionClosing {
9027                    connection_id: module_connection,
9028                },
9029            ),
9030            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9031                connection_id
9032            })) if connection_id == module_connection
9033        ));
9034        assert!(matches!(
9035            handler.refuse_to_end_module_connection_for_a_client(
9036                module_connection,
9037                79,
9038                ForwardingError::Poisoned,
9039            ),
9040            Err(RouterError::Forwarding(ForwardingError::Poisoned))
9041        ));
9042        assert!(matches!(
9043            handler.refuse_to_end_module_connection_for_a_client(
9044                module_connection,
9045                80,
9046                ForwardingError::StaleModuleEndpoint,
9047            ),
9048            Err(RouterError::Forwarding(
9049                ForwardingError::StaleModuleEndpoint
9050            ))
9051        ));
9052    }
9053
9054    /// The spawn-attestation guard is what stops a connected module from claiming
9055    /// another module's identity and being stamped `Reserved` for it. Every other
9056    /// test that supplies a consumer_identity supplies a CORRECT one, because a
9057    /// correct one is what the rest of the flow needs -- so the guard's rejection
9058    /// branch was never the subject of an assertion, only its acceptance branch.
9059    ///
9060    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
9061    /// whole subc-core library suite green; only the forwarding integration tests
9062    /// notice, and they notice for unrelated reasons. This test exists so the
9063    /// refusal itself is asserted where the guard lives: it fails if the identity
9064    /// check stops refusing, which is the direction that matters, since a guard
9065    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
9066    #[tokio::test]
9067    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9068        let registry = Arc::new(Registry::default());
9069        let forwarding = Arc::new(ForwardingTable::default());
9070        let supervisor = SupervisorHandle::new();
9071        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9072        let handler =
9073            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9074                .with_supervisor(supervisor);
9075
9076        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9077        hello_via_sink(
9078            &handler,
9079            &target_ctx,
9080            &mut target_rx,
9081            hello_frame("target", PROTOCOL_VERSION, 1),
9082        )
9083        .await;
9084
9085        // A real supervised module id presenting the wrong nonce. This is the
9086        // impersonation case: the attacker knows a privileged module_id, which is
9087        // public, and guesses at the nonce, which is not.
9088        let wrong_nonce = handler
9089            .handle_control_frame(
9090                &route_ctx(ConnectionId::new(91)).0,
9091                route_open_frame_with_admission_facts(
9092                    20,
9093                    "target",
9094                    unique_project_root("admission-facts"),
9095                    Some(subc_control::ConsumerIdentity {
9096                        module_id: "fed".to_string(),
9097                        launch_nonce: "not-the-real-nonce".to_string(),
9098                    }),
9099                    None,
9100                ),
9101            )
9102            .await
9103            .unwrap();
9104        assert_eq!(
9105            parse_error(&wrong_nonce[0])["code"],
9106            "bad_consumer_identity",
9107            "a mismatched launch nonce must be refused, not stamped Reserved"
9108        );
9109
9110        // A module id the supervisor never spawned at all, so no nonce exists to
9111        // compare against. An implementation that treats "no record" as "nothing
9112        // to check" fails open here while passing the case above.
9113        let never_spawned = handler
9114            .handle_control_frame(
9115                &route_ctx(ConnectionId::new(92)).0,
9116                route_open_frame_with_admission_facts(
9117                    21,
9118                    "target",
9119                    unique_project_root("admission-facts"),
9120                    Some(subc_control::ConsumerIdentity {
9121                        module_id: "never-spawned".to_string(),
9122                        launch_nonce: "any-nonce".to_string(),
9123                    }),
9124                    None,
9125                ),
9126            )
9127            .await
9128            .unwrap();
9129        assert_eq!(
9130            parse_error(&never_spawned[0])["code"],
9131            "bad_consumer_identity",
9132            "an unspawned module_id must be refused rather than accepted for lack of a record"
9133        );
9134    }
9135
9136    /// The refusal test above proves the guard says NO. Nothing proved it can say
9137    /// YES, and the difference is not academic: replacing the whole authorization
9138    /// with `false` -- admitting no consumer identity at all, revoking Reserved
9139    /// standing for every supervised module in the fleet -- leaves 110 of the 111
9140    /// library tests GREEN. The one that notices does so by HANGING, because it
9141    /// waits for a bind that can no longer happen.
9142    ///
9143    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
9144    /// or flaky test, invites a RETRY rather than an investigation, and the retry
9145    /// hangs too and gets blamed on the runner. So a total revocation of the
9146    /// daemon's trust grant would have shipped behind a symptom nobody attributes
9147    /// to code.
9148    ///
9149    /// The bias is structural rather than accidental. A REFUSAL looks like a
9150    /// failure someone writes a test for; a GRANT looks like the happy path. Every
9151    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
9152    /// suite for that reason, and this one is the purest case in the daemon.
9153    ///
9154    /// This test asserts the EFFECT rather than the absence of an error: the module
9155    /// receives a RouteBind and it carries `Reserved` naming the attested module.
9156    /// A guard that admitted nobody would produce no bind at all; one that admitted
9157    /// everybody would stamp the wrong principal, which the refusal test catches.
9158    #[tokio::test]
9159    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9160        let registry = Arc::new(Registry::default());
9161        let forwarding = Arc::new(ForwardingTable::default());
9162        let supervisor = SupervisorHandle::new();
9163        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9164        let handler =
9165            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9166                .with_supervisor(supervisor);
9167
9168        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9169        hello_via_sink(
9170            &handler,
9171            &target_ctx,
9172            &mut target_rx,
9173            hello_frame("target", PROTOCOL_VERSION, 1),
9174        )
9175        .await;
9176
9177        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9178        let route_handler = handler.clone();
9179        let route_task = tokio::spawn(async move {
9180            route_handler
9181                .handle_control_frame(
9182                    &client_ctx,
9183                    route_open_frame_with_admission_facts(
9184                        30,
9185                        "target",
9186                        unique_project_root("admission-facts"),
9187                        Some(subc_control::ConsumerIdentity {
9188                            module_id: "fed".to_string(),
9189                            launch_nonce: "fed-nonce".to_string(),
9190                        }),
9191                        None,
9192                    ),
9193                )
9194                .await
9195                .unwrap()
9196        });
9197
9198        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
9199        // the very mutation it exists to catch -- a guard that admits nobody -- no
9200        // bind is ever sent, so it HUNG rather than failing. That reproduces the
9201        // exact defect being fixed: a total revocation detected only as a stalled
9202        // suite, which reads as flakiness and invites a retry. An acceptance test
9203        // that waits for an effect must bound the wait, or a red becomes a hang.
9204        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9205            .await
9206            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9207            .expect("module control channel closed before route.bind");
9208        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9209        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9210            panic!("expected route.bind")
9211        };
9212        assert_eq!(
9213            principal,
9214            Some(Principal::Reserved {
9215                module_id: "fed".to_string()
9216            }),
9217            "a correctly attested consumer must be stamped Reserved for its own id"
9218        );
9219
9220        handler
9221            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9222            .await
9223            .unwrap();
9224        assert!(route_task.await.unwrap().is_empty());
9225        assert!(
9226            matches!(
9227                serde_json::from_slice::<ClientControlResponse>(
9228                    &client_rx.recv().await.unwrap().body
9229                )
9230                .unwrap(),
9231                ClientControlResponse::RouteOpen { .. }
9232            ),
9233            "the route must actually open, not merely avoid an error"
9234        );
9235    }
9236
9237    #[tokio::test(start_paused = true)]
9238    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9239        let registry = Arc::new(Registry::default());
9240        let forwarding = Arc::new(ForwardingTable::default());
9241        let supervisor = SupervisorHandle::new();
9242        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9243        let handler =
9244            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9245                .with_supervisor(supervisor);
9246
9247        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9248        hello_via_sink(
9249            &handler,
9250            &target_ctx,
9251            &mut target_rx,
9252            hello_frame("target", PROTOCOL_VERSION, 1),
9253        )
9254        .await;
9255
9256        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9257        let direct_handler = handler.clone();
9258        let direct_open = tokio::spawn(async move {
9259            direct_handler
9260                .handle_control_frame(
9261                    &direct_ctx,
9262                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
9263                )
9264                .await
9265                .unwrap()
9266        });
9267        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9268            .await
9269            .expect("no direct route.bind within 5s")
9270            .expect("target control channel closed before direct route.bind");
9271        handler
9272            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
9273            .await
9274            .unwrap();
9275        assert!(direct_open.await.unwrap().is_empty());
9276        let _ = direct_rx.recv().await.unwrap();
9277
9278        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
9279        let reserved_handler = handler.clone();
9280        let reserved_open = tokio::spawn(async move {
9281            reserved_handler
9282                .handle_control_frame(
9283                    &reserved_ctx,
9284                    route_open_frame_with_admission_facts(
9285                        3,
9286                        "target",
9287                        unique_project_root("admission-facts"),
9288                        Some(ConsumerIdentity {
9289                            module_id: "fed".to_string(),
9290                            launch_nonce: "fed-nonce".to_string(),
9291                        }),
9292                        None,
9293                    ),
9294                )
9295                .await
9296                .unwrap()
9297        });
9298        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9299            .await
9300            .expect("no reserved route.bind within 5s")
9301            .expect("target control channel closed before reserved route.bind");
9302        handler
9303            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
9304            .await
9305            .unwrap();
9306        assert!(reserved_open.await.unwrap().is_empty());
9307        let _ = reserved_rx.recv().await.unwrap();
9308
9309        forwarding
9310            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
9311            .unwrap();
9312        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
9313        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
9314            module_id: Some("target".to_string()),
9315        })
9316        .unwrap();
9317        let census_frame =
9318            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
9319        let response = handler
9320            .handle_control_frame(&census_ctx, census_frame)
9321            .await
9322            .unwrap()
9323            .pop()
9324            .unwrap();
9325        let actual: Value = serde_json::from_slice(&response.body).unwrap();
9326        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
9327        assert!(matches!(
9328            decoded,
9329            ClientControlResponse::SupervisorRoutes { .. }
9330        ));
9331        let routes = actual["modules"][0]["routes"].as_array().unwrap();
9332        assert_eq!(routes.len(), 2);
9333        assert!(routes.iter().all(|route| route["draining"] == true));
9334        // The census carries WHY: the reason the drain was begun with, in the
9335        // route.closing vocabulary, on every draining route this drain marked.
9336        assert!(
9337            routes.iter().all(|route| route["drain_reason"] == "reload"),
9338            "draining routes must name the drain's reason: {routes:?}"
9339        );
9340        assert!(routes.iter().any(|route| {
9341            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
9342        }));
9343        assert!(routes.iter().any(|route| {
9344            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
9345        }));
9346
9347        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9348            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
9349        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9350            std::fs::write(
9351                &golden_path,
9352                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
9353            )
9354            .unwrap();
9355        }
9356        let expected: Value =
9357            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
9358        assert_eq!(actual, expected);
9359    }
9360
9361    async fn query_live_roots(
9362        handler: &ControlHandler,
9363        module_ctx: &RouteCtx,
9364    ) -> ModuleControlResponseToModule {
9365        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
9366        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
9367        let response = handler
9368            .handle_control_frame(module_ctx, frame)
9369            .await
9370            .unwrap()
9371            .pop()
9372            .unwrap();
9373        serde_json::from_slice(&response.body).unwrap()
9374    }
9375
9376    #[tokio::test(start_paused = true)]
9377    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
9378        let registry = Arc::new(Registry::default());
9379        let forwarding = Arc::new(ForwardingTable::default());
9380        let handler = ControlHandler::with_forwarding(registry, forwarding);
9381        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
9382        hello_via_sink(
9383            &handler,
9384            &target_ctx,
9385            &mut target_rx,
9386            hello_frame("target", PROTOCOL_VERSION, 1),
9387        )
9388        .await;
9389        let root = unique_project_root("live-roots-known");
9390        let path = ProjectRootId::from_path_allowing_missing(root.path())
9391            .unwrap()
9392            .as_path()
9393            .to_path_buf();
9394        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
9395        let open_handler = handler.clone();
9396        let opened = tokio::spawn(async move {
9397            open_handler
9398                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
9399                .await
9400                .unwrap()
9401        });
9402        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9403            .await
9404            .unwrap()
9405            .unwrap();
9406        handler
9407            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
9408            .await
9409            .unwrap();
9410        assert!(opened.await.unwrap().is_empty());
9411        let _ = client_rx.recv().await.unwrap();
9412
9413        let root = unique_project_root("live-roots-pending");
9414        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
9415            .unwrap()
9416            .as_path()
9417            .to_path_buf();
9418        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
9419        let open_handler = handler.clone();
9420        let pending = tokio::spawn(async move {
9421            open_handler
9422                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
9423                .await
9424                .unwrap()
9425        });
9426        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9427            .await
9428            .unwrap()
9429            .unwrap();
9430        let actual = query_live_roots(&handler, &target_ctx).await;
9431        let ModuleControlResponseToModule::LiveRoots {
9432            roots,
9433            unknown_root_bindings,
9434            total_bindings,
9435        } = actual
9436        else {
9437            panic!("expected live roots")
9438        };
9439        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
9440        assert_eq!(unknown_root_bindings, 0);
9441        assert_eq!(
9442            roots.len(),
9443            2,
9444            "root-known arm must retain each canonical root"
9445        );
9446        assert_eq!(
9447            total_bindings,
9448            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9449        );
9450        let counts = roots
9451            .iter()
9452            .map(|root| (root.project_root.clone(), root.bound, root.pending))
9453            .collect::<Vec<_>>();
9454        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
9455        expected.sort_by(|a, b| a.0.cmp(&b.0));
9456        assert_eq!(
9457            counts, expected,
9458            "roots must sort by path and count pending separately"
9459        );
9460        handler
9461            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
9462            .await
9463            .unwrap();
9464        assert!(pending.await.unwrap().is_empty());
9465    }
9466
9467    #[tokio::test(start_paused = true)]
9468    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
9469        let forwarding = Arc::new(ForwardingTable::default());
9470        let handler =
9471            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9472        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
9473        hello_via_sink(
9474            &handler,
9475            &target_ctx,
9476            &mut target_rx,
9477            hello_frame("target", PROTOCOL_VERSION, 1),
9478        )
9479        .await;
9480        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
9481        let pending = forwarding
9482            .begin_route_bind_relay_for_test(
9483                client_ctx.connection_id,
9484                client_ctx.egress.clone(),
9485                2,
9486                "target",
9487            )
9488            .unwrap();
9489        forwarding
9490            .complete_pending_relay(
9491                target_ctx.connection_id,
9492                pending.corr,
9493                RouteBindRelayOutcome::Accepted,
9494            )
9495            .unwrap();
9496        let actual = query_live_roots(&handler, &target_ctx).await;
9497        let ModuleControlResponseToModule::LiveRoots {
9498            roots,
9499            unknown_root_bindings,
9500            total_bindings,
9501        } = actual
9502        else {
9503            panic!("expected live roots")
9504        };
9505        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
9506        assert_eq!(
9507            unknown_root_bindings, 1,
9508            "unknown-root arm must not read as no bindings"
9509        );
9510        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
9511        assert_eq!(
9512            total_bindings,
9513            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9514        );
9515    }
9516
9517    /// A module reads HELLO_ACK as its first frame and exits on anything else,
9518    /// so the ack has to be on its outbound queue before the module is
9519    /// routable. The connection loop writes a handler's replies only after the
9520    /// handler returns; this test stops in exactly that gap, runs a real
9521    /// route.open from another connection, and only then writes whatever the
9522    /// HELLO handler returned, the way the loop would. If the ack were still a
9523    /// reply, the route.bind request would reach the module first.
9524    #[tokio::test(start_paused = true)]
9525    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
9526        let forwarding = Arc::new(ForwardingTable::default());
9527        let handler =
9528            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9529        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
9530        let replies = handler
9531            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9532            .await
9533            .unwrap();
9534        let queued_by_hello = module_rx.len();
9535
9536        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9537        let open_handler = handler.clone();
9538        let open = tokio::spawn(async move {
9539            open_handler
9540                .handle_control_frame(
9541                    &client_ctx,
9542                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9543                )
9544                .await
9545                .unwrap()
9546        });
9547        // Let the route.open run until its route.bind is on the module's queue.
9548        let mut spins = 0;
9549        while module_rx.len() == queued_by_hello {
9550            spins += 1;
9551            assert!(spins < 10_000, "route.open never queued a route.bind");
9552            tokio::task::yield_now().await;
9553        }
9554
9555        // Now the connection loop's half: write the HELLO handler's replies.
9556        for reply in replies {
9557            module_ctx.egress.send(reply).await.unwrap();
9558        }
9559
9560        let first = module_rx.recv().await.unwrap().frame;
9561        assert_eq!(
9562            first.header.ty,
9563            FrameType::HelloAck,
9564            "the first frame a registering module reads must be its HELLO_ACK"
9565        );
9566        assert_eq!(first.header.corr, 7);
9567        let second = module_rx.recv().await.unwrap().frame;
9568        assert_eq!(second.header.ty, FrameType::Request);
9569        assert!(
9570            matches!(
9571                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9572                ModuleControlRequest::RouteBind { .. }
9573            ),
9574            "the route.bind follows the ack"
9575        );
9576        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9577
9578        handler
9579            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9580            .await
9581            .unwrap();
9582        assert!(open.await.unwrap().is_empty());
9583        let _ = client_rx.recv().await.unwrap();
9584    }
9585
9586    #[tokio::test(start_paused = true)]
9587    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9588        let handler = ControlHandler::with_forwarding(
9589            Arc::new(Registry::default()),
9590            Arc::new(ForwardingTable::default()),
9591        );
9592        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9593        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9594        hello_via_sink(
9595            &handler,
9596            &first_ctx,
9597            &mut first_rx,
9598            hello_frame("first", PROTOCOL_VERSION, 1),
9599        )
9600        .await;
9601        hello_via_sink(
9602            &handler,
9603            &second_ctx,
9604            &mut second_rx,
9605            hello_frame("second", PROTOCOL_VERSION, 2),
9606        )
9607        .await;
9608        let root = unique_project_root("second-only");
9609        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9610        let cloned = handler.clone();
9611        let open = tokio::spawn(async move {
9612            cloned
9613                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9614                .await
9615                .unwrap()
9616        });
9617        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9618            .await
9619            .unwrap()
9620            .unwrap();
9621        let first = query_live_roots(&handler, &first_ctx).await;
9622        let second = query_live_roots(&handler, &second_ctx).await;
9623        assert!(
9624            matches!(
9625                first,
9626                ModuleControlResponseToModule::LiveRoots {
9627                    total_bindings: 0,
9628                    ..
9629                }
9630            ),
9631            "cross-module scope must not expose another module's roots"
9632        );
9633        assert!(
9634            matches!(
9635                second,
9636                ModuleControlResponseToModule::LiveRoots {
9637                    total_bindings: 1,
9638                    ..
9639                }
9640            ),
9641            "second module must see its pending route"
9642        );
9643        handler
9644            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9645            .await
9646            .unwrap();
9647        assert!(open.await.unwrap().is_empty());
9648    }
9649
9650    #[tokio::test(start_paused = true)]
9651    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9652        let handler = ControlHandler::with_forwarding(
9653            Arc::new(Registry::default()),
9654            Arc::new(ForwardingTable::default()),
9655        );
9656        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9657        hello_via_sink(
9658            &handler,
9659            &target_ctx,
9660            &mut target_rx,
9661            hello_frame("target", PROTOCOL_VERSION, 1),
9662        )
9663        .await;
9664        let actual = query_live_roots(&handler, &target_ctx).await;
9665        let ModuleControlResponseToModule::LiveRoots {
9666            roots,
9667            unknown_root_bindings,
9668            total_bindings,
9669        } = actual
9670        else {
9671            panic!("expected live roots")
9672        };
9673        assert!(roots.is_empty());
9674        assert_eq!(unknown_root_bindings, 0);
9675        assert_eq!(total_bindings, 0);
9676        assert_eq!(
9677            total_bindings,
9678            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9679        );
9680    }
9681
9682    /// Read the vendored fed corpus rather than hand-building a package.
9683    ///
9684    /// A hand-built object encodes what the test author believed the carrier
9685    /// emits. These vectors are what it actually emits, and one of them exists
9686    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
9687    /// ignores additive unknown fields at the traversal emit terminus."
9688    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9689        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9690            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9691        let text = std::fs::read_to_string(&path)
9692            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9693        let vectors: Vec<(String, Value)> = text
9694            .lines()
9695            .filter(|line| !line.trim().is_empty())
9696            .map(|line| {
9697                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9698                let id = entry["corpus_id"]
9699                    .as_str()
9700                    .expect("every vector carries a corpus_id")
9701                    .to_string();
9702                (id, entry["package"].clone())
9703            })
9704            .collect();
9705        // Pin the count: a corpus that silently shrinks would take its coverage
9706        // with it, and a suite reading N-1 vectors reports the same clean pass
9707        // as one reading N.
9708        assert_eq!(
9709            vectors.len(),
9710            3,
9711            "vendored fed corpus changed size; re-sync from subc-federation"
9712        );
9713
9714        // Pin what makes the corpus DISCRIMINATING, not just present.
9715        //
9716        // The relay test below takes its expected value from the corpus, so the
9717        // corpus supplies the test's power to detect a lossy relay rather than
9718        // its correctness. A relay that dropped unrecognised fields would still
9719        // be caught -- but only by a package carrying fields it does not know.
9720        // Shrink every package to the handful of keys any implementation would
9721        // recognise and the test keeps passing over an input that can no longer
9722        // fail, which is the same clean green as a corpus that shrank away.
9723        //
9724        // So assert the precondition rather than duplicating the packages here:
9725        // at least one vector must carry a field beyond the small common set.
9726        // That is one claim to maintain instead of nine, and it fails loudly if
9727        // a re-sync ever flattens the corpus.
9728        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9729        let richest = vectors
9730            .iter()
9731            .filter_map(|(_, package)| package.as_object())
9732            .map(|object| {
9733                object
9734                    .keys()
9735                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9736                    .count()
9737            })
9738            .max()
9739            .unwrap_or(0);
9740        assert!(
9741            richest >= 2,
9742            "vendored corpus no longer carries a package with unmodelled fields, \
9743             so the relay test can no longer distinguish a verbatim relay from a lossy one"
9744        );
9745
9746        vectors
9747    }
9748
9749    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
9750    ///
9751    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
9752    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
9753    /// three-key object, so a relay that quietly dropped fields it did not
9754    /// recognise would satisfy it. These vectors carry nine keys including ones
9755    /// this crate has no type for, so a typed relay fails here and only here.
9756    #[tokio::test]
9757    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9758        for (corpus_id, package) in fed_admission_facts_vectors() {
9759            let registry = Arc::new(Registry::default());
9760            let forwarding = Arc::new(ForwardingTable::default());
9761            let supervisor = SupervisorHandle::new();
9762            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9763            let handler =
9764                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9765                    .with_supervisor(supervisor)
9766                    .with_admission_facts_config(
9767                        Some("fed".to_string()),
9768                        Some(vec!["target".to_string()]),
9769                    );
9770
9771            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9772            hello_via_sink(
9773                &handler,
9774                &target_ctx,
9775                &mut target_rx,
9776                hello_frame("target", PROTOCOL_VERSION, 1),
9777            )
9778            .await;
9779
9780            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9781            let route_handler = handler.clone();
9782            let expected = package.clone();
9783            let route_task = tokio::spawn(async move {
9784                route_handler
9785                    .handle_control_frame(
9786                        &client_ctx,
9787                        route_open_frame_with_admission_facts(
9788                            20,
9789                            "target",
9790                            unique_project_root("admission-facts"),
9791                            Some(subc_control::ConsumerIdentity {
9792                                module_id: "fed".to_string(),
9793                                launch_nonce: "fed-nonce".to_string(),
9794                            }),
9795                            Some(package),
9796                        ),
9797                    )
9798                    .await
9799                    .unwrap()
9800            });
9801
9802            let bind_frame = target_rx.recv().await.unwrap();
9803            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9804            let ModuleControlRequest::RouteBind {
9805                admission_facts, ..
9806            } = bind
9807            else {
9808                panic!("{corpus_id}: expected route.bind")
9809            };
9810            assert_eq!(
9811                admission_facts,
9812                Some(expected),
9813                "{corpus_id}: relay must not add, drop or reshape any field"
9814            );
9815
9816            handler
9817                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9818                .await
9819                .unwrap();
9820            route_task.await.unwrap();
9821        }
9822    }
9823
9824    #[tokio::test]
9825    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9826        let registry = Arc::new(Registry::default());
9827        let forwarding = Arc::new(ForwardingTable::default());
9828        let supervisor = SupervisorHandle::new();
9829        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9830        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9831        let handler =
9832            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9833                .with_supervisor(supervisor)
9834                .with_admission_facts_config(
9835                    Some("fed".to_string()),
9836                    Some(vec!["target".to_string()]),
9837                );
9838
9839        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9840        hello_via_sink(
9841            &handler,
9842            &target_ctx,
9843            &mut target_rx,
9844            hello_frame("target", PROTOCOL_VERSION, 1),
9845        )
9846        .await;
9847        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9848        hello_via_sink(
9849            &handler,
9850            &other_ctx,
9851            &mut other_rx,
9852            hello_frame("other", PROTOCOL_VERSION, 2),
9853        )
9854        .await;
9855
9856        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9857        let expected_facts = facts.clone();
9858        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9859        let route_handler = handler.clone();
9860        let route_task = tokio::spawn(async move {
9861            route_handler
9862                .handle_control_frame(
9863                    &client_ctx,
9864                    route_open_frame_with_admission_facts(
9865                        10,
9866                        "target",
9867                        unique_project_root("admission-facts"),
9868                        Some(subc_control::ConsumerIdentity {
9869                            module_id: "fed".to_string(),
9870                            launch_nonce: "fed-nonce".to_string(),
9871                        }),
9872                        Some(facts.clone()),
9873                    ),
9874                )
9875                .await
9876                .unwrap()
9877        });
9878        let bind_frame = target_rx.recv().await.unwrap();
9879        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9880        let ModuleControlRequest::RouteBind {
9881            admission_facts, ..
9882        } = bind
9883        else {
9884            panic!("expected route.bind")
9885        };
9886        assert_eq!(admission_facts, Some(expected_facts));
9887        handler
9888            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9889            .await
9890            .unwrap();
9891        assert!(route_task.await.unwrap().is_empty());
9892        assert!(matches!(
9893            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9894                .unwrap(),
9895            ClientControlResponse::RouteOpen { .. }
9896        ));
9897
9898        let direct = handler
9899            .handle_control_frame(
9900                &route_ctx(ConnectionId::new(73)).0,
9901                route_open_frame_with_admission_facts(
9902                    11,
9903                    "target",
9904                    unique_project_root("admission-facts"),
9905                    None,
9906                    Some(json!({"x": 1})),
9907                ),
9908            )
9909            .await
9910            .unwrap();
9911        assert_eq!(
9912            parse_error(&direct[0])["code"],
9913            "admission_facts_not_permitted"
9914        );
9915
9916        let different_reserved = handler
9917            .handle_control_frame(
9918                &route_ctx(ConnectionId::new(77)).0,
9919                route_open_frame_with_admission_facts(
9920                    15,
9921                    "target",
9922                    unique_project_root("admission-facts"),
9923                    Some(subc_control::ConsumerIdentity {
9924                        module_id: "other".to_string(),
9925                        launch_nonce: "other-nonce".to_string(),
9926                    }),
9927                    Some(json!({"x": 1})),
9928                ),
9929            )
9930            .await
9931            .unwrap();
9932        assert_eq!(
9933            parse_error(&different_reserved[0])["code"],
9934            "admission_facts_not_permitted"
9935        );
9936
9937        let other_target = handler
9938            .handle_control_frame(
9939                &route_ctx(ConnectionId::new(74)).0,
9940                route_open_frame_with_admission_facts(
9941                    12,
9942                    "other",
9943                    unique_project_root("admission-facts"),
9944                    Some(subc_control::ConsumerIdentity {
9945                        module_id: "fed".to_string(),
9946                        launch_nonce: "fed-nonce".to_string(),
9947                    }),
9948                    Some(json!({"x": 1})),
9949                ),
9950            )
9951            .await
9952            .unwrap();
9953        assert_eq!(
9954            parse_error(&other_target[0])["code"],
9955            "admission_facts_target_not_allowed"
9956        );
9957
9958        let nonexistent = handler
9959            .handle_control_frame(
9960                &route_ctx(ConnectionId::new(75)).0,
9961                route_open_frame_with_admission_facts(
9962                    13,
9963                    "missing",
9964                    unique_project_root("admission-facts"),
9965                    None,
9966                    Some(json!({"x": 1})),
9967                ),
9968            )
9969            .await
9970            .unwrap();
9971        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9972
9973        let described = handler
9974            .handle_control_frame(
9975                &route_ctx(ConnectionId::new(76)).0,
9976                Frame::build(
9977                    FrameType::Request,
9978                    control_flags(),
9979                    0,
9980                    0,
9981                    14,
9982                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9983                )
9984                .unwrap(),
9985            )
9986            .await
9987            .unwrap();
9988        let ClientControlResponse::ServerDescribe { capabilities, .. } =
9989            serde_json::from_slice(&described[0].body).unwrap()
9990        else {
9991            panic!("expected server.describe response")
9992        };
9993        assert!(capabilities
9994            .iter()
9995            .any(|cap| cap == "admission_facts_relay_v1"));
9996    }
9997
9998    #[tokio::test]
9999    async fn admission_facts_without_configured_carrier_are_rejected() {
10000        let registry = Arc::new(Registry::default());
10001        let forwarding = Arc::new(ForwardingTable::default());
10002        let handler = ControlHandler::with_forwarding(registry, forwarding);
10003        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10004        hello_via_sink(
10005            &handler,
10006            &target_ctx,
10007            &mut target_rx,
10008            hello_frame("target", PROTOCOL_VERSION, 1),
10009        )
10010        .await;
10011
10012        let responses = handler
10013            .handle_control_frame(
10014                &route_ctx(ConnectionId::new(79)).0,
10015                route_open_frame_with_admission_facts(
10016                    16,
10017                    "target",
10018                    unique_project_root("admission-facts"),
10019                    None,
10020                    Some(json!({"x": 1})),
10021                ),
10022            )
10023            .await
10024            .unwrap();
10025        assert_eq!(
10026            parse_error(&responses[0])["code"],
10027            "admission_facts_not_permitted"
10028        );
10029    }
10030
10031    #[tokio::test]
10032    async fn route_open_relays_consumer_capabilities_verbatim() {
10033        let registry = Arc::new(Registry::default());
10034        let forwarding = Arc::new(ForwardingTable::default());
10035        let handler =
10036            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10037        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10038        hello_via_sink(
10039            &handler,
10040            &module_ctx,
10041            &mut module_rx,
10042            hello_frame("aft", PROTOCOL_VERSION, 7),
10043        )
10044        .await;
10045
10046        let expected = vec!["elicitation".to_string(), "roots".to_string()];
10047        let expected_for_request = expected.clone();
10048        let project_root = unique_project_root("consumer-capabilities-present");
10049        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10050        let route_handler = handler.clone();
10051        let route_task = tokio::spawn(async move {
10052            route_handler
10053                .handle_control_frame(
10054                    &client_ctx,
10055                    route_open_frame_with_consumer_capabilities(
10056                        401,
10057                        "aft",
10058                        project_root,
10059                        Some(expected_for_request),
10060                    ),
10061                )
10062                .await
10063                .unwrap()
10064        });
10065        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10066            .await
10067            .unwrap()
10068            .unwrap();
10069        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10070        let ModuleControlRequest::RouteBind {
10071            consumer_capabilities,
10072            ..
10073        } = bind
10074        else {
10075            panic!("expected route.bind request, got {bind:?}");
10076        };
10077        assert_eq!(consumer_capabilities, Some(expected.clone()));
10078
10079        handler
10080            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10081            .await
10082            .unwrap();
10083        let route_response = route_task.await.unwrap();
10084        assert!(route_response.is_empty());
10085        let published = client_rx.recv().await.unwrap();
10086        assert!(matches!(
10087            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10088            ClientControlResponse::RouteOpen { .. }
10089        ));
10090    }
10091
10092    #[tokio::test]
10093    async fn route_open_without_consumer_capabilities_relays_none() {
10094        let registry = Arc::new(Registry::default());
10095        let forwarding = Arc::new(ForwardingTable::default());
10096        let handler =
10097            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10098        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10099        hello_via_sink(
10100            &handler,
10101            &module_ctx,
10102            &mut module_rx,
10103            hello_frame("aft", PROTOCOL_VERSION, 7),
10104        )
10105        .await;
10106
10107        let project_root = unique_project_root("consumer-capabilities-absent");
10108        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10109        let route_handler = handler.clone();
10110        let route_task = tokio::spawn(async move {
10111            route_handler
10112                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10113                .await
10114                .unwrap()
10115        });
10116        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10117            .await
10118            .unwrap()
10119            .unwrap();
10120        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10121        let ModuleControlRequest::RouteBind {
10122            consumer_capabilities,
10123            ..
10124        } = bind
10125        else {
10126            panic!("expected route.bind request, got {bind:?}");
10127        };
10128        assert_eq!(consumer_capabilities, None);
10129
10130        handler
10131            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10132            .await
10133            .unwrap();
10134        let route_response = route_task.await.unwrap();
10135        assert!(route_response.is_empty());
10136        let published = client_rx.recv().await.unwrap();
10137        assert!(matches!(
10138            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10139            ClientControlResponse::RouteOpen { .. }
10140        ));
10141    }
10142
10143    /// Opens a route to a freshly registered `aft` with `sent` as the
10144    /// route.open's role_versions, acks the bind, and returns the role_versions
10145    /// the module's bind carried.
10146    async fn bind_role_versions_for(
10147        sent: Option<BTreeMap<String, String>>,
10148        connection: u64,
10149    ) -> Option<BTreeMap<String, String>> {
10150        let registry = Arc::new(Registry::default());
10151        let forwarding = Arc::new(ForwardingTable::default());
10152        let handler =
10153            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10154        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10155        hello_via_sink(
10156            &handler,
10157            &module_ctx,
10158            &mut module_rx,
10159            hello_frame("aft", PROTOCOL_VERSION, 7),
10160        )
10161        .await;
10162        let project_root = unique_project_root("role-versions");
10163        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10164        let route_handler = handler.clone();
10165        let route_task = tokio::spawn(async move {
10166            route_handler
10167                .handle_control_frame(
10168                    &client_ctx,
10169                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
10170                )
10171                .await
10172                .unwrap()
10173        });
10174        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10175            .await
10176            .expect("a well-formed route.open reaches the module as a bind")
10177            .unwrap();
10178        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10179        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10180            panic!("expected route.bind request, got {bind:?}");
10181        };
10182        handler
10183            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10184            .await
10185            .unwrap();
10186        assert!(route_task.await.unwrap().is_empty());
10187        let published = client_rx.recv().await.unwrap();
10188        assert!(matches!(
10189            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10190            ClientControlResponse::RouteOpen { .. }
10191        ));
10192        role_versions
10193    }
10194
10195    #[tokio::test]
10196    async fn route_open_relays_role_versions_verbatim() {
10197        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10198        assert_eq!(
10199            bind_role_versions_for(Some(sent.clone()), 141).await,
10200            Some(sent)
10201        );
10202    }
10203
10204    /// An empty map declares nothing, so the provider sees no field rather
10205    /// than an empty object it would have to treat as a second "none".
10206    #[tokio::test]
10207    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10208        assert_eq!(bind_role_versions_for(None, 143).await, None);
10209        assert_eq!(
10210            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10211            None
10212        );
10213    }
10214
10215    #[tokio::test]
10216    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10217        let registry = Arc::new(Registry::default());
10218        let forwarding = Arc::new(ForwardingTable::default());
10219        let handler =
10220            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10221        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10222        hello_via_sink(
10223            &handler,
10224            &module_ctx,
10225            &mut module_rx,
10226            hello_frame("aft", PROTOCOL_VERSION, 7),
10227        )
10228        .await;
10229
10230        let nine: BTreeMap<String, String> = (0..9)
10231            .map(|index| (format!("role-{index}"), "v1".to_string()))
10232            .collect();
10233        for (label, malformed) in [
10234            (
10235                "invalid role name",
10236                role_versions(&[("Tool_Provider", "v1")]),
10237            ),
10238            ("invalid version", role_versions(&[("tool-provider", "v0")])),
10239            ("nine entries", nine),
10240        ] {
10241            // A refused open answers at once; one that reached the module
10242            // would wait for its bind ack and trip this timeout.
10243            let responses = tokio::time::timeout(
10244                Duration::from_secs(1),
10245                handler.handle_control_frame(
10246                    &route_ctx(ConnectionId::new(148)).0,
10247                    route_open_frame_with_role_versions(
10248                        404,
10249                        "aft",
10250                        unique_project_root("role-versions-malformed"),
10251                        Some(malformed),
10252                    ),
10253                ),
10254            )
10255            .await
10256            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10257            .unwrap();
10258            assert_eq!(responses.len(), 1, "{label}");
10259            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
10260            let error = parse_error(&responses[0]);
10261            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
10262            assert_eq!(
10263                error["detail"]["field"], "role_versions",
10264                "{label}: {error}"
10265            );
10266            assert!(
10267                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
10268                "{label}: a malformed declaration is terminal"
10269            );
10270            assert!(
10271                module_rx.try_recv().is_err(),
10272                "{label}: the module must never see a bind"
10273            );
10274        }
10275    }
10276
10277    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
10278    /// consumer can tell this daemon forwards the field from one that would
10279    /// drop it.
10280    #[tokio::test]
10281    async fn route_role_versions_capability_is_advertised() {
10282        let handler = ControlHandler::new(Arc::new(Registry::default()));
10283        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
10284        let ack = hello_via_sink(
10285            &handler,
10286            &ctx,
10287            &mut rx,
10288            hello_frame("m", PROTOCOL_VERSION, 1),
10289        )
10290        .await;
10291        let ack = parse_ack(&ack);
10292        assert!(
10293            ack.subc_capabilities
10294                .iter()
10295                .any(|c| c == "route-role-versions/v1"),
10296            "{:?}",
10297            ack.subc_capabilities
10298        );
10299
10300        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10301        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10302        let reply = handler
10303            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
10304            .await
10305            .unwrap()
10306            .pop()
10307            .unwrap();
10308        let ClientControlResponse::ServerDescribe { capabilities, .. } =
10309            serde_json::from_slice(&reply.body).unwrap()
10310        else {
10311            panic!("not a server.describe reply");
10312        };
10313        assert!(
10314            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
10315            "{capabilities:?}"
10316        );
10317    }
10318
10319    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
10320    /// catalog entry without `events` (the module declares none) from a daemon
10321    /// that never reports declared events.
10322    #[tokio::test]
10323    async fn catalog_events_capability_is_advertised() {
10324        let handler = ControlHandler::new(Arc::new(Registry::default()));
10325        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10326        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10327        let reply = handler
10328            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
10329            .await
10330            .unwrap()
10331            .pop()
10332            .unwrap();
10333        let ClientControlResponse::ServerDescribe { capabilities, .. } =
10334            serde_json::from_slice(&reply.body).unwrap()
10335        else {
10336            panic!("not a server.describe reply");
10337        };
10338        assert!(
10339            capabilities
10340                .iter()
10341                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
10342            "{capabilities:?}"
10343        );
10344    }
10345
10346    #[tokio::test]
10347    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
10348        let registry = Arc::new(Registry::default());
10349        let forwarding = Arc::new(ForwardingTable::default());
10350        let handler =
10351            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10352                .with_health_probe_timeout(Duration::from_secs(5));
10353        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
10354        hello_via_sink(
10355            &handler,
10356            &module_ctx,
10357            &mut module_rx,
10358            non_routable_hello_frame_with_control_ops(
10359                "mcp",
10360                300,
10361                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10362            ),
10363        )
10364        .await;
10365        assert!(registry
10366            .get_module("mcp")
10367            .unwrap()
10368            .unwrap()
10369            .manifest
10370            .provides
10371            .is_empty());
10372
10373        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
10374        let route_response = handler
10375            .handle_control_frame(
10376                &route_client_ctx,
10377                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
10378            )
10379            .await
10380            .unwrap();
10381        assert_eq!(route_response[0].header.ty, FrameType::Error);
10382        assert_eq!(
10383            parse_error(&route_response[0])["code"],
10384            "target_unavailable"
10385        );
10386        assert!(parse_error(&route_response[0])["message"]
10387            .as_str()
10388            .unwrap()
10389            .contains("does not provide the requested target"));
10390        assert!(module_rx.try_recv().is_err());
10391
10392        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
10393        let health_handler = handler.clone();
10394        let health_task = tokio::spawn(async move {
10395            health_handler
10396                .handle_control_frame(
10397                    &health_client_ctx,
10398                    supervisor_health_probe_frame(302, "mcp"),
10399                )
10400                .await
10401                .unwrap()
10402        });
10403        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10404            .await
10405            .unwrap()
10406            .unwrap();
10407        assert_eq!(
10408            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10409            ModuleControlRequest::HealthCheck {}
10410        );
10411        handler
10412            .handle_control_frame(
10413                &module_ctx,
10414                health_response(health_frame.header.corr, HealthStatus::Ok),
10415            )
10416            .await
10417            .unwrap();
10418        let health_response = health_task.await.unwrap();
10419        assert_eq!(health_response[0].header.ty, FrameType::Response);
10420        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10421            ClientControlResponse::SupervisorHealthProbe {
10422                module_id, status, ..
10423            } => {
10424                assert_eq!(module_id, "mcp");
10425                assert_eq!(status, HealthStatus::Ok);
10426            }
10427            other => panic!("unexpected health response: {other:?}"),
10428        }
10429
10430        // Exercise the forwarding cleanup path directly while leaving the registry
10431        // advertisement in place. If cleanup leaves a stale control sink behind,
10432        // the next probe will enqueue onto it and wait for the long probe timeout
10433        // instead of returning an immediate no-connection error.
10434        forwarding
10435            .cleanup_connection(module_ctx.connection_id)
10436            .unwrap();
10437        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
10438        let cleanup_response = tokio::time::timeout(
10439            Duration::from_millis(200),
10440            handler.handle_control_frame(
10441                &cleanup_probe_ctx,
10442                supervisor_health_probe_frame(303, "mcp"),
10443            ),
10444        )
10445        .await
10446        .expect("probe should fail immediately when the control lane is gone")
10447        .unwrap();
10448        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
10449        assert_eq!(
10450            parse_error(&cleanup_response[0])["code"],
10451            "target_unavailable"
10452        );
10453        assert!(parse_error(&cleanup_response[0])["message"]
10454            .as_str()
10455            .unwrap()
10456            .contains("no module connection"));
10457
10458        handler
10459            .cleanup_connection(module_ctx.connection_id)
10460            .unwrap();
10461    }
10462
10463    #[tokio::test]
10464    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
10465        let registry = Arc::new(Registry::default());
10466        let supervisor_handle = SupervisorHandle::new();
10467        let supervisor =
10468            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
10469                .with_handle(supervisor_handle.clone())
10470                .with_connection_file_path(
10471                    std::env::temp_dir()
10472                        .join(format!("subc-route-open-warming-{}", std::process::id())),
10473                );
10474        let module = supervisor
10475            .supervise_configured(
10476                ModuleSpec {
10477                    module_id: "warming".to_string(),
10478                    program: fake_aft_stub_path(),
10479                    args: Vec::new(),
10480                    env: Vec::new(),
10481                    reserved: false,
10482                    reserved_prefixes: Vec::new(),
10483                    protocol: ModuleProtocol::Subc,
10484                    overlap: Default::default(),
10485                },
10486                true,
10487            )
10488            .unwrap();
10489        assert_eq!(module.state().unwrap(), ModuleState::Running);
10490
10491        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10492        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
10493        let response = handler
10494            .handle_control_frame(
10495                &ctx,
10496                route_open_frame(304, "warming", unique_project_root("warming")),
10497            )
10498            .await
10499            .unwrap();
10500        module.stop().await.unwrap();
10501
10502        assert_eq!(response[0].header.ty, FrameType::Error);
10503        let error = parse_error(&response[0]);
10504        assert_eq!(error["code"], "module_warming");
10505        assert!(error["message"]
10506            .as_str()
10507            .unwrap()
10508            .contains("state=running, enabled=true, live=false"));
10509    }
10510
10511    #[test]
10512    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
10513        let handler = ControlHandler::new(Arc::new(Registry::default()));
10514        let capture = EventCapture::default();
10515        let _subscriber =
10516            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10517        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
10518        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
10519        let pending = (0..limit).collect::<Vec<_>>();
10520        let response = handler
10521            .route_open_capacity_refusal(
10522                &ctx,
10523                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
10524                "busy",
10525                pending.len(),
10526                limit,
10527            )
10528            .unwrap();
10529        assert_eq!(parse_error(&response)["code"], "target_unavailable");
10530        let event = capture
10531            .events()
10532            .into_iter()
10533            .find(|event| {
10534                event.target == "control"
10535                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
10536            })
10537            .expect("connection admission refusal event");
10538        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10539        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10540    }
10541
10542    #[test]
10543    fn route_open_target_cap_logs_admission_reason_and_capacity() {
10544        let handler = ControlHandler::new(Arc::new(Registry::default()));
10545        let capture = EventCapture::default();
10546        let _subscriber =
10547            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10548        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
10549        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
10550        let guards = (0..limit)
10551            .map(|_| {
10552                handler
10553                    .route_bind_concurrency
10554                    .try_admit("busy", limit)
10555                    .unwrap()
10556            })
10557            .collect::<Vec<_>>();
10558        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10559            Err(in_flight) => in_flight,
10560            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10561        };
10562        let response = handler
10563            .route_open_target_capacity_refusal(
10564                &ctx,
10565                &route_open_frame(397, "busy", unique_project_root("target-cap")),
10566                "busy",
10567                in_flight,
10568            )
10569            .unwrap();
10570        assert_eq!(parse_error(&response)["code"], "target_unavailable");
10571        let event = capture
10572            .events()
10573            .into_iter()
10574            .find(|event| {
10575                event.target == "control"
10576                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10577            })
10578            .expect("target admission refusal event");
10579        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10580        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10581        drop(guards);
10582    }
10583
10584    #[tokio::test(flavor = "current_thread")]
10585    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
10586        let handler = ControlHandler::new(Arc::new(Registry::default()));
10587        let capture = EventCapture::default();
10588        let _subscriber =
10589            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10590        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
10591        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10592
10593        let _ = handler
10594            .handle_client_control_request(
10595                &ctx,
10596                frame,
10597                ClientControlRequest::SupervisorSetEnabled {
10598                    module_id: "broca".to_string(),
10599                    enabled: false,
10600                },
10601            )
10602            .await
10603            .unwrap();
10604
10605        let events = capture
10606            .events()
10607            .into_iter()
10608            .filter(|event| {
10609                event.target == "control"
10610                    && event.fields.get("message").map(String::as_str)
10611                        == Some("supervisor request received")
10612            })
10613            .collect::<Vec<_>>();
10614        assert_eq!(events.len(), 1, "one supervisor request log line");
10615        let event = &events[0];
10616        assert_eq!(event.level, tracing::Level::INFO);
10617        assert_eq!(
10618            event.fields.get("op"),
10619            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10620        );
10621        assert_eq!(
10622            event.fields.get("module_id"),
10623            Some(&"\"broca\"".to_string())
10624        );
10625        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10626        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10627        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10628    }
10629
10630    #[tokio::test(flavor = "current_thread")]
10631    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10632        let registry = Arc::new(Registry::default());
10633        let module_connection = ConnectionId::new(303);
10634        registry
10635            .register_with_control_ops(
10636                manifest("aft", PROTOCOL_VERSION),
10637                PROTOCOL_VERSION,
10638                module_connection,
10639                module_baseline_control_ops(),
10640            )
10641            .unwrap();
10642        let handler = ControlHandler::new(registry);
10643        let capture = EventCapture::default();
10644        let _subscriber =
10645            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10646        let (ctx, _rx) = route_ctx(module_connection);
10647        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10648
10649        let _ = handler
10650            .handle_client_control_request(
10651                &ctx,
10652                frame,
10653                ClientControlRequest::SupervisorSetEnabled {
10654                    module_id: "broca".to_string(),
10655                    enabled: false,
10656                },
10657            )
10658            .await
10659            .unwrap();
10660
10661        let callers = capture
10662            .events()
10663            .into_iter()
10664            .filter(|event| {
10665                event.fields.get("message").map(String::as_str)
10666                    == Some("supervisor request received")
10667            })
10668            .map(|event| event.fields.get("caller").cloned())
10669            .collect::<Vec<_>>();
10670        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10671    }
10672
10673    #[tokio::test(flavor = "current_thread")]
10674    async fn supervisor_list_does_not_log_request_received() {
10675        let handler = ControlHandler::new(Arc::new(Registry::default()));
10676        let capture = EventCapture::default();
10677        let _subscriber =
10678            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10679        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10680        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10681
10682        handler
10683            .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10684            .await
10685            .unwrap();
10686
10687        assert!(capture.events().into_iter().all(|event| {
10688            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10689        }));
10690    }
10691
10692    #[tokio::test(flavor = "current_thread")]
10693    async fn supervisor_rescan_preview_does_not_log_request_received() {
10694        let handler = ControlHandler::new(Arc::new(Registry::default()));
10695        let capture = EventCapture::default();
10696        let _subscriber =
10697            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10698        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10699        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10700
10701        handler
10702            .handle_client_control_request(
10703                &ctx,
10704                frame,
10705                ClientControlRequest::SupervisorRescan { preview: true },
10706            )
10707            .await
10708            .unwrap();
10709
10710        assert!(capture.events().into_iter().all(|event| {
10711            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10712        }));
10713    }
10714
10715    /// One wire code has several senders, so the refusal line names the check
10716    /// that refused. This drives the shared refusal path for ordinary refusals
10717    /// with an unregistered
10718    /// target and requires the branch label on the event.
10719    #[tokio::test]
10720    async fn route_open_refusal_names_the_check_that_refused() {
10721        let handler = ControlHandler::new(Arc::new(Registry::default()));
10722        let capture = EventCapture::default();
10723        let _subscriber =
10724            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10725        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10726        let response = handler
10727            .handle_control_frame(
10728                &ctx,
10729                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10730            )
10731            .await
10732            .unwrap();
10733
10734        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10735        let event = capture
10736            .events()
10737            .into_iter()
10738            .find(|event| {
10739                event.target == "control"
10740                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10741            })
10742            .expect("route.open refusal event");
10743        assert_eq!(
10744            event.fields.get("reason"),
10745            Some(&"\"not_registered\"".to_string())
10746        );
10747    }
10748
10749    #[tokio::test]
10750    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10751        let registry = Arc::new(Registry::default());
10752        let supervisor_handle = SupervisorHandle::new();
10753        let supervisor =
10754            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
10755                .with_handle(supervisor_handle.clone())
10756                .with_connection_file_path(std::env::temp_dir().join(format!(
10757                    "subc-route-open-refusal-info-{}",
10758                    std::process::id()
10759                )));
10760        let module = supervisor
10761            .supervise_configured(
10762                ModuleSpec {
10763                    module_id: "warming".to_string(),
10764                    program: fake_aft_stub_path(),
10765                    args: Vec::new(),
10766                    env: Vec::new(),
10767                    reserved: false,
10768                    reserved_prefixes: Vec::new(),
10769                    protocol: ModuleProtocol::Subc,
10770                    overlap: Default::default(),
10771                },
10772                true,
10773            )
10774            .unwrap();
10775        assert_eq!(module.state().unwrap(), ModuleState::Running);
10776
10777        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10778        assert!(handler
10779            .counters()
10780            .snapshot()
10781            .get("route_open_refused_by_code")
10782            .is_none());
10783        let capture = EventCapture::default();
10784        let _subscriber =
10785            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10786        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
10787        let response = handler
10788            .handle_control_frame(
10789                &ctx,
10790                route_open_frame(394, "warming", unique_project_root("refusal-info")),
10791            )
10792            .await
10793            .unwrap();
10794        module.stop().await.unwrap();
10795
10796        assert_eq!(parse_error(&response[0])["code"], "module_warming");
10797        let event = capture
10798            .events()
10799            .into_iter()
10800            .find(|event| {
10801                event.target == "control"
10802                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
10803            })
10804            .expect("route.open refusal event");
10805        assert_eq!(
10806            event.fields.get("module_id"),
10807            Some(&"\"warming\"".to_string())
10808        );
10809        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10810        assert_eq!(
10811            event.fields.get("reason"),
10812            Some(&"\"supervised_not_registered\"".to_string())
10813        );
10814        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10815        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10816        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10817        assert_eq!(
10818            handler.counters().snapshot()["route_open_refused_by_code"],
10819            json!({ "module_warming": 1 })
10820        );
10821    }
10822
10823    const OUTAGE_START: &str = "route.open refusing module: not serving";
10824    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10825
10826    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10827        capture
10828            .events()
10829            .into_iter()
10830            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10831            .collect()
10832    }
10833
10834    fn supervise_stub(
10835        registry: &Arc<Registry>,
10836        module_id: &str,
10837        enabled: bool,
10838    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10839        let supervisor_handle = SupervisorHandle::new();
10840        let supervisor =
10841            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10842                .with_handle(supervisor_handle.clone())
10843                .with_connection_file_path(std::env::temp_dir().join(format!(
10844                    "subc-route-outage-{module_id}-{}",
10845                    std::process::id()
10846                )));
10847        let module = supervisor
10848            .supervise_configured(
10849                ModuleSpec {
10850                    module_id: module_id.to_string(),
10851                    program: fake_aft_stub_path(),
10852                    args: Vec::new(),
10853                    env: Vec::new(),
10854                    reserved: false,
10855                    reserved_prefixes: Vec::new(),
10856                    protocol: ModuleProtocol::Subc,
10857                    overlap: Default::default(),
10858                },
10859                enabled,
10860            )
10861            .unwrap();
10862        (supervisor_handle, module)
10863    }
10864
10865    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10866        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10867            module_id: module_id.to_string(),
10868            drain_timeout_ms: Some(50),
10869        })
10870        .unwrap();
10871        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10872    }
10873
10874    /// Two handlers built over one forwarding table must share one outage
10875    /// tracker; separate trackers would each log their own opening line for
10876    /// the same outage.
10877    #[test]
10878    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10879        let registry = Arc::new(Registry::default());
10880        let forwarding = Arc::new(ForwardingTable::default());
10881        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10882        let second = ControlHandler::with_forwarding(registry, forwarding);
10883        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10884    }
10885
10886    /// A client can name any module id it likes. Refusing an unknown one,
10887    /// however often, must not create outage state or outage lines, or the
10888    /// tracker would be a memory sink any client could fill.
10889    #[tokio::test(flavor = "current_thread")]
10890    async fn route_open_unknown_module_refusals_add_no_outage_state() {
10891        let handler = ControlHandler::new(Arc::new(Registry::default()));
10892        let capture = EventCapture::default();
10893        let _subscriber =
10894            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10895        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10896        for corr in 0..8 {
10897            let response = handler
10898                .handle_control_frame(
10899                    &ctx,
10900                    route_open_frame(
10901                        380 + corr,
10902                        &format!("nobody-{corr}"),
10903                        unique_project_root("outage-unknown"),
10904                    ),
10905                )
10906                .await
10907                .unwrap();
10908            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10909        }
10910
10911        assert_eq!(handler.route_outages.tracked_module_count(), 0);
10912        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10913        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10914    }
10915
10916    /// Drives the refusal path end to end: a supervised module that served
10917    /// before and stopped being registered with no instruction to stop is a
10918    /// WARN, and the same module refused after an operator `supervisor.restart`
10919    /// is an INFO.
10920    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10921    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10922        let registry = Arc::new(Registry::default());
10923        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
10924        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10925        let capture = EventCapture::default();
10926        let _subscriber =
10927            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10928        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10929        // The stub never registers, so pretend it served once: otherwise every
10930        // refusal would fall in its startup window.
10931        handler.route_outages.record_accepted("outage-restart");
10932
10933        let response = handler
10934            .handle_control_frame(
10935                &ctx,
10936                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10937            )
10938            .await
10939            .unwrap();
10940        assert_eq!(response[0].header.ty, FrameType::Error);
10941        let starts = outage_lines(&capture, OUTAGE_START);
10942        assert_eq!(starts.len(), 1, "{starts:?}");
10943        assert_eq!(starts[0].level, tracing::Level::WARN);
10944        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10945        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10946        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10947        handler.route_outages.record_accepted("outage-restart");
10948        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10949
10950        let restart = handler
10951            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10952            .await
10953            .unwrap();
10954        assert_eq!(
10955            restart[0].header.ty,
10956            FrameType::Response,
10957            "{:?}",
10958            parse_error(&restart[0])
10959        );
10960        handler
10961            .handle_control_frame(
10962                &ctx,
10963                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10964            )
10965            .await
10966            .unwrap();
10967        module.stop().await.unwrap();
10968
10969        let starts = outage_lines(&capture, OUTAGE_START);
10970        assert_eq!(starts.len(), 2, "{starts:?}");
10971        assert_eq!(starts[1].level, tracing::Level::INFO);
10972        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10973    }
10974
10975    /// A restart refused before it touched the module (here: the module is
10976    /// disabled) must clear its operator mark, so the next real outage is
10977    /// still reported as a warning.
10978    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10979    async fn failed_operator_restart_leaves_no_operator_mark() {
10980        let registry = Arc::new(Registry::default());
10981        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
10982        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10983        let capture = EventCapture::default();
10984        let _subscriber =
10985            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10986        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10987        handler.route_outages.record_accepted("outage-disabled");
10988
10989        let restart = handler
10990            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10991            .await
10992            .unwrap();
10993        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10994        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10995
10996        handler
10997            .handle_control_frame(
10998                &ctx,
10999                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11000            )
11001            .await
11002            .unwrap();
11003        let starts = outage_lines(&capture, OUTAGE_START);
11004        assert_eq!(starts.len(), 1, "{starts:?}");
11005        assert_eq!(starts[0].level, tracing::Level::WARN);
11006    }
11007
11008    #[tokio::test(flavor = "current_thread")]
11009    async fn route_open_unknown_module_escapes_target_module_id() {
11010        let handler = ControlHandler::new(Arc::new(Registry::default()));
11011        let capture = EventCapture::default();
11012        let _subscriber =
11013            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11014        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11015        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11016        let response = handler
11017            .handle_control_frame(
11018                &ctx,
11019                route_open_frame(
11020                    395,
11021                    hostile_module_id,
11022                    unique_project_root("hostile-target-module-id"),
11023                ),
11024            )
11025            .await
11026            .unwrap();
11027
11028        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11029        let event = capture
11030            .events()
11031            .into_iter()
11032            .find(|event| {
11033                event.target == "control"
11034                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11035            })
11036            .expect("route.open unknown-module refusal event");
11037        let logged = event.fields.get("module_id").expect("module_id field");
11038        assert!(!logged.bytes().any(|byte| byte < 0x20));
11039        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11040    }
11041
11042    #[tokio::test(flavor = "current_thread")]
11043    async fn route_open_module_rejection_uses_daemon_counter_key() {
11044        let registry = Arc::new(Registry::default());
11045        let forwarding = Arc::new(ForwardingTable::default());
11046        let handler =
11047            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11048        let module_connection = ConnectionId::new(95);
11049        let (module_ctx, mut module_rx) = route_ctx(module_connection);
11050        hello_via_sink(
11051            &handler,
11052            &module_ctx,
11053            &mut module_rx,
11054            hello_frame("aft", PROTOCOL_VERSION, 395),
11055        )
11056        .await;
11057
11058        let client_connection = ConnectionId::new(96);
11059        let (client_ctx, _client_rx) = route_ctx(client_connection);
11060        let capture = EventCapture::default();
11061        let _subscriber =
11062            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11063        let (route_task, bind) = relay_route_open(
11064            &handler,
11065            client_connection,
11066            &client_ctx.egress,
11067            &mut module_rx,
11068            396,
11069            "aft",
11070            "hostile-module-code",
11071        )
11072        .await;
11073        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
11074        let rejection = Frame::build(
11075            FrameType::Error,
11076            control_flags(),
11077            0,
11078            0,
11079            bind.header.corr,
11080            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
11081        )
11082        .unwrap();
11083        handler
11084            .handle_control_frame(&module_ctx, rejection)
11085            .await
11086            .unwrap();
11087
11088        let response = route_task.await.unwrap();
11089        assert_eq!(parse_error(&response[0])["code"], hostile_code);
11090        let counters = handler.counters().snapshot();
11091        assert_eq!(
11092            counters["route_open_refused_by_code"],
11093            json!({ "module_rejected": 1 })
11094        );
11095        assert!(counters["route_open_refused_by_code"]
11096            .get(hostile_code)
11097            .is_none());
11098
11099        let event = capture
11100            .events()
11101            .into_iter()
11102            .find(|event| {
11103                event.target == "control"
11104                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
11105            })
11106            .expect("route.open module-rejection refusal event");
11107        let logged = event.fields.get("module_code").expect("module_code field");
11108        assert!(!logged.bytes().any(|byte| byte < 0x20));
11109        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11110    }
11111
11112    #[tokio::test]
11113    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
11114        let registry = Arc::new(Registry::default());
11115        let supervisor_handle = SupervisorHandle::new();
11116        let missing_program = std::env::temp_dir().join(format!(
11117            "subc-route-open-missing-program-{}",
11118            std::process::id()
11119        ));
11120        let supervisor =
11121            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
11122                .with_handle(supervisor_handle.clone());
11123        let module = supervisor
11124            .supervise_configured(
11125                ModuleSpec {
11126                    module_id: "failed".to_string(),
11127                    program: missing_program,
11128                    args: Vec::new(),
11129                    env: Vec::new(),
11130                    reserved: false,
11131                    reserved_prefixes: Vec::new(),
11132                    protocol: ModuleProtocol::Subc,
11133                    overlap: Default::default(),
11134                },
11135                true,
11136            )
11137            .unwrap();
11138        assert_eq!(module.state().unwrap(), ModuleState::Failed);
11139
11140        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11141        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
11142        let response = handler
11143            .handle_control_frame(
11144                &ctx,
11145                route_open_frame(305, "failed", unique_project_root("failed")),
11146            )
11147            .await
11148            .unwrap();
11149
11150        assert_eq!(response[0].header.ty, FrameType::Error);
11151        let error = parse_error(&response[0]);
11152        assert_eq!(error["code"], "target_unavailable");
11153        assert!(error["message"]
11154            .as_str()
11155            .unwrap()
11156            .contains("state=failed, enabled=true, live=false"));
11157    }
11158
11159    #[tokio::test]
11160    async fn route_open_role_mismatch_remains_target_unavailable() {
11161        let registry = Arc::new(Registry::default());
11162        let handler = ControlHandler::new(Arc::clone(&registry));
11163        handler
11164            .handle_control(
11165                ConnectionId::new(41),
11166                non_routable_hello_frame_with_control_ops("health-only", 306, None),
11167            )
11168            .unwrap();
11169
11170        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
11171        let response = handler
11172            .handle_control_frame(
11173                &ctx,
11174                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
11175            )
11176            .await
11177            .unwrap();
11178
11179        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11180        assert!(parse_error(&response[0])["message"]
11181            .as_str()
11182            .unwrap()
11183            .contains("does not provide the requested target"));
11184    }
11185
11186    #[tokio::test]
11187    async fn route_open_inactive_registration_remains_target_unavailable() {
11188        let registry = Arc::new(Registry::default());
11189        let handler = ControlHandler::new(Arc::clone(&registry));
11190        handler
11191            .handle_control(
11192                ConnectionId::new(43),
11193                hello_frame("inactive", PROTOCOL_VERSION, 308),
11194            )
11195            .unwrap();
11196        assert!(registry
11197            .set_module_state_for_test("inactive", ChannelState::Closed)
11198            .unwrap());
11199
11200        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
11201        let response = handler
11202            .handle_control_frame(
11203                &ctx,
11204                route_open_frame(309, "inactive", unique_project_root("inactive")),
11205            )
11206            .await
11207            .unwrap();
11208
11209        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11210        assert!(parse_error(&response[0])["message"]
11211            .as_str()
11212            .unwrap()
11213            .contains("is not active"));
11214    }
11215
11216    #[tokio::test]
11217    async fn late_health_reply_is_recorded_through_the_module_response_path() {
11218        let registry = Arc::new(Registry::default());
11219        let forwarding = Arc::new(ForwardingTable::default());
11220        let supervisor_handle = SupervisorHandle::new();
11221        let supervisor =
11222            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
11223                .with_forwarding(Arc::clone(&forwarding))
11224                .with_handle(supervisor_handle.clone());
11225        let module = supervisor
11226            .supervise_configured(
11227                crate::ModuleSpec {
11228                    module_id: "late-health-response".to_string(),
11229                    program: PathBuf::from("disabled-module"),
11230                    args: Vec::new(),
11231                    env: Vec::new(),
11232                    reserved: false,
11233                    reserved_prefixes: Vec::new(),
11234                    protocol: ModuleProtocol::Subc,
11235                    overlap: Default::default(),
11236                },
11237                false,
11238            )
11239            .unwrap();
11240        let handler =
11241            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11242                .with_supervisor(supervisor_handle);
11243        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
11244        handler
11245            .handle_control_frame(
11246                &module_ctx,
11247                hello_frame_with_control_ops(
11248                    "late-health-response",
11249                    PROTOCOL_VERSION,
11250                    7,
11251                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11252                ),
11253            )
11254            .await
11255            .unwrap();
11256        let probe_started_at = Instant::now() - Duration::from_millis(80);
11257        let pending = forwarding
11258            .begin_health_probe_rpc_for(
11259                "late-health-response",
11260                MODULE_CONTROL_OP_HEALTH_CHECK,
11261                probe_started_at,
11262                Instant::now() - Duration::from_millis(1),
11263            )
11264            .unwrap();
11265        assert!(forwarding
11266            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
11267            .unwrap());
11268
11269        let responses = handler
11270            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
11271            .await
11272            .unwrap();
11273
11274        assert!(responses.is_empty());
11275        let health = module.status().unwrap().health;
11276        assert_eq!(health.late_answer_count, 1);
11277        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
11278    }
11279
11280    #[tokio::test]
11281    async fn health_probe_timeout_and_module_death_are_typed() {
11282        let registry = Arc::new(Registry::default());
11283        let forwarding = Arc::new(ForwardingTable::default());
11284        let handler =
11285            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11286                .with_health_probe_timeout(Duration::from_millis(50));
11287        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11288        hello_via_sink(
11289            &handler,
11290            &module_ctx,
11291            &mut module_rx,
11292            hello_frame_with_control_ops(
11293                "aft",
11294                PROTOCOL_VERSION,
11295                7,
11296                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11297            ),
11298        )
11299        .await;
11300
11301        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11302        let responses = handler
11303            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
11304            .await
11305            .unwrap();
11306        assert_eq!(responses[0].header.ty, FrameType::Error);
11307        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
11308        let _ = module_rx.try_recv();
11309
11310        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
11311        let health_handler = handler.clone();
11312        let death_task = tokio::spawn(async move {
11313            health_handler
11314                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
11315                .await
11316                .unwrap()
11317        });
11318        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11319            .await
11320            .unwrap()
11321            .unwrap();
11322        handler
11323            .cleanup_connection(module_ctx.connection_id)
11324            .unwrap();
11325        let responses = death_task.await.unwrap();
11326        assert_eq!(responses[0].header.ty, FrameType::Error);
11327        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
11328    }
11329
11330    #[test]
11331    fn hello_requires_exact_protocol_version() {
11332        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
11333            let registry = Arc::new(Registry::default());
11334            let handler = ControlHandler::new(Arc::clone(&registry));
11335            let responses = handler
11336                .handle_control(
11337                    ConnectionId::new(connection),
11338                    hello_frame("aft", offered, 9),
11339                )
11340                .unwrap();
11341
11342            assert_eq!(responses.len(), 1);
11343            assert_eq!(responses[0].header.ty, FrameType::Error);
11344            let error = parse_error(&responses[0]);
11345            assert_eq!(error["code"], "version_unsupported");
11346            assert!(registry.get_module("aft").unwrap().is_none());
11347            assert_eq!(registry.active_registration_count().unwrap(), 0);
11348        }
11349    }
11350
11351    #[test]
11352    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
11353        let registry = Arc::new(Registry::default());
11354        let forwarding = Arc::new(ForwardingTable::default());
11355        let handler =
11356            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11357        let module_connection = ConnectionId::new(301);
11358        let registration = registry
11359            .register_with_control_ops(
11360                manifest("aft-push", PROTOCOL_VERSION),
11361                PROTOCOL_VERSION,
11362                module_connection,
11363                module_baseline_control_ops(),
11364            )
11365            .unwrap();
11366        let (module_tx, _module_rx) = mpsc::channel(8);
11367        let endpoint = forwarding
11368            .register_module_connection(
11369                module_connection,
11370                "aft-push".to_string(),
11371                PROTOCOL_VERSION,
11372                manifest_concurrency(&registration.manifest),
11373                FrameSink::new(module_tx),
11374            )
11375            .unwrap();
11376
11377        // A push op this version does not know is ignored (forward-compat), not errored.
11378        let unknown = Frame::build(
11379            FrameType::Push,
11380            control_flags(),
11381            0,
11382            0,
11383            5,
11384            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
11385        )
11386        .unwrap();
11387        let out = handler.handle_status_update(endpoint, unknown).unwrap();
11388        assert!(
11389            out.is_empty(),
11390            "unknown push op must be ignored, got {out:?}"
11391        );
11392
11393        // A malformed body for a KNOWN op is a real error worth surfacing.
11394        let malformed = Frame::build(
11395            FrameType::Push,
11396            control_flags(),
11397            0,
11398            0,
11399            6,
11400            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
11401        )
11402        .unwrap();
11403        let out = handler.handle_status_update(endpoint, malformed).unwrap();
11404        assert_eq!(out.len(), 1);
11405        assert_eq!(out[0].header.ty, FrameType::Error);
11406        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
11407    }
11408
11409    #[test]
11410    fn hello_rejected_when_connection_already_owns_client_routes() {
11411        let registry = Arc::new(Registry::default());
11412        let forwarding = Arc::new(ForwardingTable::default());
11413        let handler =
11414            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11415        // Commits a client route on connection 202 (bound to a module on conn 101).
11416        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
11417        let client_connection = ConnectionId::new(202);
11418
11419        // That same connection now tries to register as a module: rejected, so one
11420        // connection never holds both client-route and module-endpoint state.
11421        let responses = handler
11422            .handle_control(
11423                client_connection,
11424                hello_frame("aft-second", PROTOCOL_VERSION, 9),
11425            )
11426            .unwrap();
11427        assert_eq!(responses[0].header.ty, FrameType::Error);
11428        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
11429        assert!(registry.get_module("aft-second").unwrap().is_none());
11430    }
11431
11432    #[tokio::test]
11433    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
11434        let registry = Arc::new(Registry::default());
11435        let forwarding = Arc::new(ForwardingTable::default());
11436        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
11437        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
11438        hello_via_sink(
11439            &handler,
11440            &module_ctx,
11441            &mut module_rx,
11442            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
11443        )
11444        .await;
11445        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
11446        let pending = forwarding
11447            .begin_route_bind_relay_for_test(
11448                client_ctx.connection_id,
11449                client_ctx.egress.clone(),
11450                2,
11451                "alpha",
11452            )
11453            .unwrap();
11454        forwarding
11455            .complete_pending_relay(
11456                module_ctx.connection_id,
11457                pending.corr,
11458                RouteBindRelayOutcome::Accepted,
11459            )
11460            .unwrap();
11461        client_rx.try_recv().unwrap();
11462        for module_id in ["beta", "alpha"] {
11463            let replies = handler
11464                .handle_control_frame(
11465                    &module_ctx,
11466                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
11467                )
11468                .await
11469                .unwrap();
11470            assert_eq!(replies.len(), 1, "second HELLO must be refused");
11471            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
11472        }
11473        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
11474        assert!(registry.get_module("beta").unwrap().is_none());
11475        assert!(matches!(
11476            forwarding
11477                .lookup_data_route(
11478                    client_ctx.connection_id,
11479                    pending.client_channel,
11480                    pending.client_epoch,
11481                )
11482                .unwrap(),
11483            DataRoute::Client(DataRouteState::Bound(_))
11484        ));
11485        assert!(handler
11486            .hello_launch_nonces
11487            .lock()
11488            .unwrap()
11489            .presented(module_ctx.connection_id, Some("alpha-nonce")));
11490        assert!(module_rx.try_recv().is_err());
11491    }
11492
11493    #[test]
11494    fn reserved_module_hello_requires_matching_launch_nonce() {
11495        let registry = Arc::new(Registry::default());
11496        let supervisor = SupervisorHandle::new();
11497        // The supervisor recorded the nonce it injected when it spawned the reserved
11498        // module; the HELLO verifier checks against the same shared handle.
11499        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11500        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
11501
11502        // A HELLO with NO nonce is rejected.
11503        let no_nonce = handler
11504            .handle_control(
11505                ConnectionId::new(1),
11506                hello_frame("vault", PROTOCOL_VERSION, 1),
11507            )
11508            .unwrap();
11509        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
11510        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
11511        assert!(registry.get_module("vault").unwrap().is_none());
11512
11513        // A HELLO with the WRONG nonce is rejected.
11514        let wrong = handler
11515            .handle_control(
11516                ConnectionId::new(2),
11517                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
11518            )
11519            .unwrap();
11520        assert_eq!(wrong[0].header.ty, FrameType::Error);
11521        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
11522        assert!(registry.get_module("vault").unwrap().is_none());
11523
11524        // A HELLO with the CORRECT nonce registers.
11525        let ok = handler
11526            .handle_control(
11527                ConnectionId::new(3),
11528                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
11529            )
11530            .unwrap();
11531        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
11532        assert!(registry.get_module("vault").unwrap().is_some());
11533    }
11534
11535    #[test]
11536    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
11537        let registry = Arc::new(Registry::default());
11538        let supervisor = SupervisorHandle::new();
11539        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11540        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11541        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
11542
11543        let squat = handler
11544            .handle_control(
11545                ConnectionId::new(1),
11546                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
11547            )
11548            .unwrap();
11549        assert_eq!(squat[0].header.ty, FrameType::Error);
11550        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
11551        assert!(parse_error(&squat[0])["message"]
11552            .as_str()
11553            .unwrap()
11554            .contains("fed:"));
11555
11556        let accepted_peer = handler
11557            .handle_control(
11558                ConnectionId::new(2),
11559                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
11560            )
11561            .unwrap();
11562        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
11563
11564        let accepted_short = handler
11565            .handle_control(
11566                ConnectionId::new(3),
11567                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
11568            )
11569            .unwrap();
11570        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
11571
11572        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
11573            let response = handler
11574                .handle_control(
11575                    ConnectionId::new(conn),
11576                    hello_frame(module_id, PROTOCOL_VERSION, conn),
11577                )
11578                .unwrap();
11579            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
11580        }
11581    }
11582
11583    #[test]
11584    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
11585        let registry = Arc::new(Registry::default());
11586        let supervisor = SupervisorHandle::new();
11587        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11588        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11589        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
11590        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
11591
11592        let owner_nonce = handler
11593            .handle_control(
11594                ConnectionId::new(1),
11595                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
11596            )
11597            .unwrap();
11598        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11599        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11600        assert!(registry.get_module("fed:special").unwrap().is_none());
11601
11602        let exact_nonce = handler
11603            .handle_control(
11604                ConnectionId::new(2),
11605                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11606            )
11607            .unwrap();
11608        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11609        assert!(registry.get_module("fed:special").unwrap().is_some());
11610    }
11611
11612    #[test]
11613    fn non_reserved_module_ignores_launch_nonce() {
11614        let registry = Arc::new(Registry::default());
11615        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
11616        // registration succeeds whether a spawned process echoes a nonce or not.
11617        let handler = ControlHandler::new(Arc::clone(&registry));
11618        let no_nonce = handler
11619            .handle_control(
11620                ConnectionId::new(1),
11621                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11622            )
11623            .unwrap();
11624        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11625        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11626
11627        let echoed_nonce = handler
11628            .handle_control(
11629                ConnectionId::new(2),
11630                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11631            )
11632            .unwrap();
11633        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11634        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11635    }
11636
11637    #[test]
11638    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11639        let handler = ControlHandler::default();
11640        let conn = ConnectionId::new(1);
11641        let malformed = Frame::build(
11642            FrameType::Hello,
11643            control_flags(),
11644            0,
11645            0,
11646            3,
11647            b"{not json".to_vec(),
11648        )
11649        .unwrap();
11650
11651        let error = handler.handle_control(conn, malformed).unwrap();
11652        assert_eq!(error[0].header.ty, FrameType::Error);
11653        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11654
11655        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11656        let pong = handler.handle_control(conn, ping).unwrap();
11657        assert_eq!(pong[0].header.ty, FrameType::Pong);
11658        assert_eq!(pong[0].header.corr, 4);
11659    }
11660
11661    #[test]
11662    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11663        let registry = Arc::new(Registry::default());
11664        let handler = ControlHandler::new(Arc::clone(&registry));
11665
11666        handler
11667            .handle_control(
11668                ConnectionId::new(1),
11669                hello_frame("aft", PROTOCOL_VERSION, 1),
11670            )
11671            .unwrap();
11672        let duplicate = handler
11673            .handle_control(
11674                ConnectionId::new(2),
11675                hello_frame("aft", PROTOCOL_VERSION, 2),
11676            )
11677            .unwrap();
11678
11679        assert_eq!(duplicate[0].header.ty, FrameType::Error);
11680        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11681        let registration = registry.get_module("aft").unwrap().unwrap();
11682        assert_eq!(registration.connection_id, ConnectionId::new(1));
11683    }
11684
11685    #[test]
11686    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11687        let registry = Arc::new(Registry::default());
11688        let forwarding = Arc::new(ForwardingTable::default());
11689        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11690        let handler =
11691            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11692                .with_process_liveness(process_liveness);
11693        let (ctx, route_channel, route_epoch) =
11694            bind_liveness_route(&registry, &forwarding, "aft-dead");
11695        let responses = handler
11696            .handle_route_poll(
11697                &ctx,
11698                route_poll_frame(41, PollKind::Liveness, route_channel),
11699                route_channel,
11700                route_epoch,
11701                PollKind::Liveness,
11702            )
11703            .unwrap();
11704
11705        assert_eq!(responses.len(), 1);
11706        assert_eq!(responses[0].header.ty, FrameType::Response);
11707        assert_route_poll_liveness(&responses[0], false);
11708    }
11709
11710    #[test]
11711    fn liveness_poll_without_process_source_uses_bound_route() {
11712        let registry = Arc::new(Registry::default());
11713        let forwarding = Arc::new(ForwardingTable::default());
11714        let handler =
11715            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11716        let (ctx, route_channel, route_epoch) =
11717            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
11718        let responses = handler
11719            .handle_route_poll(
11720                &ctx,
11721                route_poll_frame(42, PollKind::Liveness, route_channel),
11722                route_channel,
11723                route_epoch,
11724                PollKind::Liveness,
11725            )
11726            .unwrap();
11727
11728        assert_route_poll_liveness(&responses[0], true);
11729    }
11730
11731    #[test]
11732    fn liveness_poll_untracked_process_source_uses_bound_route() {
11733        let registry = Arc::new(Registry::default());
11734        let forwarding = Arc::new(ForwardingTable::default());
11735        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11736        let handler =
11737            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11738                .with_process_liveness(process_liveness);
11739        let (ctx, route_channel, route_epoch) =
11740            bind_liveness_route(&registry, &forwarding, "aft-untracked");
11741        let responses = handler
11742            .handle_route_poll(
11743                &ctx,
11744                route_poll_frame(43, PollKind::Liveness, route_channel),
11745                route_channel,
11746                route_epoch,
11747                PollKind::Liveness,
11748            )
11749            .unwrap();
11750
11751        assert_route_poll_liveness(&responses[0], true);
11752    }
11753
11754    #[tokio::test]
11755    async fn unknown_op_returns_unknown_control_op() {
11756        let handler = ControlHandler::default();
11757        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
11758        let request = Frame::build(
11759            FrameType::Request,
11760            control_flags(),
11761            0,
11762            0,
11763            55,
11764            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
11765        )
11766        .unwrap();
11767
11768        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11769
11770        assert_eq!(response.len(), 1);
11771        assert_eq!(response[0].header.ty, FrameType::Error);
11772        assert_eq!(response[0].header.corr, 55);
11773        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
11774    }
11775
11776    #[tokio::test]
11777    async fn supervisor_provenance_rejects_unknown_exact_module() {
11778        let handler = ControlHandler::default();
11779        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
11780        let request = Frame::build(
11781            FrameType::Request,
11782            control_flags(),
11783            0,
11784            0,
11785            57,
11786            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
11787        )
11788        .unwrap();
11789
11790        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11791
11792        assert_eq!(response.len(), 1);
11793        assert_eq!(response[0].header.ty, FrameType::Error);
11794        assert_eq!(response[0].header.corr, 57);
11795        let error = parse_error(&response[0]);
11796        assert_eq!(error["code"], "unknown_module");
11797        assert_eq!(error["message"], "module_id 'missing' is not supervised");
11798    }
11799
11800    #[test]
11801    fn provenance_probe_override_keeps_handler_tests_deterministic() {
11802        let expected = subc_control::RunningImageAgreement::Unavailable {
11803            reason: subc_control::RunningImageUnavailableReason::HashFailed,
11804        };
11805        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
11806        assert_eq!(handler.provenance_probe_override, Some(expected));
11807    }
11808
11809    #[test]
11810    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11811        let verdict = reload_verdict(
11812            std::path::Path::new("/bin/new"),
11813            Some(std::path::Path::new("/bin/old")),
11814            subc_control::RunningImageAgreement::Unavailable {
11815                reason: subc_control::RunningImageUnavailableReason::HashFailed,
11816            },
11817        );
11818        assert!(matches!(
11819            verdict.path,
11820            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11821                if configured == std::path::Path::new("/bin/new")
11822                    && spawned_from == std::path::Path::new("/bin/old")
11823        ));
11824    }
11825
11826    #[cfg(windows)]
11827    #[test]
11828    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
11829        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
11830        let configured = dir.join("module.exe");
11831        std::fs::write(&configured, b"executable path fixture").unwrap();
11832        let spawned = std::fs::canonicalize(&configured).unwrap();
11833        assert_ne!(
11834            configured, spawned,
11835            "fixture must exercise different path spellings"
11836        );
11837        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
11838        let lower_case = configured.to_string_lossy().to_lowercase();
11839        for spelling in [
11840            &configured,
11841            Path::new(&forward_slashes),
11842            Path::new(&lower_case),
11843        ] {
11844            let verdict = reload_verdict(
11845                spelling,
11846                Some(&spawned),
11847                subc_control::RunningImageAgreement::Unavailable {
11848                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11849                },
11850            );
11851            assert_eq!(
11852                verdict.path,
11853                ReloadPathAgreement::Match,
11854                "configured {spelling:?}, spawned {spawned:?}"
11855            );
11856        }
11857    }
11858
11859    #[cfg(windows)]
11860    #[test]
11861    fn windows_reload_path_keeps_different_existing_program_pending() {
11862        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
11863        let old = dir.join("old.exe");
11864        let new = dir.join("new.exe");
11865        std::fs::write(&old, b"same executable content").unwrap();
11866        std::fs::copy(&old, &new).unwrap();
11867        let spawned = std::fs::canonicalize(&old).unwrap();
11868        let image = subc_control::RunningImageAgreement::Unavailable {
11869            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11870        };
11871        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
11872        assert!(
11873            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
11874            if configured == new && spawned_from == spawned)
11875        );
11876        let missing = dir.join("missing.exe");
11877        assert!(matches!(
11878            reload_verdict(&missing, Some(&spawned), image).path,
11879            ReloadPathAgreement::Mismatch { .. }
11880        ));
11881    }
11882
11883    #[test]
11884    fn reload_verdict_detects_replaced_image_at_same_path() {
11885        let image = subc_control::RunningImageAgreement::Mismatch {
11886            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11887                digest: "old".into(),
11888            },
11889            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11890                digest: "new".into(),
11891            },
11892        };
11893        let verdict = reload_verdict(
11894            std::path::Path::new("/bin/same"),
11895            Some(std::path::Path::new("/bin/same")),
11896            image.clone(),
11897        );
11898        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11899        assert_eq!(verdict.image, image);
11900    }
11901
11902    #[test]
11903    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11904        let image = subc_control::RunningImageAgreement::Unavailable {
11905            reason: subc_control::RunningImageUnavailableReason::NotRunning,
11906        };
11907        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11908        assert_eq!(
11909            verdict.path,
11910            subc_control::ReloadPathAgreement::Unavailable {
11911                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11912            }
11913        );
11914        assert_eq!(verdict.image, image);
11915
11916        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11917            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11918        };
11919        let verdict = reload_verdict(
11920            std::path::Path::new("/bin/same"),
11921            Some(std::path::Path::new("/bin/same")),
11922            unconfirmed.clone(),
11923        );
11924        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11925        assert_eq!(verdict.image, unconfirmed);
11926    }
11927
11928    #[test]
11929    fn reload_verdict_preserves_each_image_unavailability_reason() {
11930        use subc_control::RunningImageUnavailableReason as Reason;
11931
11932        for reason in [
11933            Reason::NotRunning,
11934            Reason::UnsupportedPlatform,
11935            Reason::RunningExecutableUnreadable,
11936            Reason::SpawnedPathUnreadable,
11937            Reason::HashFailed,
11938            Reason::ProcessIdentityUnconfirmed,
11939            Reason::Unknown("future_probe_reason".to_string()),
11940        ] {
11941            let image = subc_control::RunningImageAgreement::Unavailable {
11942                reason: reason.clone(),
11943            };
11944            let verdict = reload_verdict(
11945                std::path::Path::new("/bin/same"),
11946                Some(std::path::Path::new("/bin/same")),
11947                image.clone(),
11948            );
11949            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11950            assert_eq!(verdict.image, image, "{reason:?}");
11951        }
11952    }
11953
11954    #[tokio::test]
11955    async fn malformed_control_bodies_return_invalid_control_body() {
11956        let handler = ControlHandler::default();
11957        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11958
11959        for (corr, body) in [
11960            (56, br#"{"route_channel":1}"#.as_slice()),
11961            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11962            (
11963                58,
11964                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11965            ),
11966        ] {
11967            let request = Frame::build(
11968                FrameType::Request,
11969                control_flags(),
11970                0,
11971                0,
11972                corr,
11973                body.to_vec(),
11974            )
11975            .unwrap();
11976            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11977
11978            assert_eq!(response.len(), 1);
11979            assert_eq!(response[0].header.ty, FrameType::Error);
11980            assert_eq!(response[0].header.corr, corr);
11981            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11982        }
11983    }
11984
11985    #[tokio::test]
11986    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11987        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11988        let registry = Arc::new(Registry::default());
11989        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
11990        let router = Router::with_control_handler(Arc::clone(&control));
11991        let connection = router.begin_connection();
11992        let (ctx, mut rx) = route_ctx(connection.id());
11993
11994        router
11995            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11996            .await
11997            .unwrap();
11998        let response = rx.recv().await.unwrap();
11999        let ack = parse_ack(&response);
12000        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12001        let channel = 1;
12002
12003        let goodbye =
12004            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12005        router.route_for_connection(&ctx, goodbye).await.unwrap();
12006        assert!(rx.try_recv().is_err());
12007        assert!(registry.get_module("aft").unwrap().is_none());
12008
12009        router
12010            .route_for_connection(&ctx, channel_request(channel, 13))
12011            .await
12012            .unwrap();
12013        let error_frame = rx.recv().await.unwrap();
12014        assert_eq!(error_frame.header.ty, FrameType::Error);
12015        assert_eq!(error_frame.header.channel, channel);
12016        let captured = crate::router::test_log::captured_logs(&logs);
12017        assert_eq!(
12018            captured
12019                .lines()
12020                .filter(|line| {
12021                    line.contains("module_id=aft")
12022                        && line.contains("reason=explicit_goodbye")
12023                        && line.contains("module registration ended")
12024                })
12025                .count(),
12026            1,
12027            "unexpected GOODBYE registry log: {captured}"
12028        );
12029    }
12030
12031    #[tokio::test]
12032    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
12033        let registry = Arc::new(Registry::default());
12034        let handler = ControlHandler::new(registry).with_capability_config(
12035            [("prov".to_string(), true), ("cons".to_string(), true)],
12036            BTreeMap::new(),
12037        );
12038        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
12039        register_capability_manifest(
12040            &handler,
12041            &provider_ctx,
12042            &mut provider_rx,
12043            capability_manifest("prov", &["thing/v1"], &[]),
12044            1,
12045        )
12046        .await;
12047        let mut consumer = capability_manifest("cons", &[], &[]);
12048        consumer.capabilities.as_mut().unwrap().requires.push(
12049            subc_protocol::manifest::CapabilityRequirement {
12050                capability: "thing/v1".to_string(),
12051                need: subc_protocol::manifest::CapabilityNeed::Required,
12052            },
12053        );
12054        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
12055        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
12056        assert_eq!(
12057            handler.capability_evaluator.verdict("cons", "thing/v1"),
12058            Some(CapabilityVerdict::Provided)
12059        );
12060        let (mut client_rx, _) = open_route_for_capability_test(
12061            &handler,
12062            &provider_ctx,
12063            &mut provider_rx,
12064            703,
12065            3,
12066            "prov",
12067            None,
12068        )
12069        .await;
12070        let goodbye =
12071            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12072        handler
12073            .handle_control_frame(&provider_ctx, goodbye)
12074            .await
12075            .unwrap();
12076        assert_eq!(
12077            handler.capability_evaluator.verdict("cons", "thing/v1"),
12078            Some(CapabilityVerdict::NeverProvided)
12079        );
12080        let closed = client_rx
12081            .try_recv()
12082            .expect("GOODBYE pushes route.closed before route GOODBYE");
12083        assert!(
12084            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
12085            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
12086        );
12087        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
12088        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12089    }
12090
12091    struct RouteClosedFixture {
12092        handler: ControlHandler,
12093        #[cfg(unix)]
12094        supervisor: Supervisor,
12095        module_ctx: RouteCtx,
12096        #[cfg(unix)]
12097        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12098        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12099        route: Arc<crate::forwarding::RouteBinding>,
12100    }
12101
12102    async fn route_closed_fixture() -> RouteClosedFixture {
12103        let registry = Arc::new(Registry::default());
12104        let handler = ControlHandler::new(Arc::clone(&registry));
12105        #[cfg(unix)]
12106        let supervisor = Supervisor::new(registry, RestartPolicy::default())
12107            .with_forwarding(Arc::clone(&handler.forwarding));
12108        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
12109        register_capability_manifest(
12110            &handler,
12111            &module_ctx,
12112            &mut module_rx,
12113            capability_manifest("prov", &[], &[]),
12114            1,
12115        )
12116        .await;
12117        let (client_rx, bind) = open_route_for_capability_test(
12118            &handler,
12119            &module_ctx,
12120            &mut module_rx,
12121            702,
12122            2,
12123            "prov",
12124            None,
12125        )
12126        .await;
12127        let ModuleControlRequest::RouteBind {
12128            route_channel: channel,
12129            epoch: route_epoch,
12130            ..
12131        } = bind
12132        else {
12133            panic!("expected route.bind");
12134        };
12135        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
12136            handler
12137                .forwarding
12138                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
12139                .unwrap()
12140        else {
12141            panic!("the acknowledged route must be bound");
12142        };
12143        RouteClosedFixture {
12144            handler,
12145            #[cfg(unix)]
12146            supervisor,
12147            module_ctx,
12148            #[cfg(unix)]
12149            module_rx,
12150            client_rx,
12151            route,
12152        }
12153    }
12154
12155    impl RouteClosedFixture {
12156        #[cfg(unix)]
12157        async fn cut(&mut self) {
12158            let notices = async {
12159                let mut draining = self.module_rx.recv().await.unwrap();
12160                assert!(matches!(
12161                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
12162                        &draining.body
12163                    )
12164                    .unwrap(),
12165                    subc_protocol::session::ModuleControlCommand::Draining {
12166                        reason: RouteCloseReason::Restart,
12167                        ..
12168                    }
12169                ));
12170                draining.flushed.take().unwrap().send(()).unwrap();
12171                let mut closing = self.client_rx.recv().await.unwrap();
12172                assert!(matches!(
12173                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
12174                    ClientControlPush::RouteClosing {
12175                        reason: RouteCloseReason::Restart,
12176                        ..
12177                    }
12178                ));
12179                closing.flushed.take().unwrap().send(()).unwrap();
12180            };
12181            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
12182            result.unwrap();
12183        }
12184
12185        fn disconnect(&mut self) -> ClientControlPush {
12186            self.handler
12187                .cleanup_connection(self.module_ctx.connection_id)
12188                .unwrap();
12189            let closed = self.client_rx.try_recv().unwrap();
12190            assert_eq!(closed.header.ty, FrameType::Push);
12191            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
12192            assert!(matches!(
12193                &push,
12194                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
12195                    if channels == &[self.route.client_channel]
12196            ));
12197            assert_eq!(
12198                self.client_rx.try_recv().unwrap().header.ty,
12199                FrameType::Goodbye
12200            );
12201            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
12202            push
12203        }
12204    }
12205
12206    #[cfg(unix)]
12207    #[tokio::test(start_paused = true)]
12208    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
12209        let mut fixture = route_closed_fixture().await;
12210        fixture.cut().await;
12211        assert!(matches!(
12212            fixture.disconnect(),
12213            ClientControlPush::RouteClosed {
12214                reason: RouteCloseReason::Restart,
12215                drained: true,
12216                ..
12217            }
12218        ));
12219    }
12220
12221    #[cfg(unix)]
12222    #[tokio::test(start_paused = true)]
12223    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
12224        let mut fixture = route_closed_fixture().await;
12225        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
12226        let started = tokio::time::Instant::now();
12227        fixture.cut().await;
12228        assert!(started.elapsed() >= Duration::from_secs(2));
12229        assert_eq!(fixture.route.flow.in_flight(), 1);
12230        assert!(matches!(
12231            fixture.disconnect(),
12232            ClientControlPush::RouteClosed {
12233                reason: RouteCloseReason::Restart,
12234                drained: false,
12235                ..
12236            }
12237        ));
12238    }
12239
12240    #[tokio::test]
12241    async fn module_connection_crash_reports_crash_and_not_drained() {
12242        let mut fixture = route_closed_fixture().await;
12243        // The server calls cleanup_connection on module EOF. Without a planned
12244        // daemon shutdown, that disconnect must remain a crash with no prior
12245        // route.closing notice, even when the route has no in-flight calls.
12246        assert!(matches!(
12247            fixture.disconnect(),
12248            ClientControlPush::RouteClosed {
12249                reason: RouteCloseReason::Crash,
12250                drained: false,
12251                ..
12252            }
12253        ));
12254    }
12255
12256    #[tokio::test]
12257    async fn dropping_router_connection_releases_registration() {
12258        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12259        let registry = Arc::new(Registry::default());
12260        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
12261        let router = Router::with_control_handler(Arc::clone(&control));
12262        let connection = router.begin_connection();
12263        let connection_id = connection.id();
12264        let (ctx, mut rx) = route_ctx(connection_id);
12265
12266        router
12267            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
12268            .await
12269            .unwrap();
12270        let response = rx.recv().await.unwrap();
12271        let ack = parse_ack(&response);
12272        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12273        assert!(registry.get_module("aft").unwrap().is_some());
12274
12275        drop(connection);
12276
12277        assert!(registry.get_module("aft").unwrap().is_none());
12278        assert_eq!(registry.active_registration_count().unwrap(), 0);
12279
12280        control
12281            .cleanup_connection(ConnectionId::new(u64::MAX))
12282            .unwrap();
12283        let captured = crate::router::test_log::captured_logs(&logs);
12284        println!("captured registry lifecycle logs:\n{captured}");
12285        let events: Vec<_> = captured
12286            .lines()
12287            .filter(|line| {
12288                line.contains("module registered module_id=aft ")
12289                    || line.contains("module registration ended")
12290            })
12291            .collect();
12292        assert_eq!(
12293            events.len(),
12294            2,
12295            "unexpected registry lifecycle logs: {captured}"
12296        );
12297        assert!(events[0].contains("module registered module_id=aft "));
12298        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
12299        assert!(events[1].contains(&format!(
12300            "module_id=aft connection_id={}",
12301            connection_id.get()
12302        )));
12303        assert!(events[1].contains("reason=connection_closed"));
12304        assert!(events[1].contains("module registration ended"));
12305        assert_eq!(
12306            captured
12307                .lines()
12308                .filter(|line| line.contains("module registered module_id=aft "))
12309                .count(),
12310            1,
12311            "legacy registration admission line must appear once: {captured}"
12312        );
12313    }
12314
12315    #[tokio::test]
12316    async fn hello_registration_keeps_legacy_module_registered_line_once() {
12317        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12318        let registry = Arc::new(Registry::default());
12319        let control = ControlHandler::new(Arc::clone(&registry));
12320        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
12321        hello_via_sink(
12322            &control,
12323            &ctx,
12324            &mut rx,
12325            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
12326        )
12327        .await;
12328
12329        let captured = crate::router::test_log::captured_logs(&logs);
12330        let admissions: Vec<_> = captured
12331            .lines()
12332            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
12333            .collect();
12334        assert_eq!(
12335            admissions.len(),
12336            1,
12337            "legacy admission line must remain exactly once: {captured}"
12338        );
12339        assert!(admissions[0].contains("routable_provider=true"));
12340        assert!(admissions[0].contains("connection_id=777"));
12341    }
12342
12343    fn capability_manifest(
12344        module_id: &str,
12345        provides: &[&str],
12346        must_never_reach: &[&str],
12347    ) -> ModuleManifest {
12348        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
12349        manifest.capabilities = Some(CapabilityDeclarations {
12350            provides: provides
12351                .iter()
12352                .map(|capability| (*capability).to_string())
12353                .collect(),
12354            requires: Vec::new(),
12355            must_never_reach: must_never_reach
12356                .iter()
12357                .map(|capability| (*capability).to_string())
12358                .collect(),
12359        });
12360        manifest
12361    }
12362
12363    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
12364        Frame::build(
12365            FrameType::Hello,
12366            control_flags(),
12367            0,
12368            0,
12369            corr,
12370            serde_json::to_vec(&ModuleHelloBody {
12371                protocol_ver: manifest.protocol_ver,
12372                manifest,
12373                control_ops: None,
12374                launch_nonce: None,
12375            })
12376            .expect("capability test HELLO serializes"),
12377        )
12378        .expect("capability test HELLO frame builds")
12379    }
12380
12381    fn catalog_update_with_capabilities_frame(
12382        corr: u64,
12383        capabilities: CapabilityDeclarations,
12384    ) -> Frame {
12385        Frame::build(
12386            FrameType::Request,
12387            control_flags(),
12388            0,
12389            0,
12390            corr,
12391            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12392                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
12393                capabilities: Some(capabilities),
12394                ready: None,
12395            })
12396            .expect("capability catalog.update serializes"),
12397        )
12398        .expect("capability catalog.update frame builds")
12399    }
12400
12401    async fn register_capability_manifest(
12402        handler: &ControlHandler,
12403        ctx: &RouteCtx,
12404        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12405        manifest: ModuleManifest,
12406        corr: u64,
12407    ) {
12408        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
12409    }
12410
12411    async fn open_route_for_capability_test(
12412        handler: &ControlHandler,
12413        target_ctx: &RouteCtx,
12414        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12415        client_connection_id: u64,
12416        corr: u64,
12417        target_module_id: &str,
12418        consumer_identity: Option<ConsumerIdentity>,
12419    ) -> (
12420        mpsc::Receiver<crate::router::OutboundFrame>,
12421        ModuleControlRequest,
12422    ) {
12423        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
12424        let route_handler = handler.clone();
12425        let target_module_id = target_module_id.to_string();
12426        let route_task = tokio::spawn(async move {
12427            route_handler
12428                .handle_control_frame(
12429                    &client_ctx,
12430                    route_open_frame_with_admission_facts(
12431                        corr,
12432                        &target_module_id,
12433                        unique_project_root("admission-facts"),
12434                        consumer_identity,
12435                        None,
12436                    ),
12437                )
12438                .await
12439                .expect("capability test route.open succeeds")
12440        });
12441        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
12442            .await
12443            .expect("capability test route.open must reach route.bind")
12444            .expect("target control receiver stays open");
12445        let bind_request: ModuleControlRequest =
12446            serde_json::from_slice(&bind.body).expect("route.bind decodes");
12447        handler
12448            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
12449            .await
12450            .expect("capability test route.bind ACK succeeds");
12451        assert!(route_task.await.expect("route.open task joins").is_empty());
12452        let opened = client_rx
12453            .recv()
12454            .await
12455            .expect("successful route.open publishes a response");
12456        assert!(matches!(
12457            serde_json::from_slice::<ClientControlResponse>(&opened.body),
12458            Ok(ClientControlResponse::RouteOpen { .. })
12459        ));
12460        (client_rx, bind_request)
12461    }
12462
12463    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
12464        assert_eq!(frame.header.ty, FrameType::Push);
12465        assert_eq!(frame.header.channel, 0);
12466        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
12467            .expect("route.closed control push decodes");
12468        let ClientControlPush::RouteClosed { channels, .. } = &push else {
12469            panic!("expected route.closed");
12470        };
12471        assert_eq!(channels.len(), 1, "exactly one violating route closed");
12472        let channels = channels.clone();
12473        assert_eq!(
12474            push,
12475            ClientControlPush::RouteClosed {
12476                module_id: target_module_id.to_string(),
12477                channels,
12478                reason: RouteCloseReason::CapabilityDenied,
12479                drained: false,
12480                abandoned: 0,
12481                excluded_subscriptions: 0,
12482                terminal: Some(false),
12483            }
12484        );
12485    }
12486
12487    #[tokio::test]
12488    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
12489        let registry = Arc::new(Registry::default());
12490        let forwarding = Arc::new(ForwardingTable::default());
12491        let supervisor = SupervisorHandle::new();
12492        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12493        let handler =
12494            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12495                .with_supervisor(supervisor);
12496        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
12497        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
12498        register_capability_manifest(
12499            &handler,
12500            &target_ctx,
12501            &mut target_rx,
12502            capability_manifest("target", &["credentials-provider/v1"], &[]),
12503            1,
12504        )
12505        .await;
12506        register_capability_manifest(
12507            &handler,
12508            &opener_ctx,
12509            &mut opener_rx,
12510            capability_manifest("opener", &[], &["credentials-provider/v1"]),
12511            2,
12512        )
12513        .await;
12514
12515        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
12516        let replies = handler
12517            .handle_control_frame(
12518                &client_ctx,
12519                route_open_frame_with_admission_facts(
12520                    3,
12521                    "target",
12522                    unique_project_root("admission-facts"),
12523                    Some(ConsumerIdentity {
12524                        module_id: "opener".to_string(),
12525                        launch_nonce: "opener-nonce".to_string(),
12526                    }),
12527                    None,
12528                ),
12529            )
12530            .await
12531            .expect("denied route.open returns a typed frame");
12532        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12533        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12534        assert!(
12535            target_rx.try_recv().is_err(),
12536            "forbidden route.open must not relay route.bind"
12537        );
12538    }
12539
12540    #[tokio::test]
12541    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
12542        let registry = Arc::new(Registry::default());
12543        let forwarding = Arc::new(ForwardingTable::default());
12544        let supervisor = SupervisorHandle::new();
12545        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12546        let handler =
12547            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12548                .with_supervisor(supervisor);
12549        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
12550        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
12551        register_capability_manifest(
12552            &handler,
12553            &target_ctx,
12554            &mut target_rx,
12555            capability_manifest("target", &["credentials-provider/v1"], &[]),
12556            1,
12557        )
12558        .await;
12559        register_capability_manifest(
12560            &handler,
12561            &old_opener_ctx,
12562            &mut old_opener_rx,
12563            capability_manifest("opener", &[], &[]),
12564            2,
12565        )
12566        .await;
12567        let (mut client_rx, _) = open_route_for_capability_test(
12568            &handler,
12569            &target_ctx,
12570            &mut target_rx,
12571            712,
12572            3,
12573            "target",
12574            Some(ConsumerIdentity {
12575                module_id: "opener".to_string(),
12576                launch_nonce: "opener-nonce".to_string(),
12577            }),
12578        )
12579        .await;
12580        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12581
12582        handler
12583            .cleanup_connection(old_opener_ctx.connection_id)
12584            .expect("old opener registration cleans up");
12585        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
12586        register_capability_manifest(
12587            &handler,
12588            &new_opener_ctx,
12589            &mut new_opener_rx,
12590            capability_manifest("opener", &[], &["credentials-provider/v1"]),
12591            4,
12592        )
12593        .await;
12594
12595        assert_capability_denied_push(
12596            client_rx
12597                .try_recv()
12598                .expect("HELLO deny addition must emit route.closed")
12599                .frame,
12600            "target",
12601        );
12602        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12603        assert!(matches!(
12604            target_rx.try_recv(),
12605            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12606        ));
12607    }
12608
12609    #[tokio::test]
12610    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
12611        let registry = Arc::new(Registry::default());
12612        let forwarding = Arc::new(ForwardingTable::default());
12613        let supervisor = SupervisorHandle::new();
12614        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12615        let handler =
12616            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12617                .with_supervisor(supervisor);
12618        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
12619        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
12620        register_capability_manifest(
12621            &handler,
12622            &target_ctx,
12623            &mut target_rx,
12624            capability_manifest("target", &[], &[]),
12625            1,
12626        )
12627        .await;
12628        register_capability_manifest(
12629            &handler,
12630            &opener_ctx,
12631            &mut opener_rx,
12632            capability_manifest("opener", &[], &["credentials-provider/v1"]),
12633            2,
12634        )
12635        .await;
12636        let (mut client_rx, _) = open_route_for_capability_test(
12637            &handler,
12638            &target_ctx,
12639            &mut target_rx,
12640            722,
12641            3,
12642            "target",
12643            Some(ConsumerIdentity {
12644                module_id: "opener".to_string(),
12645                launch_nonce: "opener-nonce".to_string(),
12646            }),
12647        )
12648        .await;
12649        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12650
12651        let replies = handler
12652            .handle_control_frame(
12653                &target_ctx,
12654                catalog_update_with_capabilities_frame(
12655                    4,
12656                    CapabilityDeclarations {
12657                        provides: vec!["credentials-provider/v1".to_string()],
12658                        requires: Vec::new(),
12659                        must_never_reach: Vec::new(),
12660                    },
12661                ),
12662            )
12663            .await
12664            .expect("claim catalog.update succeeds");
12665        assert!(matches!(
12666            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
12667            Ok(ModuleControlResponseToModule::CatalogUpdate {})
12668        ));
12669        assert_capability_denied_push(
12670            client_rx
12671                .try_recv()
12672                .expect("claim addition must emit route.closed")
12673                .frame,
12674            "target",
12675        );
12676        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12677        assert!(matches!(
12678            target_rx.try_recv(),
12679            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12680        ));
12681    }
12682
12683    #[tokio::test]
12684    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
12685        let registry = Arc::new(Registry::default());
12686        let forwarding = Arc::new(ForwardingTable::default());
12687        let supervisor = SupervisorHandle::new();
12688        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12689        let handler =
12690            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12691                .with_supervisor(supervisor);
12692        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
12693        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
12694        register_capability_manifest(
12695            &handler,
12696            &target_ctx,
12697            &mut target_rx,
12698            capability_manifest("target", &["credentials-provider/v1"], &[]),
12699            1,
12700        )
12701        .await;
12702        register_capability_manifest(
12703            &handler,
12704            &opener_ctx,
12705            &mut opener_rx,
12706            capability_manifest("opener", &[], &[]),
12707            2,
12708        )
12709        .await;
12710        let (mut client_rx, _) = open_route_for_capability_test(
12711            &handler,
12712            &target_ctx,
12713            &mut target_rx,
12714            732,
12715            3,
12716            "target",
12717            Some(ConsumerIdentity {
12718                module_id: "opener".to_string(),
12719                launch_nonce: "opener-nonce".to_string(),
12720            }),
12721        )
12722        .await;
12723        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12724
12725        handler
12726            .handle_control_frame(
12727                &target_ctx,
12728                catalog_update_with_capabilities_frame(
12729                    4,
12730                    CapabilityDeclarations {
12731                        provides: Vec::new(),
12732                        requires: Vec::new(),
12733                        must_never_reach: Vec::new(),
12734                    },
12735                ),
12736            )
12737            .await
12738            .expect("claim removal catalog.update succeeds");
12739        assert_eq!(
12740            forwarding.active_binding_count().unwrap(),
12741            1,
12742            "removing an attested target claim must leave the route census unchanged"
12743        );
12744        assert!(
12745            client_rx.try_recv().is_err(),
12746            "claim removal must not emit route.closed capability_denied"
12747        );
12748        assert!(
12749            target_rx.try_recv().is_err(),
12750            "claim removal must not send the target a route GOODBYE"
12751        );
12752    }
12753
12754    /// A direct client may open a route to a denied capability provider; this
12755    /// policy applies only to attested supervised module origins, not to direct clients.
12756    #[tokio::test]
12757    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
12758        let registry = Arc::new(Registry::default());
12759        let forwarding = Arc::new(ForwardingTable::default());
12760        let supervisor = SupervisorHandle::new();
12761        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12762        let handler =
12763            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12764                .with_supervisor(supervisor);
12765        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
12766        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
12767        register_capability_manifest(
12768            &handler,
12769            &target_ctx,
12770            &mut target_rx,
12771            capability_manifest("target", &["credentials-provider/v1"], &[]),
12772            1,
12773        )
12774        .await;
12775        register_capability_manifest(
12776            &handler,
12777            &opener_ctx,
12778            &mut opener_rx,
12779            capability_manifest("opener", &[], &["credentials-provider/v1"]),
12780            2,
12781        )
12782        .await;
12783
12784        let (_client_rx, bind) = open_route_for_capability_test(
12785            &handler,
12786            &target_ctx,
12787            &mut target_rx,
12788            742,
12789            3,
12790            "target",
12791            None,
12792        )
12793        .await;
12794        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12795            panic!("direct scope-honesty route must bind");
12796        };
12797        assert_eq!(principal, Some(Principal::Direct));
12798        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12799    }
12800
12801    /// A module that denies a capability receives no self-route exemption when it
12802    /// also attestedly provides that capability.
12803    #[tokio::test]
12804    async fn must_never_reach_self_route_is_capability_forbidden() {
12805        let registry = Arc::new(Registry::default());
12806        let forwarding = Arc::new(ForwardingTable::default());
12807        let supervisor = SupervisorHandle::new();
12808        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
12809        let handler =
12810            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12811                .with_supervisor(supervisor);
12812        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
12813        register_capability_manifest(
12814            &handler,
12815            &self_ctx,
12816            &mut self_rx,
12817            capability_manifest(
12818                "self-provider",
12819                &["credentials-provider/v1"],
12820                &["credentials-provider/v1"],
12821            ),
12822            1,
12823        )
12824        .await;
12825
12826        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
12827        let replies = handler
12828            .handle_control_frame(
12829                &client_ctx,
12830                route_open_frame_with_admission_facts(
12831                    2,
12832                    "self-provider",
12833                    unique_project_root("admission-facts"),
12834                    Some(ConsumerIdentity {
12835                        module_id: "self-provider".to_string(),
12836                        launch_nonce: "self-nonce".to_string(),
12837                    }),
12838                    None,
12839                ),
12840            )
12841            .await
12842            .expect("self-route refusal returns a typed frame");
12843        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12844        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12845        assert!(
12846            self_rx.try_recv().is_err(),
12847            "self denial must not relay route.bind"
12848        );
12849    }
12850
12851    #[test]
12852    fn unsupported_channel_zero_frame_returns_error() {
12853        let handler = ControlHandler::default();
12854        let request = Frame::build(
12855            FrameType::Request,
12856            control_flags(),
12857            0,
12858            0,
12859            21,
12860            b"opaque".to_vec(),
12861        )
12862        .unwrap();
12863
12864        let response = handler
12865            .handle_control(ConnectionId::new(1), request)
12866            .unwrap();
12867
12868        assert_eq!(response[0].header.ty, FrameType::Error);
12869        assert_eq!(
12870            parse_error(&response[0])["code"],
12871            "unsupported_control_frame"
12872        );
12873    }
12874
12875    /// Blue/green swap at the control-plane boundary. The supervisor that opens
12876    /// a swap is not wired yet, so the candidate is registered here directly
12877    /// into the registry and forwarding candidate slots, the way the swap's
12878    /// HELLO admission will.
12879    mod swap {
12880        use super::*;
12881
12882        const INCUMBENT: ConnectionId = ConnectionId::new(30);
12883        const CANDIDATE: ConnectionId = ConnectionId::new(40);
12884
12885        struct Swap {
12886            registry: Arc<Registry>,
12887            forwarding: Arc<ForwardingTable>,
12888            handler: ControlHandler,
12889            incumbent_ctx: RouteCtx,
12890            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12891            candidate_ctx: RouteCtx,
12892            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12893        }
12894
12895        async fn swap_with_incumbent() -> Swap {
12896            let registry = Arc::new(Registry::default());
12897            let forwarding = Arc::new(ForwardingTable::default());
12898            let handler =
12899                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12900            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
12901            hello_via_sink(
12902                &handler,
12903                &incumbent_ctx,
12904                &mut incumbent_rx,
12905                hello_frame("aft", PROTOCOL_VERSION, 7),
12906            )
12907            .await;
12908            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
12909            Swap {
12910                registry,
12911                forwarding,
12912                handler,
12913                incumbent_ctx,
12914                incumbent_rx,
12915                candidate_ctx,
12916                candidate_rx,
12917            }
12918        }
12919
12920        fn register_candidate(swap: &Swap, ready: Option<bool>) {
12921            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
12922            candidate_manifest.ready = ready;
12923            let registration = swap
12924                .registry
12925                .register_candidate_with_control_ops(
12926                    candidate_manifest,
12927                    PROTOCOL_VERSION,
12928                    CANDIDATE,
12929                    module_baseline_control_ops(),
12930                )
12931                .unwrap();
12932            swap.forwarding
12933                .register_candidate_module_connection(
12934                    CANDIDATE,
12935                    "aft".to_string(),
12936                    PROTOCOL_VERSION,
12937                    manifest_concurrency(&registration.manifest),
12938                    swap.candidate_ctx.egress.clone(),
12939                )
12940                .unwrap();
12941        }
12942
12943        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
12944            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
12945            swap.registry.promote_candidate("aft").unwrap().unwrap();
12946            cutover.incumbent.unwrap()
12947        }
12948
12949        fn keyed_total(counters: &Value, key: &str) -> u64 {
12950            counters[key]
12951                .as_object()
12952                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
12953                .unwrap_or(0)
12954        }
12955
12956        #[tokio::test]
12957        async fn replacement_logs_old_end_and_new_admission_once() {
12958            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12959            let mut swap = swap_with_incumbent().await;
12960            swap.handler
12961                .supervisor
12962                .open_swap("aft", "candidate-nonce".to_string());
12963            hello_via_sink(
12964                &swap.handler,
12965                &swap.candidate_ctx,
12966                &mut swap.candidate_rx,
12967                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
12968            )
12969            .await;
12970            cutover(&swap);
12971            swap.handler.cleanup_connection(INCUMBENT).unwrap();
12972
12973            let captured = crate::router::test_log::captured_logs(&logs);
12974            println!("captured replacement registry logs:\n{captured}");
12975            let new_admissions: Vec<_> = captured
12976                .lines()
12977                .filter(|line| {
12978                    line.contains("connection_id=40")
12979                        && line.contains("swap candidate registered; not routable until cutover")
12980                })
12981                .collect();
12982            assert_eq!(
12983                new_admissions.len(),
12984                1,
12985                "unexpected admission logs: {captured}"
12986            );
12987            assert!(new_admissions[0].contains("module_id=aft"));
12988            assert!(new_admissions[0].contains("connection_id=40"));
12989            assert!(
12990                new_admissions[0].contains("swap candidate registered; not routable until cutover")
12991            );
12992            assert!(new_admissions[0].contains("ready=true"));
12993
12994            let old_admissions: Vec<_> = captured
12995                .lines()
12996                .filter(|line| {
12997                    line.contains("module registered module_id=aft ")
12998                        && line.contains("connection_id=30")
12999                })
13000                .collect();
13001            assert_eq!(
13002                old_admissions.len(),
13003                1,
13004                "unexpected incumbent admission: {captured}"
13005            );
13006
13007            let old_ends: Vec<_> = captured
13008                .lines()
13009                .filter(|line| {
13010                    line.contains("connection_id=30") && line.contains("module registration ended")
13011                })
13012                .collect();
13013            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
13014            assert!(old_ends[0].contains("module_id=aft"));
13015            assert!(old_ends[0].contains("connection_id=30"));
13016            assert!(old_ends[0].contains("reason=replaced"));
13017            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
13018            assert_eq!(
13019                captured
13020                    .lines()
13021                    .filter(|line| line.contains("module registration promoted"))
13022                    .count(),
13023                1,
13024                "unexpected promotion logs: {captured}"
13025            );
13026        }
13027
13028        /// An ack from the incumbent for a bind it was sent before cutover,
13029        /// arriving before the incumbent is drained. The incumbent is the live
13030        /// connection carrying every other client's routes, so the ack must
13031        /// not end it: the waiting client is told to retry, the reservation is
13032        /// given back, and the incumbent is told to drop just that binding.
13033        #[tokio::test]
13034        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
13035            let mut swap = swap_with_incumbent().await;
13036            let handler = swap.handler.clone();
13037
13038            // A co-tenant route, bound on the incumbent before the swap.
13039            let cotenant = ConnectionId::new(31);
13040            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13041            let (cotenant_task, cotenant_bind) = relay_route_open(
13042                &handler,
13043                cotenant,
13044                &cotenant_ctx.egress,
13045                &mut swap.incumbent_rx,
13046                100,
13047                "aft",
13048                "swap-cotenant",
13049            )
13050            .await;
13051            handler
13052                .handle_control_frame(
13053                    &swap.incumbent_ctx,
13054                    route_bind_ack(cotenant_bind.header.corr),
13055                )
13056                .await
13057                .unwrap();
13058            assert!(cotenant_task.await.unwrap().is_empty());
13059            let (cotenant_channel, cotenant_epoch) =
13060                published_route(&cotenant_rx.recv().await.unwrap());
13061
13062            // A second route.open, relayed to the incumbent and not yet acked.
13063            let caller = ConnectionId::new(32);
13064            let (caller_ctx, mut caller_rx) = route_ctx(caller);
13065            let (caller_task, caller_bind) = relay_route_open(
13066                &handler,
13067                caller,
13068                &caller_ctx.egress,
13069                &mut swap.incumbent_rx,
13070                101,
13071                "aft",
13072                "swap-caller",
13073            )
13074            .await;
13075            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
13076
13077            register_candidate(&swap, None);
13078            cutover(&swap);
13079
13080            // The incumbent acks after promotion and before any drain.
13081            let ack = handler
13082                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
13083                .await;
13084            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13085            if module_loop_error.is_some() {
13086                // What the connection loop does with an untranslated router
13087                // error: end the connection, releasing every route on it.
13088                handler.cleanup_connection(INCUMBENT).unwrap();
13089            }
13090
13091            // 1. The incumbent's other routes survive.
13092            assert!(
13093                cotenant_rx.try_recv().is_err(),
13094                "the co-tenant route on the incumbent was torn down by one late ack: \
13095                 {module_loop_error:?}"
13096            );
13097            assert!(matches!(
13098                swap.forwarding
13099                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13100                    .unwrap(),
13101                DataRoute::Client(DataRouteState::Bound(_))
13102            ));
13103            assert_eq!(module_loop_error, None);
13104            assert!(swap
13105                .registry
13106                .get_module_by_connection(INCUMBENT)
13107                .unwrap()
13108                .is_some());
13109
13110            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
13111            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
13112                .await
13113                .expect("the incumbent is told to drop the abandoned binding")
13114                .unwrap()
13115                .frame;
13116            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13117            assert_eq!(goodbye.header.channel, abandoned_channel);
13118            assert_eq!(goodbye.header.epoch, abandoned_epoch);
13119            assert!(swap.incumbent_rx.try_recv().is_err());
13120
13121            // 3. The waiting client gets a retryable refusal and no route.
13122            let response = caller_task.await.unwrap();
13123            assert_eq!(response.len(), 1);
13124            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
13125            assert!(caller_rx.try_recv().is_err());
13126
13127            // 4. The reservation pair is given back, and the pending bind
13128            //    settled exactly once: one accepted open (the co-tenant) and one
13129            //    refused open (the caller), nothing counted twice.
13130            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
13131            let counters = handler.counters().snapshot();
13132            assert_eq!(
13133                keyed_total(&counters, "route_open_accepted_by_principal"),
13134                1
13135            );
13136            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
13137            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
13138        }
13139
13140        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
13141        /// id would resolve to the promoted candidate and every new route.open
13142        /// would be refused as reloading, leaving neither process routable.
13143        #[tokio::test]
13144        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
13145            let mut swap = swap_with_incumbent().await;
13146            register_candidate(&swap, None);
13147            let incumbent = cutover(&swap);
13148            swap.forwarding
13149                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
13150                .unwrap()
13151                .expect("the incumbent is still registered");
13152
13153            let client = ConnectionId::new(33);
13154            let (client_ctx, mut client_rx) = route_ctx(client);
13155            let route_handler = swap.handler.clone();
13156            let open_ctx = RouteCtx {
13157                connection_id: client,
13158                egress: client_ctx.egress.clone(),
13159            };
13160            let mut route_task = tokio::spawn(async move {
13161                route_handler
13162                    .handle_control_frame(
13163                        &open_ctx,
13164                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
13165                    )
13166                    .await
13167                    .unwrap()
13168            });
13169            let bind = tokio::select! {
13170                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
13171                response = &mut route_task => {
13172                    let response = response.unwrap();
13173                    panic!(
13174                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
13175                        parse_error(&response[0])["code"]
13176                    );
13177                }
13178            };
13179            swap.handler
13180                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
13181                .await
13182                .unwrap();
13183            assert!(route_task.await.unwrap().is_empty());
13184            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
13185            match swap
13186                .forwarding
13187                .lookup_data_route(client, channel, epoch)
13188                .unwrap()
13189            {
13190                DataRoute::Client(DataRouteState::Bound(route)) => {
13191                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
13192                }
13193                other => panic!("expected a bound route on the candidate, got {other:?}"),
13194            }
13195            assert!(swap.incumbent_rx.try_recv().is_err());
13196        }
13197
13198        /// A candidate declares itself ready with `catalog.update` on its own
13199        /// connection. If the connection-keyed registry lookups searched only the
13200        /// active slot, this would answer `not_registered` and the candidate
13201        /// would never become ready.
13202        #[tokio::test]
13203        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
13204            let swap = swap_with_incumbent().await;
13205            register_candidate(&swap, Some(false));
13206            let update = Frame::build(
13207                FrameType::Request,
13208                control_flags(),
13209                0,
13210                0,
13211                55,
13212                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13213                    provides: manifest("aft", PROTOCOL_VERSION).provides,
13214                    capabilities: None,
13215                    ready: Some(true),
13216                })
13217                .unwrap(),
13218            )
13219            .unwrap();
13220
13221            let replies = swap
13222                .handler
13223                .handle_control_frame(&swap.candidate_ctx, update)
13224                .await
13225                .unwrap();
13226
13227            assert_eq!(replies.len(), 1);
13228            assert_eq!(
13229                replies[0].header.ty,
13230                FrameType::Response,
13231                "candidate catalog.update was refused: {:?}",
13232                serde_json::from_slice::<Value>(&replies[0].body).ok()
13233            );
13234            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
13235            assert_eq!(
13236                swap.registry
13237                    .get_module("aft")
13238                    .unwrap()
13239                    .unwrap()
13240                    .connection_id,
13241                INCUMBENT
13242            );
13243        }
13244    }
13245
13246    /// The HELLO gate while the supervisor has a swap open: only the nonce it
13247    /// minted for the candidate admits a second process, into the candidate
13248    /// slot, and that check runs ahead of the reserved-module gate.
13249    mod swap_admission {
13250        use super::*;
13251
13252        const INCUMBENT_NONCE: &str = "incumbent-nonce";
13253        const CANDIDATE_NONCE: &str = "candidate-nonce";
13254
13255        fn handler_with_incumbent(
13256            module_id: &str,
13257            reserved: bool,
13258        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
13259            let registry = Arc::new(Registry::default());
13260            let supervisor = SupervisorHandle::new();
13261            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
13262            if reserved {
13263                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
13264            }
13265            let handler =
13266                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
13267            let incumbent = handler
13268                .handle_control(
13269                    ConnectionId::new(1),
13270                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
13271                )
13272                .unwrap();
13273            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
13274            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
13275            (registry, supervisor, handler)
13276        }
13277
13278        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
13279        /// admits every nonce, so while a swap is open the swap gate is the only
13280        /// thing between a key-holder and the candidate slot. A nonce the
13281        /// supervisor did not mint, or none at all, is refused, and neither the
13282        /// incumbent's registration nor the candidate slot moves.
13283        #[test]
13284        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
13285            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
13286
13287            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
13288                let replies = handler
13289                    .handle_control(
13290                        ConnectionId::new(connection),
13291                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
13292                    )
13293                    .unwrap();
13294                assert_eq!(replies[0].header.ty, FrameType::Error);
13295                assert_eq!(
13296                    parse_error(&replies[0])["code"],
13297                    "swap_token_invalid",
13298                    "nonce {nonce:?}"
13299                );
13300            }
13301            assert!(registry.get_candidate("aft").unwrap().is_none());
13302            assert_eq!(
13303                registry.get_module("aft").unwrap().unwrap().connection_id,
13304                ConnectionId::new(1)
13305            );
13306
13307            // Control: the minted token is admitted, into the candidate slot,
13308            // and only once.
13309            let admitted = handler
13310                .handle_control(
13311                    ConnectionId::new(4),
13312                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
13313                )
13314                .unwrap();
13315            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
13316            assert_eq!(
13317                registry
13318                    .get_candidate("aft")
13319                    .unwrap()
13320                    .unwrap()
13321                    .connection_id,
13322                ConnectionId::new(4)
13323            );
13324            assert_eq!(
13325                registry.get_module("aft").unwrap().unwrap().connection_id,
13326                ConnectionId::new(1),
13327                "the candidate must not take the active slot"
13328            );
13329            let replayed = handler
13330                .handle_control(
13331                    ConnectionId::new(5),
13332                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
13333                )
13334                .unwrap();
13335            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
13336
13337            // The case only this gate covers: the incumbent has died mid-swap,
13338            // so its duplicate refusal is gone too, and without the gate a
13339            // key-holder would take the id's ACTIVE slot.
13340            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
13341            let squatter = handler
13342                .handle_control(
13343                    ConnectionId::new(6),
13344                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
13345                )
13346                .unwrap();
13347            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
13348            assert!(
13349                registry.get_module("aft").unwrap().is_none(),
13350                "a squatter took the active slot of an id being swapped"
13351            );
13352        }
13353
13354        /// Design mutation arm (iii). A reserved module's candidate presents a
13355        /// nonce the reserved gate has never seen (that gate holds the
13356        /// incumbent's), so the swap gate must run first or the candidate is
13357        /// refused `reserved_module` and a reserved module can never be swapped.
13358        #[test]
13359        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
13360            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
13361
13362            let replies = handler
13363                .handle_control(
13364                    ConnectionId::new(2),
13365                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13366                )
13367                .unwrap();
13368
13369            assert_eq!(
13370                replies[0].header.ty,
13371                FrameType::HelloAck,
13372                "reserved candidate refused: {:?}",
13373                serde_json::from_slice::<Value>(&replies[0].body).ok()
13374            );
13375            assert_eq!(
13376                registry
13377                    .get_candidate("vault")
13378                    .unwrap()
13379                    .unwrap()
13380                    .connection_id,
13381                ConnectionId::new(2)
13382            );
13383        }
13384
13385        /// With no swap open the gate is inert: the incumbent's reserved gate
13386        /// and duplicate refusal behave exactly as before.
13387        #[test]
13388        fn without_an_open_swap_the_ordinary_gates_decide() {
13389            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
13390            supervisor.close_swap("vault");
13391
13392            let candidate = handler
13393                .handle_control(
13394                    ConnectionId::new(2),
13395                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13396                )
13397                .unwrap();
13398            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
13399            let duplicate = handler
13400                .handle_control(
13401                    ConnectionId::new(3),
13402                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
13403                )
13404                .unwrap();
13405            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13406            assert!(registry.get_candidate("vault").unwrap().is_none());
13407        }
13408    }
13409
13410    /// `scope.sync` and `scope.describe` through the real control handler: who
13411    /// may sync is decided by the registration and launch nonce of the module
13412    /// connection, never by the request body.
13413    mod scopes {
13414        mod activity {
13415            include!("activity_scope_tests.rs");
13416        }
13417        mod delta {
13418            include!("scope_handler_delta_tests.rs");
13419        }
13420
13421        use subc_protocol::scope::{
13422            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
13423            ScopeStatus,
13424        };
13425
13426        use super::*;
13427
13428        const OWNER: &str = "prefrontal-core";
13429
13430        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
13431            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
13432        }
13433
13434        async fn call(
13435            handler: &ControlHandler,
13436            ctx: &RouteCtx,
13437            request: &ModuleControlRequestFromModule,
13438        ) -> Frame {
13439            let body = serde_json::to_vec(request).unwrap();
13440            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
13441            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
13442            assert_eq!(replies.len(), 1, "{replies:?}");
13443            replies.pop().unwrap()
13444        }
13445
13446        async fn sync(
13447            handler: &ControlHandler,
13448            ctx: &RouteCtx,
13449            generation: u64,
13450            scopes: Vec<ScopeRecord>,
13451        ) -> Result<ModuleControlResponseToModule, String> {
13452            let reply = call(
13453                handler,
13454                ctx,
13455                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
13456            )
13457            .await;
13458            match reply.header.ty {
13459                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
13460                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
13461            }
13462        }
13463
13464        async fn describe(
13465            handler: &ControlHandler,
13466            ctx: &RouteCtx,
13467            owner: &str,
13468            scope_ref: &str,
13469        ) -> ModuleControlResponseToModule {
13470            let reply = call(
13471                handler,
13472                ctx,
13473                &ModuleControlRequestFromModule::ScopeDescribe {
13474                    owner: Principal::Reserved {
13475                        module_id: owner.to_string(),
13476                    },
13477                    scope_ref: scope_ref.to_string(),
13478                },
13479            )
13480            .await;
13481            assert_eq!(
13482                reply.header.ty,
13483                FrameType::Response,
13484                "{:?}",
13485                parse_error(&reply)
13486            );
13487            serde_json::from_slice(&reply.body).unwrap()
13488        }
13489
13490        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
13491        async fn module(
13492            handler: &ControlHandler,
13493            connection: u64,
13494            module_id: &str,
13495            nonce: Option<&str>,
13496        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13497            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
13498            hello_via_sink(
13499                handler,
13500                &ctx,
13501                &mut rx,
13502                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
13503            )
13504            .await;
13505            (ctx, rx)
13506        }
13507
13508        /// `direct` and every other client connection has no registration, so
13509        /// it can neither sync nor own a scope.
13510        #[tokio::test]
13511        async fn a_client_connection_cannot_sync_or_describe() {
13512            let handler = ControlHandler::new(Arc::new(Registry::default()));
13513            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
13514            for request in [
13515                ModuleControlRequestFromModule::ScopeSync {
13516                    generation: 1,
13517                    scopes: vec![head("s", 1)],
13518                },
13519                ModuleControlRequestFromModule::ScopeDescribe {
13520                    owner: Principal::Direct,
13521                    scope_ref: "s".to_string(),
13522                },
13523            ] {
13524                let reply = call(&handler, &ctx, &request).await;
13525                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
13526            }
13527            assert!(
13528                !handler
13529                    .scopes
13530                    .read()
13531                    .unwrap()
13532                    .describe(
13533                        &Principal::Reserved {
13534                            module_id: OWNER.to_string()
13535                        },
13536                        "s"
13537                    )
13538                    .owner_synced
13539            );
13540        }
13541
13542        /// A module the supervisor did not spawn registers without a launch
13543        /// nonce, so it is never an owner's current launch.
13544        #[tokio::test]
13545        async fn a_module_without_a_supervised_launch_cannot_sync() {
13546            let handler = ControlHandler::new(Arc::new(Registry::default()));
13547            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
13548            assert_eq!(
13549                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
13550                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13551            );
13552        }
13553
13554        #[tokio::test]
13555        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
13556            let supervisor = SupervisorHandle::new();
13557            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13558            let handler = ControlHandler::new(Arc::new(Registry::default()))
13559                .with_supervisor(supervisor.clone());
13560            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13561            sync(&handler, &incumbent, 1, vec![head("s", 1)])
13562                .await
13563                .expect("the current launch syncs");
13564
13565            // A swap candidate registers with the swap token and is refused
13566            // while the incumbent keeps syncing.
13567            supervisor.open_swap(OWNER, "n2".to_string());
13568            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
13569            assert_eq!(
13570                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13571                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13572            );
13573            sync(&handler, &incumbent, 2, vec![head("s", 1)])
13574                .await
13575                .expect("the serving owner syncs during the swap");
13576
13577            // The swap fails and is rolled back. The candidate never held sync
13578            // authority, and still cannot sync.
13579            supervisor.close_swap(OWNER);
13580            assert_eq!(
13581                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13582                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13583            );
13584            sync(&handler, &incumbent, 3, vec![head("s", 1)])
13585                .await
13586                .expect("the serving owner syncs after the rollback");
13587            handler.cleanup_connection(candidate.connection_id).unwrap();
13588
13589            // A swap that cuts over. Promotion records the candidate's nonce as
13590            // the module's spawn nonce, which is what `set_spawn_nonce` does
13591            // here; the promoted connection then takes authority at any
13592            // generation and the superseded incumbent is refused.
13593            supervisor.open_swap(OWNER, "n3".to_string());
13594            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
13595            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
13596            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
13597                .await
13598                .expect("the promoted launch takes authority");
13599            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13600                panic!("unexpected reply {reply:?}");
13601            };
13602            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
13603            assert_eq!(
13604                sync(&handler, &incumbent, 4, Vec::new()).await,
13605                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13606            );
13607        }
13608
13609        /// Authority dies with its connection: the cleanup path releases it,
13610        /// so the owner's next connection takes it at any generation.
13611        #[tokio::test]
13612        async fn closing_the_authority_connection_frees_sync_authority() {
13613            let supervisor = SupervisorHandle::new();
13614            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13615            let handler = ControlHandler::new(Arc::new(Registry::default()))
13616                .with_supervisor(supervisor.clone());
13617            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13618            sync(&handler, &first, 10, vec![head("s", 1)])
13619                .await
13620                .unwrap();
13621            handler.cleanup_connection(first.connection_id).unwrap();
13622
13623            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
13624            sync(&handler, &second, 1, vec![head("s", 1)])
13625                .await
13626                .expect("the next connection takes the released authority");
13627        }
13628
13629        #[tokio::test]
13630        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
13631            let supervisor = SupervisorHandle::new();
13632            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13633            let handler =
13634                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
13635            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
13636            sync(&handler, &first, 10, vec![head("s", 1)])
13637                .await
13638                .unwrap();
13639            handler
13640                .handle_control_frame(
13641                    &first,
13642                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
13643                )
13644                .await
13645                .unwrap();
13646            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
13647            sync(&handler, &second, 1, vec![head("s", 1)])
13648                .await
13649                .expect("GOODBYE releases authority even if the old socket remains open");
13650        }
13651
13652        #[tokio::test]
13653        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
13654            let registry = Arc::new(Registry::default());
13655            let supervisor_handle = SupervisorHandle::new();
13656            let supervisor =
13657                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
13658                    .with_handle(supervisor_handle.clone())
13659                    .with_daemon_incarnation("incarnation-7".to_string());
13660            // Configured with enabled: false, so the supervisor lists the
13661            // module without spawning a process for it.
13662            supervisor
13663                .supervise_configured(
13664                    ModuleSpec {
13665                        module_id: OWNER.to_string(),
13666                        program: PathBuf::from("/nonexistent/prefrontal-core"),
13667                        args: Vec::new(),
13668                        env: Vec::new(),
13669                        reserved: false,
13670                        reserved_prefixes: Vec::new(),
13671                        protocol: ModuleProtocol::Subc,
13672                        overlap: Default::default(),
13673                    },
13674                    false,
13675                )
13676                .unwrap();
13677            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
13678            let handler =
13679                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13680            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
13681
13682            // Configured but not yet synced: a reader waits for the owner.
13683            let ModuleControlResponseToModule::ScopeDescribe {
13684                status,
13685                daemon_incarnation,
13686                owner_synced,
13687                owner_configured,
13688                scope,
13689                ..
13690            } = describe(&handler, &reader, OWNER, "s").await
13691            else {
13692                panic!("not a describe reply");
13693            };
13694            assert_eq!(status, ScopeStatus::NotLive);
13695            assert_eq!(daemon_incarnation, "incarnation-7");
13696            assert!(!owner_synced);
13697            assert!(owner_configured);
13698            assert!(scope.is_none());
13699
13700            // Not a supervised module: the owner will never sync, and a reader
13701            // refuses rather than waits.
13702            let ModuleControlResponseToModule::ScopeDescribe {
13703                status,
13704                owner_configured,
13705                ..
13706            } = describe(&handler, &reader, "ghost", "s").await
13707            else {
13708                panic!("not a describe reply");
13709            };
13710            assert_eq!(status, ScopeStatus::NotLive);
13711            assert!(!owner_configured);
13712
13713            // Live, with the stamp fields and the computed owner_authorized.
13714            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
13715            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
13716            let ModuleControlResponseToModule::ScopeDescribe {
13717                status,
13718                scope_epoch,
13719                owner_synced,
13720                scope,
13721                ..
13722            } = describe(&handler, &reader, OWNER, "s").await
13723            else {
13724                panic!("not a describe reply");
13725            };
13726            assert_eq!(status, ScopeStatus::Live);
13727            assert_eq!(scope_epoch, Some(4));
13728            assert!(owner_synced);
13729            let stamp = scope.expect("a live scope carries its stamp");
13730            assert!(
13731                stamp.owner_authorized,
13732                "prefrontal-core is the default authority"
13733            );
13734            assert_eq!(stamp.kind, ScopeKind::Head);
13735        }
13736
13737        #[tokio::test]
13738        async fn scope_authority_owners_decides_owner_authorized() {
13739            let supervisor = SupervisorHandle::new();
13740            supervisor.set_spawn_nonce("broca", "b1".to_string());
13741            let handler = ControlHandler::new(Arc::new(Registry::default()))
13742                .with_supervisor(supervisor)
13743                .with_scope_authority_owners(vec!["broca".to_string()]);
13744            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
13745            let mut gated = head("s", 1);
13746            gated.attributes.agent_id = Some("agent".to_string());
13747            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
13748            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
13749                describe(&handler, &broca, "broca", "s").await
13750            else {
13751                panic!("not a describe reply");
13752            };
13753            assert!(scope.unwrap().owner_authorized);
13754        }
13755
13756        /// With route admission, the stamp, the commit re-check and drains in
13757        /// place, the feature is advertised: the module ops in HELLO_ACK, and
13758        /// `scopes/v1` in HELLO_ACK and `server.describe`.
13759        #[tokio::test]
13760        async fn scope_ops_and_the_scopes_capability_are_advertised() {
13761            let handler = ControlHandler::new(Arc::new(Registry::default()));
13762            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
13763            let ack = hello_via_sink(
13764                &handler,
13765                &ctx,
13766                &mut rx,
13767                hello_frame("m", PROTOCOL_VERSION, 1),
13768            )
13769            .await;
13770            let ack = parse_ack(&ack);
13771            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
13772                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
13773            }
13774            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
13775
13776            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
13777            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13778            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13779            let reply = handler
13780                .handle_control_frame(&client, frame)
13781                .await
13782                .unwrap()
13783                .pop()
13784                .unwrap();
13785            let ClientControlResponse::ServerDescribe { capabilities, .. } =
13786                serde_json::from_slice(&reply.body).unwrap()
13787            else {
13788                panic!("not a server.describe reply");
13789            };
13790            assert!(
13791                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
13792                "{capabilities:?}"
13793            );
13794        }
13795
13796        // ---- route admission, stamps, commit re-check and drains ----------
13797
13798        const PLEXUS: &str = "plexus";
13799        const OTHER: &str = "other";
13800        const AFT: &str = "aft";
13801        const BROCA: &str = "broca";
13802        const MAGIC: &str = "magic-context";
13803
13804        fn nonce(module_id: &str) -> String {
13805            format!("nonce-{module_id}")
13806        }
13807
13808        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13809            let (tx, rx) = mpsc::channel(64);
13810            (
13811                RouteCtx {
13812                    connection_id: ConnectionId::new(connection),
13813                    egress: FrameSink::new(tx),
13814                },
13815                rx,
13816            )
13817        }
13818
13819        /// A daemon with a configured owner (prefrontal-core) registered on its
13820        /// own module connection, two routable targets (plexus, other), and
13821        /// launch nonces minted for the modules that open routes as carriers.
13822        struct Rig {
13823            handler: ControlHandler,
13824            forwarding: Arc<ForwardingTable>,
13825            owner: RouteCtx,
13826            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13827            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
13828            generation: u64,
13829            next_connection: u64,
13830            _supervisor: Supervisor,
13831        }
13832
13833        async fn rig() -> Rig {
13834            rig_with_flow_support(true).await
13835        }
13836
13837        async fn rig_with_flow_support(flow_support: bool) -> Rig {
13838            rig_with_scope_capabilities(flow_support, false).await
13839        }
13840
13841        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
13842            let registry = Arc::new(Registry::default());
13843            let forwarding = Arc::new(ForwardingTable::default());
13844            let supervisor_handle = SupervisorHandle::new();
13845            let supervisor =
13846                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
13847                    .with_handle(supervisor_handle.clone());
13848            supervisor
13849                .supervise_configured(
13850                    ModuleSpec {
13851                        module_id: OWNER.to_string(),
13852                        program: PathBuf::from("/nonexistent/prefrontal-core"),
13853                        args: Vec::new(),
13854                        env: Vec::new(),
13855                        reserved: false,
13856                        reserved_prefixes: Vec::new(),
13857                        protocol: ModuleProtocol::Subc,
13858                        overlap: Default::default(),
13859                    },
13860                    false,
13861                )
13862                .unwrap();
13863            for module_id in [OWNER, AFT, BROCA, MAGIC] {
13864                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
13865            }
13866            let handler =
13867                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13868                    .with_supervisor(supervisor_handle);
13869            let (owner, mut owner_rx) = wide_ctx(1);
13870            hello_via_sink(
13871                &handler,
13872                &owner,
13873                &mut owner_rx,
13874                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
13875            )
13876            .await;
13877            let mut modules = BTreeMap::new();
13878            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
13879                let (ctx, mut rx) = wide_ctx(connection);
13880                let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
13881                let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13882                // A decoder version alone must not admit flow routes. Every
13883                // target here declares wire crate version 0.29.0; only one that
13884                // declares `flow-scopes/v1` promises flow behaviour.
13885                body["manifest"]["provenance"] =
13886                    serde_json::json!({"wire_crate_version": "0.29.0"});
13887                if flow_support || run_support {
13888                    let mut provides = Vec::new();
13889                    if flow_support {
13890                        provides.push("flow-scopes/v1");
13891                    }
13892                    if run_support {
13893                        provides.push("agent-run-scopes/v1");
13894                    }
13895                    body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
13896                }
13897                let hello = Frame::build(
13898                    FrameType::Hello,
13899                    control_flags(),
13900                    0,
13901                    0,
13902                    connection,
13903                    serde_json::to_vec(&body).unwrap(),
13904                )
13905                .unwrap();
13906                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
13907                modules.insert(module_id.to_string(), (ctx, rx));
13908            }
13909            Rig {
13910                handler,
13911                forwarding,
13912                owner,
13913                _owner_rx: owner_rx,
13914                modules,
13915                generation: 0,
13916                next_connection: 100,
13917                _supervisor: supervisor,
13918            }
13919        }
13920
13921        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
13922            ScopeCarrier::new(Principal::Reserved {
13923                module_id: module_id.to_string(),
13924            })
13925            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
13926        }
13927
13928        /// The scope most tests open under: aft carries to any module, broca
13929        /// only to plexus and other, and the owner delegates as agent-1.
13930        fn session(scope_epoch: u64) -> ScopeRecord {
13931            let mut record = head("s", scope_epoch);
13932            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
13933            record.attributes.agent_id = Some("agent-1".to_string());
13934            record.attributes.delegates = true;
13935            record
13936        }
13937
13938        impl Rig {
13939            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
13940                self.generation += 1;
13941                sync(&self.handler, &self.owner, self.generation, scopes)
13942                    .await
13943                    .expect("the owner's sync is accepted");
13944            }
13945
13946            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13947                ScopeSelector {
13948                    owner: Principal::Reserved {
13949                        module_id: OWNER.to_string(),
13950                    },
13951                    scope_ref: scope_ref.to_string(),
13952                    scope_epoch,
13953                }
13954            }
13955
13956            fn open_frame(
13957                &mut self,
13958                opener: Option<&str>,
13959                target: &str,
13960                scope: Option<ScopeSelector>,
13961            ) -> (
13962                RouteCtx,
13963                mpsc::Receiver<crate::router::OutboundFrame>,
13964                Frame,
13965            ) {
13966                self.next_connection += 1;
13967                let (ctx, rx) = wide_ctx(self.next_connection);
13968                let root = unique_project_root("scoped-open");
13969                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13970                    target: RouteTarget::ToolProvider {
13971                        module_id: target.to_string(),
13972                    },
13973                    identity: BindIdentity::new(
13974                        root.path().to_path_buf(),
13975                        "unit".to_string(),
13976                        "session".to_string(),
13977                    ),
13978                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
13979                        module_id: module_id.to_string(),
13980                        launch_nonce: nonce(module_id),
13981                    }),
13982                    consumer_capabilities: None,
13983                    role_versions: None,
13984                    admission_facts: None,
13985                    scope,
13986                })
13987                .unwrap();
13988                let frame = Frame::build(
13989                    FrameType::Request,
13990                    control_flags(),
13991                    0,
13992                    0,
13993                    self.next_connection,
13994                    body,
13995                )
13996                .unwrap();
13997                (ctx, rx, frame)
13998            }
13999
14000            /// Open and expect a refusal before anything is relayed.
14001            async fn refused(
14002                &mut self,
14003                opener: Option<&str>,
14004                target: &str,
14005                scope: Option<ScopeSelector>,
14006            ) -> String {
14007                self.refusal_body(opener, target, scope).await["code"]
14008                    .as_str()
14009                    .unwrap()
14010                    .to_string()
14011            }
14012
14013            async fn refusal_body(
14014                &mut self,
14015                opener: Option<&str>,
14016                target: &str,
14017                scope: Option<ScopeSelector>,
14018            ) -> Value {
14019                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
14020                let replies = tokio::time::timeout(
14021                    Duration::from_secs(2),
14022                    self.handler.handle_control_frame(&ctx, frame),
14023                )
14024                .await
14025                .expect("the open must be refused before waiting for a bind ack")
14026                .unwrap();
14027                assert_eq!(replies.len(), 1, "{replies:?}");
14028                assert_eq!(replies[0].header.ty, FrameType::Error);
14029                let (_, module_rx) = self.modules.get_mut(target).unwrap();
14030                assert!(
14031                    module_rx.try_recv().is_err(),
14032                    "a refused open relays nothing"
14033                );
14034                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
14035                parse_error(&replies[0])
14036            }
14037
14038            /// Start an open and return its task and the bind the target got.
14039            async fn relayed(
14040                &mut self,
14041                opener: Option<&str>,
14042                target: &str,
14043                scope: Option<ScopeSelector>,
14044            ) -> Relayed {
14045                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
14046                let handler = self.handler.clone();
14047                let task_ctx = ctx.clone();
14048                let task = tokio::spawn(async move {
14049                    handler
14050                        .handle_control_frame(&task_ctx, frame)
14051                        .await
14052                        .unwrap()
14053                });
14054                let (_, module_rx) = self.modules.get_mut(target).unwrap();
14055                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
14056                    .await
14057                    .expect("the target receives the relayed route.bind")
14058                    .unwrap()
14059                    .frame;
14060                Relayed {
14061                    target: target.to_string(),
14062                    client: ctx,
14063                    client_rx: rx,
14064                    task,
14065                    bind,
14066                }
14067            }
14068
14069            async fn ack(&self, relayed: &Relayed) {
14070                let (module, _) = &self.modules[&relayed.target];
14071                self.handler
14072                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
14073                    .await
14074                    .unwrap();
14075            }
14076
14077            /// Open, ack and return the bound route.
14078            async fn bound(
14079                &mut self,
14080                opener: Option<&str>,
14081                target: &str,
14082                scope: Option<ScopeSelector>,
14083            ) -> Bound {
14084                let relayed = self.relayed(opener, target, scope).await;
14085                self.ack(&relayed).await;
14086                let Relayed {
14087                    target,
14088                    client,
14089                    mut client_rx,
14090                    task,
14091                    bind,
14092                } = relayed;
14093                assert!(
14094                    task.await.unwrap().is_empty(),
14095                    "the open is answered by commit"
14096                );
14097                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
14098                Bound {
14099                    target,
14100                    client,
14101                    client_rx,
14102                    channel,
14103                    epoch,
14104                    bind,
14105                }
14106            }
14107
14108            fn live(&self, route: &Bound) -> bool {
14109                matches!(
14110                    self.forwarding
14111                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14112                        .unwrap(),
14113                    DataRoute::Client(DataRouteState::Bound(_))
14114                )
14115            }
14116        }
14117
14118        struct Relayed {
14119            target: String,
14120            client: RouteCtx,
14121            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14122            task: tokio::task::JoinHandle<Vec<Frame>>,
14123            bind: Frame,
14124        }
14125
14126        struct Bound {
14127            target: String,
14128            client: RouteCtx,
14129            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14130            channel: u16,
14131            epoch: u32,
14132            bind: Frame,
14133        }
14134
14135        impl Bound {
14136            /// The reason of the `route.closed` this client was sent, after
14137            /// checking it also got a GOODBYE on exactly this route.
14138            fn closed_reason(&mut self) -> RouteCloseReason {
14139                let mut reason = None;
14140                let mut goodbye = false;
14141                while let Ok(outbound) = self.client_rx.try_recv() {
14142                    let frame = outbound.frame;
14143                    match frame.header.ty {
14144                        FrameType::Goodbye => {
14145                            assert_eq!(
14146                                (frame.header.channel, frame.header.epoch),
14147                                (self.channel, self.epoch)
14148                            );
14149                            goodbye = true;
14150                        }
14151                        FrameType::Push => {
14152                            let ClientControlPush::RouteClosed {
14153                                reason: r,
14154                                module_id,
14155                                ..
14156                            } = serde_json::from_slice(&frame.body).unwrap()
14157                            else {
14158                                panic!("unexpected push");
14159                            };
14160                            assert_eq!(module_id, self.target);
14161                            reason = Some(r);
14162                        }
14163                        other => panic!("unexpected frame {other:?}"),
14164                    }
14165                }
14166                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
14167                reason.expect("the client is told why the route closed")
14168            }
14169
14170            fn untouched(&mut self) -> bool {
14171                self.client_rx.try_recv().is_err()
14172            }
14173
14174            fn stamp(&self) -> Option<ScopeStamp> {
14175                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
14176                    ModuleControlRequest::RouteBind { scope, .. } => scope,
14177                    other => panic!("expected a route.bind, got {other:?}"),
14178                }
14179            }
14180        }
14181
14182        #[tokio::test]
14183        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
14184        ) {
14185            let mut rig = rig().await;
14186            let mut record = session(1);
14187            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
14188            record.child_owners = vec![Principal::Reserved {
14189                module_id: MAGIC.to_string(),
14190            }];
14191            rig.sync(vec![record]).await;
14192            let scope = || Some(rig_selector("s", Some(1)));
14193
14194            // Admitted: the owner, a bare carrier to any module, a targeted
14195            // carrier to its listed module.
14196            rig.bound(Some(OWNER), PLEXUS, scope()).await;
14197            rig.bound(Some(AFT), OTHER, scope()).await;
14198            rig.bound(Some(BROCA), PLEXUS, scope()).await;
14199
14200            // Refused scope_not_carrier: a targeted carrier to an unlisted
14201            // module, a module that is not listed at all (a child owner is not
14202            // a carrier), and a direct key-holder.
14203            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
14204                assert_eq!(
14205                    rig.refused(opener, target, scope()).await,
14206                    error_codes::SCOPE_NOT_CARRIER,
14207                    "{opener:?} -> {target}"
14208                );
14209            }
14210        }
14211
14212        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14213            ScopeSelector {
14214                owner: Principal::Reserved {
14215                    module_id: OWNER.to_string(),
14216                },
14217                scope_ref: scope_ref.to_string(),
14218                scope_epoch,
14219            }
14220        }
14221
14222        #[tokio::test]
14223        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
14224            let mut rig = rig().await;
14225            rig.sync(vec![session(1)]).await;
14226            for opener in [OWNER, AFT] {
14227                assert_eq!(
14228                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
14229                        .await,
14230                    error_codes::SCOPE_EPOCH_REQUIRED,
14231                    "{opener}"
14232                );
14233            }
14234        }
14235
14236        #[tokio::test]
14237        async fn admission_separates_not_synced_not_live_and_ended() {
14238            let mut rig = rig().await;
14239            // Before the configured owner's first sync: retryable.
14240            let code = rig
14241                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14242                .await;
14243            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
14244            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
14245
14246            // An owner that is not configured will never sync: terminal.
14247            let ghost = ScopeSelector {
14248                owner: Principal::Reserved {
14249                    module_id: "ghost".to_string(),
14250                },
14251                scope_ref: "s".to_string(),
14252                scope_epoch: Some(1),
14253            };
14254            assert_eq!(
14255                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
14256                error_codes::SCOPE_NOT_LIVE
14257            );
14258
14259            rig.sync(vec![session(2)]).await;
14260            assert_eq!(
14261                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
14262                    .await,
14263                error_codes::SCOPE_NOT_LIVE
14264            );
14265            for epoch in [1, 3] {
14266                assert_eq!(
14267                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
14268                        .await,
14269                    error_codes::SCOPE_ENDED,
14270                    "epoch {epoch}"
14271                );
14272            }
14273            // Control: the live epoch is admitted.
14274            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
14275                .await;
14276        }
14277
14278        #[tokio::test]
14279        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
14280            let mut rig = rig().await;
14281            rig.sync(vec![session(1)]).await;
14282            let route = rig
14283                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14284                .await;
14285            let stamp = route.stamp().expect("a scoped bind carries the stamp");
14286            assert_eq!(stamp.scope_ref, "s");
14287            assert_eq!(stamp.scope_epoch, 1);
14288            assert_eq!(stamp.kind, ScopeKind::Head);
14289            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
14290            assert!(stamp.attributes.delegates);
14291            assert!(stamp.owner_authorized);
14292            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14293            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
14294
14295            // broca owns a scope of its own on its own module connection; it is
14296            // not in scope_authority_owners, so its stamp is not authorized.
14297            let (broca, mut broca_rx) = wide_ctx(50);
14298            hello_via_sink(
14299                &rig.handler,
14300                &broca,
14301                &mut broca_rx,
14302                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
14303            )
14304            .await;
14305            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
14306                .await
14307                .unwrap();
14308            let own = ScopeSelector {
14309                owner: Principal::Reserved {
14310                    module_id: BROCA.to_string(),
14311                },
14312                scope_ref: "b".to_string(),
14313                scope_epoch: Some(1),
14314            };
14315            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
14316            assert!(!route.stamp().unwrap().owner_authorized);
14317        }
14318
14319        #[tokio::test]
14320        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
14321            let mut rig = rig().await;
14322            let mut record = head("s", 1);
14323            record.carriers = vec![carrier(AFT, None)];
14324            let flow_id = "Flow:run-7/step_2!~";
14325            record.attributes.flow_id = Some(flow_id.to_string());
14326            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
14327                .await
14328                .unwrap();
14329            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14330                panic!("not a sync reply");
14331            };
14332            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
14333            let route = rig
14334                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14335                .await;
14336            assert!(rig.live(&route), "the stamped bind committed");
14337            let stamp = route.stamp().expect("a flow scope carries a stamp");
14338            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
14339            assert_eq!(stamp.attributes.agent_id, None);
14340            assert!(!stamp.attributes.delegates);
14341            assert!(stamp.owner_authorized);
14342            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14343            assert_eq!(unscoped.stamp(), None);
14344        }
14345
14346        #[tokio::test]
14347        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
14348            let mut rig = rig_with_flow_support(false).await;
14349            let mut record = session(1);
14350            record.attributes.flow_id = Some("flow:7".to_string());
14351            rig.sync(vec![record]).await;
14352            for opener in [OWNER, AFT] {
14353                let body = rig
14354                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14355                    .await;
14356                assert_eq!(body["code"], "target_flow_unsupported");
14357                let message = body["message"].as_str().unwrap();
14358                for required in [PLEXUS, "flow-scopes/v1"] {
14359                    assert!(message.contains(required), "{message}");
14360                }
14361            }
14362        }
14363
14364        #[tokio::test]
14365        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
14366            let mut rig = rig_with_flow_support(true).await;
14367            let mut record = session(1);
14368            record.attributes.flow_id = Some("flow:7".to_string());
14369            rig.sync(vec![record]).await;
14370            for opener in [OWNER, AFT] {
14371                let route = rig
14372                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14373                    .await;
14374                assert!(rig.live(&route));
14375                assert_eq!(
14376                    route.stamp().unwrap().attributes.flow_id.as_deref(),
14377                    Some("flow:7")
14378                );
14379            }
14380        }
14381
14382        #[tokio::test]
14383        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
14384            use std::future::Future;
14385
14386            let mut rig = rig_with_flow_support(true).await;
14387            let mut record = session(1);
14388            record.attributes.flow_id = Some("flow:7".to_string());
14389            rig.sync(vec![record]).await;
14390            let (client, mut client_rx, frame) =
14391                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
14392            // Hold the route-open response permit so admission sees the first
14393            // target but relay reservation cannot capture an endpoint yet.
14394            for _ in 0..64 {
14395                client.egress.try_send(route_bind_ack(1)).unwrap();
14396            }
14397            let handler = rig.handler.clone();
14398            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
14399            std::future::poll_fn(|cx| {
14400                assert!(open.as_mut().poll(cx).is_pending());
14401                std::task::Poll::Ready(())
14402            })
14403            .await;
14404            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14405
14406            let old_connection = rig.modules[PLEXUS].0.connection_id;
14407            rig.handler.cleanup_connection(old_connection).unwrap();
14408            let (replacement, mut replacement_rx) = wide_ctx(200);
14409            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
14410            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14411            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14412            let hello = Frame::build(
14413                FrameType::Hello,
14414                control_flags(),
14415                0,
14416                0,
14417                200,
14418                serde_json::to_vec(&body).unwrap(),
14419            )
14420            .unwrap();
14421            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
14422
14423            client_rx.try_recv().unwrap();
14424            let replies = tokio::time::timeout(Duration::from_secs(2), open)
14425                .await
14426                .expect("the replacement is refused without waiting for a bind ack")
14427                .unwrap();
14428            assert_eq!(replies.len(), 1);
14429            let body = parse_error(&replies[0]);
14430            assert_eq!(body["code"], "target_flow_unsupported");
14431            for required in [PLEXUS, "flow-scopes/v1"] {
14432                assert!(body["message"].as_str().unwrap().contains(required));
14433            }
14434            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
14435            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
14436            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14437            assert!(rig
14438                .handler
14439                .registry
14440                .get_module_by_connection(replacement.connection_id)
14441                .unwrap()
14442                .is_some());
14443        }
14444
14445        #[tokio::test]
14446        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
14447        {
14448            let mut rig = rig_with_flow_support(false).await;
14449            rig.sync(vec![session(1)]).await;
14450            let route = rig
14451                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14452                .await;
14453            assert!(rig.live(&route));
14454            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
14455            let mut record = session(1);
14456            record.attributes.flow_id = Some("flow:7".to_string());
14457            rig.sync(vec![record]).await;
14458            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
14459            assert!(rig.live(&unscoped));
14460            assert_eq!(unscoped.stamp(), None);
14461        }
14462
14463        #[tokio::test]
14464        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
14465            let mut rig = rig().await;
14466            let mut record = session(1);
14467            record.attributes.flow_id = Some("flow:7".to_string());
14468            rig.sync(vec![record.clone()]).await;
14469            let before = rig.forwarding.published_scope_tag(OWNER, "s");
14470            let mut owner_route = rig
14471                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14472                .await;
14473            let mut carrier_route = rig
14474                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
14475                .await;
14476            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14477            rig.sync(vec![record.clone()]).await;
14478            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14479            assert!(rig.live(&owner_route) && owner_route.untouched());
14480            assert!(rig.live(&carrier_route) && carrier_route.untouched());
14481
14482            record.attributes.flow_id = Some("flow:8".to_string());
14483            rig.sync(vec![record]).await;
14484            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
14485            let before = before.unwrap();
14486            assert_eq!(after.scope_epoch, before.scope_epoch);
14487            assert!(after.version > before.version);
14488            for route in [&mut owner_route, &mut carrier_route] {
14489                assert!(!rig.live(route));
14490                assert_eq!(
14491                    route.closed_reason(),
14492                    RouteCloseReason::ScopeDelegationChanged
14493                );
14494            }
14495            assert!(rig.live(&unscoped) && unscoped.untouched());
14496            // Each provider also receives a GOODBYE for its drained route;
14497            // consume it before expecting the next route.bind on that sink.
14498            for target in [PLEXUS, OTHER] {
14499                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
14500                let goodbye = module_rx
14501                    .try_recv()
14502                    .expect("the provider sees the drain")
14503                    .frame;
14504                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14505                assert!(module_rx.try_recv().is_err());
14506            }
14507            let rebound = rig
14508                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14509                .await;
14510            assert_eq!(
14511                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
14512                Some("flow:8")
14513            );
14514        }
14515
14516        /// The owner's sync lands between admission and the module's ack. The
14517        /// open is refused by name, the module's other routes stay up, and the
14518        /// reserved pair is released. Changed content is retryable; an ended
14519        /// scope is not.
14520        #[tokio::test]
14521        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
14522            let mut rig = rig().await;
14523            rig.sync(vec![session(1)]).await;
14524            let mut cotenant = rig
14525                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14526                .await;
14527
14528            let mut changed = session(1);
14529            changed.child_owners.push(Principal::Reserved {
14530                module_id: MAGIC.to_string(),
14531            });
14532            let mut ended = None;
14533            for (code, next) in [
14534                (error_codes::SCOPE_CHANGED, vec![changed]),
14535                (error_codes::SCOPE_ENDED, Vec::new()),
14536            ] {
14537                let relayed = rig
14538                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14539                    .await;
14540                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
14541                ended = Some(next.is_empty());
14542                rig.sync(next).await;
14543                rig.ack(&relayed).await;
14544                let replies = relayed.task.await.unwrap();
14545                assert_eq!(replies.len(), 1, "{replies:?}");
14546                assert_eq!(parse_error(&replies[0])["code"], code);
14547                assert_eq!(
14548                    subc_protocol::error_codes::is_retryable_route_open(code),
14549                    code == error_codes::SCOPE_CHANGED
14550                );
14551                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14552                // The module is told to drop just the binding it created.
14553                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14554                // Collected, because ending the scope also closes the co-tenant
14555                // route, whose GOODBYE comes first.
14556                let mut goodbyes = Vec::new();
14557                while let Ok(outbound) = plexus_rx.try_recv() {
14558                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
14559                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
14560                }
14561                assert!(
14562                    goodbyes.contains(&(bind_channel, bind_epoch)),
14563                    "{goodbyes:?}"
14564                );
14565                assert!(rig
14566                    .handler
14567                    .registry
14568                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
14569                    .unwrap()
14570                    .is_some());
14571            }
14572            assert_eq!(ended, Some(true));
14573            // The co-tenant stayed up through the change, and closed only when
14574            // the scope ended, by the drain rule rather than by the commit.
14575            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
14576        }
14577
14578        /// Each row of the drain table on one set of routes: the owner's, a
14579        /// bare carrier's, and a targeted carrier's to each of its targets.
14580        #[tokio::test]
14581        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
14582            struct Case {
14583                name: &'static str,
14584                change: fn(&mut ScopeRecord),
14585                /// Closed routes by index: owner->plexus, aft->plexus,
14586                /// broca->plexus, broca->other.
14587                closed: [Option<RouteCloseReason>; 4],
14588            }
14589            use RouteCloseReason::*;
14590            let cases = [
14591                Case {
14592                    name: "a carrier entry removed",
14593                    change: |r| {
14594                        r.carriers.retain(|c| {
14595                            c.principal
14596                                != Principal::Reserved {
14597                                    module_id: AFT.to_string(),
14598                                }
14599                        })
14600                    },
14601                    closed: [None, Some(ScopeCarrierRemoved), None, None],
14602                },
14603                Case {
14604                    name: "a target removed from a carrier",
14605                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
14606                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
14607                },
14608                Case {
14609                    name: "a bare carrier narrowed to targets",
14610                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
14611                    closed: [None, Some(ScopeCarrierRemoved), None, None],
14612                },
14613                Case {
14614                    name: "delegates turned off",
14615                    change: |r| r.attributes.delegates = false,
14616                    closed: [Some(ScopeDelegationChanged); 4],
14617                },
14618                Case {
14619                    name: "agent_id changed",
14620                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
14621                    closed: [Some(ScopeDelegationChanged); 4],
14622                },
14623                Case {
14624                    name: "a carrier added, child owners changed, the record re-sent",
14625                    change: |r| {
14626                        r.carriers.push(carrier(MAGIC, None));
14627                        r.child_owners.push(Principal::Reserved {
14628                            module_id: MAGIC.to_string(),
14629                        });
14630                    },
14631                    closed: [None; 4],
14632                },
14633                Case {
14634                    name: "a target added",
14635                    change: |r| {
14636                        r.carriers[1]
14637                            .targets
14638                            .as_mut()
14639                            .unwrap()
14640                            .push("third".to_string())
14641                    },
14642                    closed: [None; 4],
14643                },
14644                Case {
14645                    name: "delegates turned on",
14646                    change: |r| r.attributes.delegates = true,
14647                    closed: [None; 4],
14648                },
14649            ];
14650            for case in cases {
14651                let mut rig = rig().await;
14652                rig.sync(vec![session(1)]).await;
14653                let scope = || Some(rig_selector("s", Some(1)));
14654                let mut routes = [
14655                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
14656                    rig.bound(Some(AFT), PLEXUS, scope()).await,
14657                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
14658                    rig.bound(Some(BROCA), OTHER, scope()).await,
14659                ];
14660                let mut record = session(1);
14661                (case.change)(&mut record);
14662                rig.sync(vec![record]).await;
14663                for (index, expected) in case.closed.iter().enumerate() {
14664                    let route = &mut routes[index];
14665                    match expected {
14666                        Some(reason) => {
14667                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
14668                            assert_eq!(
14669                                route.closed_reason(),
14670                                *reason,
14671                                "{}: route {index}",
14672                                case.name
14673                            );
14674                        }
14675                        None => {
14676                            assert!(rig.live(route), "{}: route {index} closed", case.name);
14677                            assert!(
14678                                route.untouched(),
14679                                "{}: route {index} was told something",
14680                                case.name
14681                            );
14682                        }
14683                    }
14684                }
14685            }
14686        }
14687
14688        #[tokio::test]
14689        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
14690            // Removed, and replaced by a higher epoch.
14691            for next in [Vec::new(), vec![session(2)]] {
14692                let mut rig = rig().await;
14693                rig.sync(vec![session(1)]).await;
14694                let mut route = rig
14695                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14696                    .await;
14697                rig.sync(next).await;
14698                assert!(!rig.live(&route));
14699                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
14700            }
14701
14702            // A child whose parent ends: its routes close as parent-ended, the
14703            // child stays live, and routes under the parent close as ended.
14704            let mut rig = rig().await;
14705            let mut child = session(1);
14706            child.scope_ref = "child".to_string();
14707            child.kind = ScopeKind::Worker;
14708            child.parent = Some(ScopeParent::new(
14709                Principal::Reserved {
14710                    module_id: OWNER.to_string(),
14711                },
14712                "s".to_string(),
14713                1,
14714            ));
14715            rig.sync(vec![session(1), child.clone()]).await;
14716            let mut child_route = rig
14717                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
14718                .await;
14719            assert_eq!(
14720                child_route.stamp().unwrap().parent_state,
14721                Some(ParentState::Linked)
14722            );
14723            rig.sync(vec![child]).await;
14724            assert!(!rig.live(&child_route));
14725            assert_eq!(
14726                child_route.closed_reason(),
14727                RouteCloseReason::ScopeParentEnded
14728            );
14729        }
14730
14731        #[tokio::test]
14732        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
14733        ) {
14734            let mut rig = rig().await;
14735            rig.sync(vec![session(1)]).await;
14736            let mut route = rig
14737                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14738                .await;
14739            let before = rig.forwarding.published_scope_tag(OWNER, "s");
14740            rig.sync(vec![session(1)]).await;
14741            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14742            assert!(rig.live(&route) && route.untouched());
14743
14744            // A call in flight on the route when another carrier is added. A
14745            // forwarded REQUEST holds one credit on the route's flow until the
14746            // module answers; the router takes it exactly like this.
14747            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
14748                .forwarding
14749                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14750                .unwrap()
14751            else {
14752                panic!("the route is bound");
14753            };
14754            binding.flow.acquire_tagged(9, false).await.unwrap();
14755            let mut widened = session(1);
14756            widened.carriers.push(carrier(MAGIC, None));
14757            rig.sync(vec![widened]).await;
14758            assert!(rig.live(&route) && route.untouched());
14759            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14760            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
14761            // The call's credit is still held on an open flow, so its answer
14762            // will be delivered: closing the route would have closed the flow.
14763            assert_eq!(binding.flow.in_flight(), 1);
14764            binding
14765                .flow
14766                .acquire_tagged(10, false)
14767                .await
14768                .expect("the flow is still open");
14769        }
14770
14771        /// A swap's superseded endpoint keeps its routes until drained; ending
14772        /// the scope closes them there too.
14773        #[tokio::test]
14774        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
14775            let mut rig = rig().await;
14776            rig.sync(vec![session(1)]).await;
14777            let mut on_incumbent = rig
14778                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14779                .await;
14780
14781            // Swap plexus: register a candidate and cut over, leaving the
14782            // incumbent superseded with the route still on it.
14783            let (candidate, _candidate_rx) = wide_ctx(9);
14784            let registration = rig
14785                .handler
14786                .registry
14787                .register_candidate_with_control_ops(
14788                    manifest(PLEXUS, PROTOCOL_VERSION),
14789                    PROTOCOL_VERSION,
14790                    candidate.connection_id,
14791                    module_baseline_control_ops(),
14792                )
14793                .unwrap();
14794            rig.forwarding
14795                .register_candidate_module_connection(
14796                    candidate.connection_id,
14797                    PLEXUS.to_string(),
14798                    PROTOCOL_VERSION,
14799                    manifest_concurrency(&registration.manifest),
14800                    candidate.egress.clone(),
14801                )
14802                .unwrap();
14803            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
14804            rig.handler
14805                .registry
14806                .promote_candidate(PLEXUS)
14807                .unwrap()
14808                .unwrap();
14809            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
14810
14811            rig.sync(Vec::new()).await;
14812            assert!(!rig.live(&on_incumbent));
14813            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
14814            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14815            let goodbye = incumbent_rx
14816                .try_recv()
14817                .expect("the superseded endpoint is told")
14818                .frame;
14819            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14820        }
14821    }
14822}
14823
14824#[cfg(test)]
14825mod concurrency_default_exposure_tests {
14826    use super::*;
14827
14828    fn hello_body(role_json: &str) -> Vec<u8> {
14829        format!(
14830            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
14831        )
14832        .into_bytes()
14833    }
14834
14835    fn manifest_from(body: &[u8]) -> ModuleManifest {
14836        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
14837        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
14838            .expect("manifest parses")
14839    }
14840
14841    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
14842
14843    #[test]
14844    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
14845        let body = hello_body(&format!(
14846            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
14847        ));
14848        let manifest = manifest_from(&body);
14849        // Precondition: serde really resolved it to the default, so the typed
14850        // manifest alone cannot answer the question this probe exists for.
14851        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14852        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
14853    }
14854
14855    #[test]
14856    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
14857        let body = hello_body(&format!(
14858            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
14859        ));
14860        let manifest = manifest_from(&body);
14861        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14862        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14863    }
14864
14865    #[test]
14866    fn non_management_roles_are_never_reported() {
14867        let body = hello_body(
14868            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
14869        );
14870        let manifest = manifest_from(&body);
14871        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14872    }
14873}