Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
26        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
27        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
28    },
29    scope::{
30        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32        SCOPE_SYNC_OP,
33    },
34    session::{
35        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38        ROLE_VERSIONS_FIELD,
39    },
40    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41    Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47    caller_identity::CallerIdentity,
48    capability_requirements::{
49        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51        RuntimeModule,
52    },
53    child_roster::DaemonShutdownFlag,
54    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55    forwarding::{
56        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59    },
60    observability::{
61        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62    },
63    privileged_prompt::{prompt_text, PrivilegedOperation},
64    provenance::{
65        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66    },
67    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68    router::{RouteCtx, RouterError},
69    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71    stderr_tail::{CaptureState, TailEntry},
72    supervise::{
73        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74        SupervisorHandle, SwapHelloAdmission,
75    },
76    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79/// Lowest envelope version this subc build will negotiate.
80///
81/// Module HELLO negotiation is exact: peers must use the daemon's locked
82/// protocol version. Older and newer peers receive `version_unsupported` and
83/// are not registered.
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93    ops::ACTIVITY_RECORD,
94    ops::ACTIVITY_QUERY,
95    ops::ACTIVITY_SUBSCRIBE,
96    ops::SERVER_DESCRIBE,
97    ops::CATALOG_LIST,
98    ops::ROUTE_OPEN,
99    ops::ROUTE_POLL,
100    ops::ROUTE_CLOSING,
101    ops::ROUTE_CLOSED,
102    ops::SUPERVISOR_LIST,
103    ops::SUPERVISOR_RESTART,
104    ops::SUPERVISOR_SWAP,
105    ops::SUPERVISOR_RELOAD,
106    ops::SUPERVISOR_RESCAN,
107    ops::SUPERVISOR_RELEASE_RESERVED,
108    ops::SUPERVISOR_SET_ENABLED,
109    ops::SUPERVISOR_HEALTH_PROBE,
110    ops::SUPERVISOR_HEALTH,
111    ops::SUPERVISOR_STDERR_TAIL,
112    ops::SUPERVISOR_TERMINALS,
113    ops::SUPERVISOR_ROUTES,
114    ops::SUPERVISOR_PROVENANCE,
115    ops::SUPERVISOR_SPAWN_SNAPSHOT,
116    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
120/// announced by a capability in `server.describe` instead, and clients check
121/// that capability before sending it. `ck daemon --json` prints `subc_ops`
122/// verbatim and scripts read that output, so a new name there would change it.
123/// A malformed body for one of these is still refused `invalid_control_body`
124/// rather than as an unknown op.
125const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132    "supervisor.live_roots",
133    SCOPE_SYNC_OP,
134    SCOPE_APPLY_OP,
135    SCOPE_DESCRIBE_OP,
136    "operator.confirm",
137];
138
139/// Module-originated ops the daemon answers but does not advertise in
140/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
141/// belongs to is incomplete, so no module is told it works before it does.
142const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146/// How long subc waits for a module to ack a relayed route.bind before returning
147/// `module_timeout`. The ack waits on the module's own configure, which for AFT
148/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
149/// and DB-open work — on a cold page cache or a large repo that legitimately
150/// exceeds a couple of seconds. The default is generous because rejecting a VALID
151/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
152/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
153pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155/// How many CONSECUTIVE full-budget relay timeouts against one target module
156/// open that module's bind-relay breaker.
157///
158/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
159/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
160/// so three consecutive full-budget timeouts take ~36s to observe: every client
161/// whose open contributed to opening the breaker had already given up on its
162/// own. That is what makes opening the breaker unable to turn a call that would
163/// have succeeded into a refusal — it can only make an already-failing module
164/// fail faster.
165///
166/// Two would be reachable inside one default deadline. One would convict a
167/// module on a single cold-cache bind, which is exactly the valid-but-slow case
168/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
169pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171/// How long a module's bind-relay breaker stays open before exactly one
172/// `route.open` is let through as a probe.
173///
174/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
175/// re-pays a full-budget stall almost continuously, and the breaker stops being
176/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
177/// deadline: a client that starts retrying after the module recovers has to get
178/// a probe opportunity inside its own deadline, or the breaker converts a
179/// recovered module into a failed call — the failure it exists to prevent,
180/// pointed the other way.
181///
182/// 20s sits between those with room on both sides, and it caps what a wedged
183/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
184/// rather than one per `route.open` per connection. The stall that motivated
185/// this, with its measurements, is written up in
186/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
187/// waited the whole budget out.
188pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194    configured: &Path,
195    spawned_from: Option<&Path>,
196    image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198    let path = match spawned_from {
199        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200            ReloadPathAgreement::Match
201        }
202        Some(spawned_from) => ReloadPathAgreement::Mismatch {
203            configured: configured.to_path_buf(),
204            spawned_from: spawned_from.to_path_buf(),
205        },
206        None => ReloadPathAgreement::Unavailable {
207            reason: if matches!(
208                image,
209                subc_control::RunningImageAgreement::Unavailable {
210                    reason: subc_control::RunningImageUnavailableReason::NotRunning
211                }
212            ) {
213                ReloadPathUnavailableReason::NotRunning
214            } else {
215                ReloadPathUnavailableReason::SpawnedPathUnavailable
216            },
217        },
218    };
219    PendingReloadVerdict {
220        path,
221        image,
222        spec: None,
223    }
224}
225
226fn launch_spec_agreement(
227    configured: &crate::supervise::ModuleSpec,
228    running: bool,
229    spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232    if !running {
233        return Agreement::Unavailable {
234            reason: Reason::NotRunning,
235        };
236    }
237    match spawned {
238        None => Agreement::Unavailable {
239            reason: Reason::SpawnedSpecUnavailable,
240        },
241        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242            Agreement::Match
243        }
244        Some(_) => Agreement::Mismatch,
245    }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249    if configured == spawned_from {
250        return true;
251    }
252    #[cfg(windows)]
253    {
254        // Windows spawn records the resolved verbatim path (\\?\...), while
255        // configuration keeps its original spelling. Resolve both with the OS
256        // instead of stripping prefixes or folding case, which could conflate
257        // distinct names in case-sensitive directories. Different resolved
258        // paths still differ even if their files have identical content.
259        match (
260            std::fs::canonicalize(configured),
261            std::fs::canonicalize(spawned_from),
262        ) {
263            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264            _ => false,
265        }
266    }
267    #[cfg(not(windows))]
268    {
269        false
270    }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275    build: DaemonBuildProvenance,
276    pid: Option<u32>,
277    started_at_ms: Option<u64>,
278    start_clock: Option<crate::clock::StartClock>,
279    executable_path: Option<PathBuf>,
280    executable_identity: Option<SpawnedFileIdentity>,
281    process_start_time: Option<u64>,
282    probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286    fn default() -> Self {
287        Self {
288            build: DaemonBuildProvenance {
289                build_git_sha: None,
290                build_lock_digest: None,
291            },
292            pid: None,
293            started_at_ms: None,
294            start_clock: None,
295            executable_path: None,
296            executable_identity: None,
297            process_start_time: None,
298            probe: ExecutableIdentityProbe::default(),
299        }
300    }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305    supervisor: Supervisor,
306    config_path: PathBuf,
307    configured_port: Option<u16>,
308    storage_config: Option<crate::daemon_config::StorageConfig>,
309    admission_facts_carrier_module_id: Option<String>,
310    admission_facts_targets: Option<Vec<String>>,
311    scope_authority_owners: Vec<String>,
312    privileged_operator_prompt: PrivilegedOperatorPrompt,
313    /// Counts executed rescans that set out to change the supervised modules'
314    /// staged or applied configuration, counted before the first change so a
315    /// rescan that fails partway still counts. With the daemon incarnation it
316    /// names the state a `supervisor.rescan_checked` preview was computed
317    /// against.
318    generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321/// Refusal labels passed to `observe_route_open_refusal` that mean the target
322/// module is not serving right now, and so open or extend an outage in the
323/// route outage tracker. Every one of them is only reachable after the target
324/// was found in the registry, which is what keeps an arbitrary client-chosen
325/// id from ever creating tracker state.
326///
327/// Deliberately absent: `not_registered` and `removed` (the id may be
328/// anything a client sent, and a removed module is gone on purpose),
329/// `protocol_none` (such a module never serves routes, so nothing is out),
330/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
331/// capability and admission-facts refusals (they refuse the caller, not a
332/// module outage), and `relay_reservation_failed` (its code ranges over
333/// capacity limits as well as a vanished connection). Capacity, breaker,
334/// relay-timeout and module-rejection refusals do not pass through that
335/// function at all; the breaker logs its own transitions.
336///
337/// The two not-serving refusals that bypass that function record themselves
338/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
339/// `required_capability_unprovided` is not tracked: the module itself is up,
340/// and the outage belongs to the missing provider.
341const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342    "reloading",
343    "supervisor_not_live",
344    "registration_not_active",
345    "no_forwarding_connection",
346    "relay_send_failed",
347];
348
349/// Real channel-0 control handler for subc itself.
350#[derive(Clone)]
351pub struct ControlHandler {
352    registry: Arc<Registry>,
353    activity: Option<crate::activity::ActivityLedger>,
354    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355    forwarding: Arc<ForwardingTable>,
356    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357    supervisor: SupervisorHandle,
358    subc_capabilities: Arc<[String]>,
359    /// Daemon-wide route.bind relay budget. Used as the fallback when the
360    /// target module has no per-module override in
361    /// `route_bind_relay_timeouts`.
362    route_bind_relay_timeout: Duration,
363    /// Per-module route.bind relay budget overrides, keyed by module id. When
364    /// `handle_route_open` resolves the deadline for a target module, a
365    /// per-module entry wins over the daemon-wide value above.
366    route_bind_relay_timeouts: BTreeMap<String, Duration>,
367    /// Per-target-module bind-relay breaker state. Shared with the forwarding
368    /// table, which is where a new module connection resets it.
369    route_bind_breakers: RouteBindBreakers,
370    /// Live relay admissions keyed by target module. Shared through the
371    /// forwarding table so cloned or separately built handlers enforce one cap.
372    route_bind_concurrency: RouteBindConcurrency,
373    /// Start and end of each module's not-serving period as seen by
374    /// `route.open`, so an outage gets one line at each edge instead of only
375    /// the per-refusal INFO lines. Taken from the forwarding table, so every
376    /// handler built over one table shares it.
377    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378    /// Consecutive relay timeouts that open a module's breaker.
379    route_bind_breaker_threshold: u32,
380    /// How long a breaker stays open before one probe is admitted.
381    route_bind_breaker_cooldown: Duration,
382    health_probe_timeout: Duration,
383    /// Central storage policy. When set, each registering module receives its
384    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
385    storage_config: Option<crate::daemon_config::StorageConfig>,
386    /// The machine id established at boot, served on every HELLO_ACK and on
387    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
388    /// changes the file, never this value. `None` serves no id.
389    machine_id: Option<crate::machine_id::MachineId>,
390    admission_facts_carrier_module_id: Option<String>,
391    admission_facts_targets: Option<Vec<String>>,
392    /// Scope records with their sync authorities and tombstones; see
393    /// `crate::scopes`. Shared by clones of this handler, so every connection
394    /// reads and writes one table.
395    scopes: Arc<RwLock<ScopeTable>>,
396    /// Wakes the expiry loop when an accepted scope change may have added a
397    /// deadline, so the loop can wait without ticking while no live scope has
398    /// one. A permit is stored when nothing waits, so no wake is lost.
399    scope_deadline_added: Arc<tokio::sync::Notify>,
400    /// The configured `scope_authority_owners`, kept so a rescan can report a
401    /// changed value as needing a daemon restart; rescan never applies it.
402    scope_authority_owners: Vec<String>,
403    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
404    privileged_operator_prompt: PrivilegedOperatorPrompt,
405    /// The local and peer address of each accepted TCP client connection, read
406    /// from the daemon's own socket, so a privileged request can name its caller.
407    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408    #[cfg(test)]
409    caller_lookup: Option<TestCallerLookup>,
410    /// The launch nonce each module connection presented at HELLO, which is how
411    /// a `scope.sync` is matched to the owner's current launch.
412    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413    rescan: Option<SupervisorRescanContext>,
414    connected_clients: ConnectedClients,
415    counters: DaemonCounters,
416    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
418    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
419    /// their usual severity (see [`log_requirement_events`]). A handler built
420    /// without one has a flag that is never set.
421    daemon_shutdown: DaemonShutdownFlag,
422    daemon_provenance: DaemonProvenanceFacts,
423    #[cfg(test)]
424    control_dispatch_delay: Option<Duration>,
425    #[cfg(test)]
426    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427    /// Holds `route.open` after the route.bind is queued to the module and
428    /// before it is marked enqueued, so a test can make the module's reply (or
429    /// a client teardown) land exactly inside that window.
430    #[cfg(test)]
431    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432    /// Holds a HELLO after its registry entry is reserved and before its
433    /// forwarding endpoint is installed, so a test can read the catalog and
434    /// open routes while the module is registered but not yet routable. A
435    /// blocking barrier because the HELLO handler is synchronous; the test
436    /// runs the HELLO on its own thread.
437    #[cfg(test)]
438    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443        f.debug_struct("ControlHandler")
444            .field("registry", &self.registry)
445            .field("forwarding", &self.forwarding)
446            .field("process_liveness", &self.process_liveness.is_some())
447            .field("supervisor", &self.supervisor)
448            .field("subc_capabilities", &self.subc_capabilities)
449            .finish()
450    }
451}
452
453struct RouteOpenRequest {
454    target: RouteTarget,
455    identity: BindIdentity,
456    consumer_identity: Option<ConsumerIdentity>,
457    consumer_capabilities: Option<Vec<String>>,
458    role_versions: Option<BTreeMap<String, String>>,
459    admission_facts: Option<serde_json::Value>,
460    scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465    Admission,
466    ProjectRoot,
467    EgressReservation,
468    BindSend,
469    BindAckWait,
470}
471
472/// Store elapsed time for each route-open phase, including attempts that end
473/// in an early refusal or backend error. Admission also includes rechecks and
474/// commit bookkeeping performed between the other timed phases.
475struct RouteOpenTimings {
476    module_id: String,
477    principal: String,
478    admission: Duration,
479    project_root: Duration,
480    egress_reservation: Duration,
481    bind_send: Duration,
482    bind_ack_wait: Duration,
483    phase: Option<RouteOpenPhase>,
484    phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488    fn new() -> Self {
489        Self {
490            module_id: String::new(),
491            // Refusals before attestation must not claim a verified principal.
492            principal: "unresolved".to_string(),
493            admission: Duration::ZERO,
494            project_root: Duration::ZERO,
495            egress_reservation: Duration::ZERO,
496            bind_send: Duration::ZERO,
497            bind_ack_wait: Duration::ZERO,
498            phase: Some(RouteOpenPhase::Admission),
499            phase_started_at: Instant::now(),
500        }
501    }
502
503    fn finish(&mut self) {
504        let elapsed = self.phase_started_at.elapsed();
505        match self.phase.take() {
506            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511            None => {}
512        }
513    }
514
515    fn enter(&mut self, phase: RouteOpenPhase) {
516        self.finish();
517        self.phase = Some(phase);
518        self.phase_started_at = Instant::now();
519    }
520}
521
522struct RouteBindReservationGuard {
523    forwarding: Arc<ForwardingTable>,
524    endpoint: ModuleEndpointId,
525    relay_corr: u64,
526    armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530    forwarding: Arc<ForwardingTable>,
531    endpoint: ModuleEndpointId,
532    corr: u64,
533    armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538        Self {
539            forwarding,
540            endpoint,
541            corr,
542            armed: true,
543        }
544    }
545
546    fn disarm(&mut self) {
547        self.armed = false;
548    }
549}
550
551impl Drop for ModuleControlRpcGuard {
552    fn drop(&mut self) {
553        if self.armed {
554            let _ = self
555                .forwarding
556                .cancel_module_control_rpc(self.endpoint, self.corr);
557        }
558    }
559}
560
561impl RouteBindReservationGuard {
562    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563        Self {
564            forwarding,
565            endpoint,
566            relay_corr,
567            armed: true,
568        }
569    }
570
571    fn release_and_disarm(&mut self) {
572        if !self.armed {
573            return;
574        }
575        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576            self.endpoint,
577            self.relay_corr,
578            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579        ) {
580            send_goodbye_target_best_effort(
581                &self.forwarding.counters(),
582                &target,
583                "canceled route.bind",
584            );
585        }
586        self.armed = false;
587    }
588
589    fn disarm(&mut self) {
590        self.armed = false;
591    }
592}
593
594impl Drop for RouteBindReservationGuard {
595    fn drop(&mut self) {
596        self.release_and_disarm();
597    }
598}
599
600/// Per-target-module circuit breaker around the `route.bind` relay.
601///
602/// The connection reader is serial per connection, so a module whose `on_bind`
603/// sits on the ack blocks every LATER frame on the connections that call it,
604/// including calls to unrelated modules. This does not make any module's bind
605/// fast; it stops the daemon paying the full budget again and again for a
606/// condition it has already observed.
607///
608/// State is keyed by TARGET MODULE and shared by every connection: a wedged
609/// module wedges everyone, so what one connection learned should protect the
610/// rest.
611///
612/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
613/// relay to that module has actually timed out, and is removed again when a
614/// relay is accepted or the module reconnects, so it cannot grow with traffic
615/// or with modules that behave.
616///
617/// # Why a `std` mutex here is not the head-of-line defect again
618///
619/// Acquisition never awaits. The critical section is a hash lookup plus a few
620/// integer updates, with no I/O and no `.await` inside it, so a reader task
621/// cannot be descheduled behind it the way it can behind
622/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
623/// primitive, held for the same kind of work, as the refusal counter this very
624/// path already increments.
625///
626/// It is also NOT on the data-plane splice path: only `route.open` and module
627/// registration touch it, so bound-route frames gain no state check and no
628/// contention.
629#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636    modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640    concurrency: RouteBindConcurrency,
641    module_id: String,
642}
643
644impl RouteBindConcurrency {
645    /// Admit without waiting. Waiting here would move the bind stall from the
646    /// module reply to a semaphore and restore reader head-of-line blocking.
647    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648        let mut modules = self
649            .modules
650            .lock()
651            .expect("route.bind concurrency mutex poisoned");
652        let in_flight = modules.entry(module_id.to_string()).or_default();
653        if *in_flight >= limit {
654            return Err(*in_flight);
655        }
656        *in_flight += 1;
657        Ok(RouteBindConcurrencyGuard {
658            concurrency: self.clone(),
659            module_id: module_id.to_string(),
660        })
661    }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665    fn drop(&mut self) {
666        let mut modules = self
667            .concurrency
668            .modules
669            .lock()
670            .expect("route.bind concurrency mutex poisoned");
671        let remove = {
672            let in_flight = modules
673                .get_mut(&self.module_id)
674                .expect("admitted route.bind has a concurrency entry");
675            *in_flight -= 1;
676            *in_flight == 0
677        };
678        if remove {
679            modules.remove(&self.module_id);
680        }
681    }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686    /// Relay timeouts observed with no accepted relay in between.
687    consecutive_timeouts: u32,
688    /// `Some` while the breaker is open: the instant the cooldown expires and
689    /// the next arrival may probe. `None` means closed.
690    cooldown_until: Option<Instant>,
691    /// A half-open probe has been admitted and has not settled yet. This is
692    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
693    /// that read the cooldown, so concurrent opens arriving at the moment the
694    /// cooldown expires cannot all decide that they are the probe.
695    probe_in_flight: Option<Arc<()>>,
696}
697
698/// What the breaker decided for one `route.open`, before any relay work.
699enum RouteBindAdmission<'a> {
700    Admitted {
701        guard: RouteBindBreakerGuard<'a>,
702        /// This open is the single half-open probe, so the transition is worth
703        /// one log line.
704        probe: bool,
705    },
706    Refused {
707        consecutive_timeouts: u32,
708        /// What is left of the cooldown. Zero when the refusal is because the
709        /// one probe is already in flight rather than because the cooldown has
710        /// not elapsed.
711        retry_in: Duration,
712        probe_in_flight: bool,
713    },
714}
715
716/// An outstanding admission, which must be told how its relay settled.
717///
718/// `Drop` settles it as inconclusive, so an early return between admission and
719/// the relay -- or the whole handler being cancelled when the client
720/// disconnects -- releases a half-open probe slot instead of leaving the
721/// breaker wedged half-open with no further probes.
722struct RouteBindBreakerGuard<'a> {
723    breakers: RouteBindBreakers,
724    module_id: &'a str,
725    probe_token: Option<Arc<()>>,
726    settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730    /// The module answered within the budget and took the bind. THE ONLY
731    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
732    /// breaker, which is a transition worth logging.
733    fn record_accepted(&mut self) -> bool {
734        self.settled = true;
735        self.breakers.record_accepted(self.module_id)
736    }
737
738    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
739    /// COUNTS TOWARD OPENING.
740    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741        self.settled = true;
742        self.breakers.record_timeout(
743            self.module_id,
744            self.probe_token.as_ref(),
745            threshold,
746            cooldown,
747        )
748    }
749
750    /// Everything else: the module REJECTED the bind, its connection went away
751    /// mid-relay, or the waiter was cancelled.
752    ///
753    /// None of these is evidence that a module is slow, and each already has
754    /// its own refusal with its own code. A module that rejects a bind in
755    /// microseconds is healthy and must never be convicted for it; a module
756    /// that died has said nothing about the module that replaces it. So these
757    /// neither increment nor reset the count -- they only release a probe slot.
758    fn record_inconclusive(&mut self) {
759        self.settled = true;
760        self.breakers
761            .record_inconclusive(self.module_id, self.probe_token.as_ref());
762    }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766    fn drop(&mut self) {
767        if !self.settled {
768            self.breakers
769                .record_inconclusive(self.module_id, self.probe_token.as_ref());
770        }
771    }
772}
773
774/// The breaker moved to open, reported so the caller can log it outside the
775/// lock. Opening is rare and load-bearing; the refusals that follow are
776/// frequent and are counted rather than logged.
777struct BreakerOpened {
778    consecutive_timeouts: u32,
779    /// True when a failed probe re-opened an already-open breaker, which reads
780    /// very differently in a log from a first opening.
781    reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786        self.modules
787            .lock()
788            .expect("route.bind breaker mutex poisoned")
789    }
790
791    /// Decide whether this `route.open` may attempt its relay. Takes the map
792    /// lock and nothing else, and never awaits.
793    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795            probe: probe_token.is_some(),
796            guard: RouteBindBreakerGuard {
797                breakers: self.clone(),
798                module_id,
799                probe_token,
800                settled: false,
801            },
802        };
803
804        let mut modules = self.lock();
805        let Some(state) = modules.get_mut(module_id) else {
806            return admitted(None);
807        };
808        let Some(cooldown_until) = state.cooldown_until else {
809            return admitted(None);
810        };
811        if state.probe_in_flight.is_some() {
812            return RouteBindAdmission::Refused {
813                consecutive_timeouts: state.consecutive_timeouts,
814                retry_in: Duration::ZERO,
815                probe_in_flight: true,
816            };
817        }
818        let now = Instant::now();
819        if now < cooldown_until {
820            return RouteBindAdmission::Refused {
821                consecutive_timeouts: state.consecutive_timeouts,
822                retry_in: cooldown_until - now,
823                probe_in_flight: false,
824            };
825        }
826        let token = Arc::new(());
827        state.probe_in_flight = Some(Arc::clone(&token));
828        admitted(Some(token))
829    }
830
831    fn record_accepted(&self, module_id: &str) -> bool {
832        self.lock()
833            .remove(module_id)
834            .is_some_and(|state| state.cooldown_until.is_some())
835    }
836
837    fn record_timeout(
838        &self,
839        module_id: &str,
840        probe_token: Option<&Arc<()>>,
841        threshold: u32,
842        cooldown: Duration,
843    ) -> Option<BreakerOpened> {
844        let mut modules = self.lock();
845        let state = modules.entry(module_id.to_string()).or_default();
846        let was_open = state.cooldown_until.is_some();
847        let was_probe = Self::owns_probe(state, probe_token);
848        if was_probe {
849            state.probe_in_flight = None;
850        }
851        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852        if state.consecutive_timeouts < threshold {
853            return None;
854        }
855        state.cooldown_until = Some(Instant::now() + cooldown);
856        Some(BreakerOpened {
857            consecutive_timeouts: state.consecutive_timeouts,
858            reopened_after_probe: was_open && was_probe,
859        })
860    }
861
862    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863        // A relay can finish after the breaker was reset or after it opened
864        // again and started a new probe. Only the guard whose token matches the
865        // active probe may release it, so a late relay never frees a newer probe.
866        state
867            .probe_in_flight
868            .as_ref()
869            .zip(token)
870            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871    }
872
873    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874        if let Some(state) = self.lock().get_mut(module_id) {
875            if Self::owns_probe(state, probe_token) {
876                state.probe_in_flight = None;
877            }
878        }
879    }
880
881    /// Discard what was learned about a module, because the process it was
882    /// learned about is gone. Returns the discarded count when it was non-zero.
883    ///
884    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
885    /// `module_id` is a configuration identity that outlives any particular
886    /// child; what the breaker observed was the process behind the module
887    /// connection of the moment. When a new connection registers under that id
888    /// the verdict's subject no longer exists, so the verdict is stale by
889    /// construction rather than merely likely to be wrong. Keeping it would
890    /// apply a dead process's record to a live one, which is the same defect
891    /// class this breaker exists to stop the daemon committing.
892    ///
893    /// A half-open probe in flight is discarded with the rest: it was a
894    /// question about the old process.
895    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896        self.lock()
897            .remove(module_id)
898            .map(|state| state.consecutive_timeouts)
899            .filter(|discarded| *discarded > 0)
900    }
901
902    /// Open breakers, for the `server.describe` counters object. `None` when
903    /// none is open, so the key stays absent rather than present-and-empty.
904    ///
905    /// This is the operator's answer to "is this module refusing instantly or
906    /// is it fine?", which look identical from a client that retries and then
907    /// succeeds.
908    fn open_snapshot(&self) -> Option<serde_json::Value> {
909        let now = Instant::now();
910        let modules = self.lock();
911        let open = modules
912            .iter()
913            .filter_map(|(module_id, state)| {
914                let cooldown_until = state.cooldown_until?;
915                Some((
916                    module_id.clone(),
917                    serde_json::json!({
918                        "consecutive_timeouts": state.consecutive_timeouts,
919                        "cooldown_remaining_ms":
920                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
921                        "probe_in_flight": state.probe_in_flight.is_some(),
922                    }),
923                ))
924            })
925            .collect::<serde_json::Map<String, serde_json::Value>>();
926        (!open.is_empty()).then_some(serde_json::Value::Object(open))
927    }
928}
929
930impl ControlHandler {
931    pub fn new(registry: Arc<Registry>) -> Self {
932        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933    }
934
935    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936        let counters = forwarding.counters();
937        // Taken from the forwarding table rather than created here, so that the
938        // breaker a `route.open` consults is the same one a module's
939        // registration resets, however many handlers are built over one table.
940        let route_bind_breakers = forwarding.route_bind_breakers();
941        let route_bind_concurrency = forwarding.route_bind_concurrency();
942        let route_outages = forwarding.route_outages();
943        Self {
944            registry,
945            activity: None,
946            activity_streams: Arc::new(Mutex::new(HashMap::new())),
947            forwarding,
948            process_liveness: None,
949            supervisor: SupervisorHandle::new(),
950            subc_capabilities: Arc::from([
951                CAP_MANIFEST_REGISTRATION.to_string(),
952                CAP_CHANNEL_LIFECYCLE.to_string(),
953                CAP_PING_PONG.to_string(),
954                CAP_SESSION_ATTACH.to_string(),
955                CAP_ADMISSION_FACTS_RELAY.to_string(),
956                CAP_SCOPES_V1.to_string(),
957                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966            ]),
967            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968            route_bind_relay_timeouts: BTreeMap::new(),
969            route_bind_breakers,
970            route_bind_concurrency,
971            route_outages,
972            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975            storage_config: None,
976            machine_id: None,
977            admission_facts_carrier_module_id: None,
978            admission_facts_targets: None,
979            scopes: Arc::new(RwLock::new(ScopeTable::new(
980                crate::daemon_config::default_scope_authority_owners(),
981            ))),
982            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986            #[cfg(test)]
987            caller_lookup: None,
988            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989            rescan: None,
990            connected_clients: ConnectedClients::new(),
991            counters,
992            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993            daemon_shutdown: DaemonShutdownFlag::default(),
994            daemon_provenance: DaemonProvenanceFacts::default(),
995            #[cfg(test)]
996            control_dispatch_delay: None,
997            #[cfg(test)]
998            provenance_probe_override: None,
999            #[cfg(test)]
1000            route_bind_relay_sent_hold: None,
1001            #[cfg(test)]
1002            hello_registry_to_forwarding_hold: None,
1003        }
1004    }
1005
1006    /// Set the central storage policy: registering modules then receive their
1007    /// resolved storage descriptor in HELLO_ACK.
1008    pub fn with_storage_config(
1009        mut self,
1010        storage_config: Option<crate::daemon_config::StorageConfig>,
1011    ) -> Self {
1012        self.storage_config = storage_config;
1013        self
1014    }
1015
1016    /// Open the ledger on its own writer thread; an unavailable sink does not
1017    /// prevent the daemon from serving routing or supervisor requests.
1018    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019        self.activity = Some(crate::activity::ActivityLedger::open(
1020            run_dir.join("activity.jsonl"),
1021            incarnation,
1022        ));
1023        self.supervisor.set_activity(self.activity.clone());
1024        self
1025    }
1026
1027    /// Set the machine id served to every registering module (HELLO_ACK) and on
1028    /// `server.describe`.
1029    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030        self.machine_id = machine_id;
1031        self
1032    }
1033
1034    /// Configure the exact reserved module and target ids permitted to relay
1035    /// opaque admission facts. Config-file loading validates this authority;
1036    /// this builder keeps the same policy available to embedded test daemons.
1037    pub fn with_admission_facts_config(
1038        mut self,
1039        carrier_module_id: Option<String>,
1040        targets: Option<Vec<String>>,
1041    ) -> Self {
1042        self.admission_facts_carrier_module_id = carrier_module_id;
1043        self.admission_facts_targets = targets;
1044        self
1045    }
1046
1047    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1048    /// Replaces the scope table with an empty one under the new list, so call it
1049    /// while building the handler, before any module can sync.
1050    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052        self.scope_authority_owners = owners;
1053        self
1054    }
1055
1056    /// Set the startup policy before installing the rescan context or serving requests.
1057    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058        self.privileged_operator_prompt = mode;
1059        self
1060    }
1061
1062    /// Override the route.bind relay timeout. Used by tests that assert the
1063    /// timeout path so they don't block on the production-safe default.
1064    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065        self.route_bind_relay_timeout = timeout;
1066        self
1067    }
1068
1069    /// Install per-module route.bind relay budget overrides. A module id
1070    /// listed here wins over the daemon-wide default set via
1071    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1072    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1073    /// the same `Duration` the bind path will use.
1074    pub fn with_route_bind_relay_timeouts(
1075        mut self,
1076        timeouts: impl IntoIterator<Item = (String, Duration)>,
1077    ) -> Self {
1078        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079        self
1080    }
1081
1082    /// Resolve the route.bind relay budget for a specific target module id.
1083    /// Per-module overrides win; the daemon-wide value (set via
1084    /// `with_route_bind_relay_timeout` or the built-in default) is the
1085    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1086    /// the same resolution `handle_route_open` will use.
1087    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088        self.route_bind_relay_timeouts
1089            .get(module_id)
1090            .copied()
1091            .unwrap_or(self.route_bind_relay_timeout)
1092    }
1093
1094    /// Override the per-module bind-relay breaker policy.
1095    ///
1096    /// Used by tests, which cannot spend three production budgets opening a
1097    /// breaker or twenty seconds waiting for its cooldown. The production
1098    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1099    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1100    /// reasoning for the numbers.
1101    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102        self.route_bind_breaker_threshold = threshold.max(1);
1103        self.route_bind_breaker_cooldown = cooldown;
1104        self
1105    }
1106
1107    #[cfg(test)]
1108    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109        self.health_probe_timeout = timeout;
1110        self
1111    }
1112
1113    #[cfg(test)]
1114    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115        self.control_dispatch_delay = Some(delay);
1116        self
1117    }
1118
1119    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1120    /// announce it is parked, once to be released. Build the barrier for two
1121    /// parties, the test being the other one.
1122    #[cfg(test)]
1123    pub(crate) fn with_route_bind_relay_sent_hold(
1124        mut self,
1125        hold: Arc<tokio::sync::Barrier>,
1126    ) -> Self {
1127        self.route_bind_relay_sent_hold = Some(hold);
1128        self
1129    }
1130
1131    /// A HELLO waits on `hold` twice between reserving its registry entry and
1132    /// installing its forwarding endpoint: once to announce it is parked, once
1133    /// to be released. Build the barrier for two parties, the test being the
1134    /// other one, and run the HELLO on a thread of its own.
1135    #[cfg(test)]
1136    pub(crate) fn with_hello_registry_to_forwarding_hold(
1137        mut self,
1138        hold: Arc<std::sync::Barrier>,
1139    ) -> Self {
1140        self.hello_registry_to_forwarding_hold = Some(hold);
1141        self
1142    }
1143
1144    #[cfg(test)]
1145    fn hold_between_registry_and_forwarding(&self) {
1146        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147            hold.wait();
1148            hold.wait();
1149        }
1150    }
1151
1152    pub fn with_process_liveness(
1153        mut self,
1154        process_liveness: Arc<dyn ModuleProcessLiveness>,
1155    ) -> Self {
1156        self.process_liveness = Some(process_liveness);
1157        self
1158    }
1159
1160    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161        self.supervisor = supervisor;
1162        self.supervisor.set_activity(self.activity.clone());
1163        self
1164    }
1165
1166    pub fn with_daemon_provenance(
1167        mut self,
1168        pid: u32,
1169        started_at_ms: u64,
1170        executable_path: Option<PathBuf>,
1171        build_git_sha: Option<String>,
1172        build_lock_digest: Option<String>,
1173    ) -> Self {
1174        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175        let process_start_time = process_start_time(pid);
1176        self.daemon_provenance = DaemonProvenanceFacts {
1177            build: DaemonBuildProvenance {
1178                build_git_sha,
1179                build_lock_digest,
1180            },
1181            pid: Some(pid),
1182            started_at_ms: Some(started_at_ms),
1183            start_clock: None,
1184            executable_path,
1185            executable_identity,
1186            process_start_time,
1187            probe: ExecutableIdentityProbe::default(),
1188        };
1189        self
1190    }
1191
1192    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193        self.daemon_provenance.start_clock = Some(clock);
1194        self
1195    }
1196
1197    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1198    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199        self.scopes
1200            .write()
1201            .unwrap_or_else(|poisoned| poisoned.into_inner())
1202            .set_wall_clock(Arc::new(clock));
1203        self
1204    }
1205
1206    #[cfg(test)]
1207    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208        self.provenance_probe_override = Some(result);
1209        self
1210    }
1211
1212    /// Install the configured module set and its reserved capability bindings.
1213    /// Bindings are configuration-scoped and may point at a provider that has not
1214    /// been installed yet, so this does not require the bound module to exist.
1215    pub fn with_capability_config(
1216        self,
1217        modules: impl IntoIterator<Item = (String, bool)>,
1218        reserved_capabilities: BTreeMap<String, String>,
1219    ) -> Self {
1220        self.capability_evaluator
1221            .configure(modules, reserved_capabilities);
1222        self
1223    }
1224
1225    /// Share the supervisor's daemon-shutdown flag with this handler, so
1226    /// requirement changes after the daemon has begun its announced shutdown
1227    /// are not reported as faults.
1228    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229        self.daemon_shutdown = flag;
1230        self
1231    }
1232
1233    pub fn with_supervisor_rescan(
1234        mut self,
1235        supervisor: Supervisor,
1236        config_path: impl Into<PathBuf>,
1237        configured_port: Option<u16>,
1238    ) -> Self {
1239        self.rescan = Some(SupervisorRescanContext {
1240            supervisor,
1241            config_path: config_path.into(),
1242            configured_port,
1243            storage_config: self.storage_config.clone(),
1244            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245            admission_facts_targets: self.admission_facts_targets.clone(),
1246            scope_authority_owners: self.scope_authority_owners.clone(),
1247            privileged_operator_prompt: self.privileged_operator_prompt,
1248            generation: Arc::default(),
1249        });
1250        self
1251    }
1252
1253    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254        self.connected_clients = connected_clients;
1255        self
1256    }
1257
1258    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259        Arc::clone(&self.forwarding)
1260    }
1261
1262    pub(crate) fn counters(&self) -> DaemonCounters {
1263        self.counters.clone()
1264    }
1265
1266    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1267    /// requirement event without depending on an operator polling a status command.
1268    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269        tokio::spawn(async move {
1270            loop {
1271                self.capability_evaluator
1272                    .wait_for_change_or_deadline()
1273                    .await;
1274                self.refresh_capability_requirements();
1275            }
1276        });
1277    }
1278
1279    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1280    /// deadline passed during sleep is enforced on the first tick after wake.
1281    /// While no live scope carries a deadline the loop waits for an accepted
1282    /// scope change instead of ticking, so a daemon without deadlines never
1283    /// wakes for expiry.
1284    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285        tokio::spawn(async move {
1286            loop {
1287                let has_deadlines = self
1288                    .scopes
1289                    .read()
1290                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1291                    .has_deadlines();
1292                if !has_deadlines {
1293                    self.scope_deadline_added.notified().await;
1294                    continue;
1295                }
1296                tokio::time::sleep(Duration::from_secs(1)).await;
1297                if let Err(error) = self.sweep_expired_scopes() {
1298                    warn!(%error, "scope expiry sweep failed");
1299                }
1300            }
1301        });
1302    }
1303
1304    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305        let mut table = self
1306            .scopes
1307            .write()
1308            .unwrap_or_else(|poisoned| poisoned.into_inner());
1309        let swept = table.sweep_expired();
1310        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311        drop(table);
1312        self.close_scope_drained_routes(drained);
1313        Ok(())
1314    }
1315
1316    fn publish_scope_changes(
1317        &self,
1318        changes: &[crate::scopes::ScopeTagChange],
1319        expired: &[crate::scopes::ScopeExpired],
1320    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321        if changes.is_empty() {
1322            return Ok(Vec::new());
1323        }
1324        let drained = self
1325            .forwarding
1326            .publish_scope_changes(changes)
1327            .map_err(RouterError::Forwarding)?;
1328        let mut counts = HashMap::new();
1329        for route in &drained {
1330            *counts
1331                .entry((
1332                    route.scope.owner.as_str(),
1333                    route.scope.scope_ref.as_str(),
1334                    route.scope.tag.scope_epoch,
1335                ))
1336                .or_insert(0usize) += 1;
1337        }
1338        for scope in expired {
1339            let routes_closed = counts
1340                .get(&(
1341                    scope.owner.as_str(),
1342                    scope.scope_ref.as_str(),
1343                    scope.scope_epoch,
1344                ))
1345                .copied()
1346                .unwrap_or(0);
1347            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349                routes_closed, "scope expired");
1350        }
1351        Ok(drained)
1352    }
1353
1354    fn runtime_capability_snapshot(
1355        &self,
1356    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357        let runtime = self
1358            .supervisor
1359            .list()
1360            .into_iter()
1361            .map(|module| {
1362                let status = module.status().map_err(|err| {
1363                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364                })?;
1365                Ok(RuntimeModule {
1366                    module_id: status.module_id,
1367                    state: status.state,
1368                    enabled: status.enabled,
1369                })
1370            })
1371            .collect::<Result<Vec<_>, RouterError>>()?;
1372        // Include every registration, published or not. The capability
1373        // evaluator counts a module as providing its capabilities as soon as it
1374        // has registered, even before `catalog.list` shows it, so a HELLO in
1375        // progress does not change other modules' required-capability
1376        // verdicts. See `Registry::list_registered_modules`.
1377        let registrations = self.registry.list_registered_modules().map_err(|err| {
1378            RouterError::backend(
1379                0,
1380                0,
1381                format!("failed to list capability registrations: {err}"),
1382            )
1383        })?;
1384        let registrations = registrations
1385            .into_iter()
1386            .map(|registration| RegisteredModule {
1387                module_id: registration.manifest.module_id,
1388                module_version: registration.manifest.module_version,
1389                capabilities: registration.manifest.capabilities,
1390            })
1391            .collect();
1392        Ok((runtime, registrations))
1393    }
1394
1395    /// The capability side effects of a module becoming the active registration
1396    /// for its id: cache its manifest (warning if its claims drifted), run the
1397    /// deny census when its declarations call for one, and recompute the
1398    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1399    /// candidate's does not, and the supervisor does it at promotion instead,
1400    /// through [`crate::supervise::SwapPromotionObserver`].
1401    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402        let cached_registration = RegisteredModule {
1403            module_id: registration.manifest.module_id.clone(),
1404            module_version: registration.manifest.module_version.clone(),
1405            capabilities: registration.manifest.capabilities.clone(),
1406        };
1407        if self.capability_evaluator.record_hello(&cached_registration) {
1408            warn!(
1409                module_id = %cached_registration.module_id,
1410                "capability claims drifted from the cached manifest"
1411            );
1412        }
1413        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414            self.enforce_capability_denies();
1415        }
1416        self.refresh_capability_requirements();
1417    }
1418
1419    /// Point the shared supervisor handle at this handler for swap promotions.
1420    /// Called wherever a handler is put behind the `Arc` the router serves, so
1421    /// it can be held weakly.
1422    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425        self.supervisor.set_swap_promotion_observer(observer);
1426    }
1427
1428    pub fn refresh_capability_requirements(&self) {
1429        match self.runtime_capability_snapshot() {
1430            Ok((runtime, registrations)) => {
1431                log_requirement_events(
1432                    self.capability_evaluator
1433                        .evaluate_now(&runtime, &registrations),
1434                    &self.daemon_shutdown,
1435                );
1436            }
1437            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438        }
1439    }
1440
1441    /// Reconcile only live, attested route bindings after a capability deny edge
1442    /// or target claim was added. This is deliberately a control-plane census:
1443    /// the opaque forwarding hot path must not grow a per-frame capability check.
1444    fn enforce_capability_denies(&self) {
1445        let registrations = match self.registry.list_registered_modules() {
1446            Ok(snapshot) => snapshot,
1447            Err(err) => {
1448                warn!(error = %err, "failed to read registrations for capability deny census");
1449                return;
1450            }
1451        };
1452        let manifests = registrations
1453            .into_iter()
1454            .map(|registration| {
1455                (
1456                    registration.manifest.module_id.clone(),
1457                    registration.manifest,
1458                )
1459            })
1460            .collect::<BTreeMap<_, _>>();
1461        let census = match self.forwarding.route_census(None) {
1462            Ok(census) => census,
1463            Err(err) => {
1464                warn!(error = %err, "failed to read route census for capability deny enforcement");
1465                return;
1466            }
1467        };
1468
1469        for (target_module_id, routes) in census {
1470            let Some(target_manifest) = manifests.get(&target_module_id) else {
1471                continue;
1472            };
1473            let mut closed_routes = Vec::new();
1474            let mut module_goodbyes = Vec::new();
1475            for route in routes {
1476                let Principal::Reserved {
1477                    module_id: opening_module_id,
1478                } = &route.principal
1479                else {
1480                    continue;
1481                };
1482                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483                    continue;
1484                };
1485                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486                    continue;
1487                };
1488
1489                match self.forwarding.release_client_route(
1490                    route.goodbye_target.connection_id,
1491                    route.goodbye_target.channel,
1492                    route.goodbye_target.epoch,
1493                ) {
1494                    Ok(RouteRelease::Removed(module_goodbye)) => {
1495                        warn!(
1496                            opening_module_id,
1497                            target_module_id,
1498                            capability,
1499                            "force-closing route because an attested capability deny edge now matches"
1500                        );
1501                        closed_routes.push(route);
1502                        module_goodbyes.push(module_goodbye);
1503                    }
1504                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505                    Err(err) => warn!(
1506                        opening_module_id,
1507                        target_module_id,
1508                        capability,
1509                        error = %err,
1510                        "failed to force-close capability-denied route"
1511                    ),
1512                }
1513            }
1514
1515            if closed_routes.is_empty() {
1516                continue;
1517            }
1518            send_route_control_pushes(
1519                &self.forwarding,
1520                closed_routes,
1521                ClientControlPush::RouteClosed {
1522                    module_id: target_module_id,
1523                    channels: Vec::new(),
1524                    reason: RouteCloseReason::CapabilityDenied,
1525                    drained: false,
1526                    abandoned: 0,
1527                    excluded_subscriptions: 0,
1528                    terminal: Some(false),
1529                },
1530            );
1531            self.emit_route_goodbyes(module_goodbyes);
1532        }
1533    }
1534
1535    /// Why a registered module is not accepting new route binds, or `None` when
1536    /// it is. This is the module's effective readiness: its declared readiness
1537    /// first, then every `need: required` capability it declares evaluating to
1538    /// `provided`. `route.open` and `catalog.list` both read it here so the
1539    /// catalog never reports a module routable that `route.open` would refuse.
1540    fn not_ready_reason(
1541        &self,
1542        registration: &crate::registry::ModuleRegistration,
1543    ) -> Option<NotReadyReason> {
1544        if !registration.ready {
1545            return Some(NotReadyReason {
1546                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1547                capability: None,
1548            });
1549        }
1550        self.first_unprovided_required_capability(registration)
1551            .map(|capability| NotReadyReason {
1552                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1553                capability: Some(capability),
1554            })
1555    }
1556
1557    /// The lexicographically first capability this registration declares
1558    /// `need: required` whose evaluator verdict is not `provided`.
1559    ///
1560    /// The verdicts are the capability evaluator's own; nothing here decides
1561    /// what "provided" means. The evaluator counts a capability provided as
1562    /// soon as a module claiming it has REGISTERED, not once that module is
1563    /// ready. That distinction is what keeps two modules that require each
1564    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1565    /// is ready", each would wait for the other to become ready first and
1566    /// neither ever would. Do not tighten it to readiness.
1567    ///
1568    /// A required capability with no verdict at all means this registration's
1569    /// HELLO or catalog.update landed after the last recompute; recompute once
1570    /// rather than let a missing verdict read as either answer. If it is still
1571    /// missing (the recompute itself failed) the capability counts as
1572    /// unprovided: the refusal is retryable, and routing a module whose
1573    /// required provider is unknown is the outcome this check exists to stop.
1574    fn first_unprovided_required_capability(
1575        &self,
1576        registration: &crate::registry::ModuleRegistration,
1577    ) -> Option<String> {
1578        let required = registration
1579            .manifest
1580            .capabilities
1581            .iter()
1582            .flat_map(|declarations| declarations.requires.iter())
1583            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1584            .map(|requirement| requirement.capability.as_str())
1585            .collect::<BTreeSet<_>>();
1586        if required.is_empty() {
1587            return None;
1588        }
1589        let module_id = registration.manifest.module_id.as_str();
1590        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1591        if required
1592            .iter()
1593            .any(|capability| verdict(capability).is_none())
1594        {
1595            self.refresh_capability_requirements();
1596        }
1597        required
1598            .into_iter()
1599            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1600            .map(str::to_string)
1601    }
1602
1603    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1604        self.capability_evaluator
1605            .statuses()
1606            .into_iter()
1607            .map(capability_requirement_status)
1608            .collect()
1609    }
1610
1611    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1612    /// registration-release watch. The signal is what the supervisor waits on
1613    /// before spawning a replacement, so it must only fire once forwarding
1614    /// teardown is also done (see [`Self::cleanup_connection`] /
1615    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1616    /// admitted but its forwarding endpoint could not be installed.
1617    fn deregister_connection(
1618        &self,
1619        connection_id: ConnectionId,
1620        reason: RegistrationEndReason,
1621    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1622        self.registry
1623            .deregister_connection_with_reason(connection_id, reason)
1624    }
1625
1626    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1627        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1628            return None;
1629        }
1630        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1631            parse_client_control_request(&frame.body)
1632        else {
1633            return None;
1634        };
1635        Some(target_module_id(&target).to_string())
1636    }
1637
1638    pub(crate) fn route_open_capacity_refusal(
1639        &self,
1640        ctx: &RouteCtx,
1641        frame: &Frame,
1642        target_module_id: &str,
1643        in_flight: usize,
1644        limit: usize,
1645    ) -> Result<Frame, RouterError> {
1646        // Sent before the request reaches `handle_route_open`, so the caller
1647        // fields come from the raw frame here.
1648        let caller = self.route_open_caller_from_frame(frame);
1649        self.route_open_admission_refusal_frame(
1650            ctx,
1651            &caller,
1652            frame,
1653            target_module_id,
1654            "open_admission_full",
1655            (in_flight, limit),
1656            format!(
1657                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1658            ),
1659        )
1660    }
1661
1662    fn route_open_target_capacity_refusal(
1663        &self,
1664        ctx: &RouteCtx,
1665        caller: &RouteOpenCaller,
1666        frame: &Frame,
1667        target_module_id: &str,
1668        in_flight: usize,
1669    ) -> Result<Frame, RouterError> {
1670        self.route_open_admission_refusal_frame(
1671            ctx,
1672            caller,
1673            frame,
1674            target_module_id,
1675            "target_binds_full",
1676            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1677            format!(
1678                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1679            ),
1680        )
1681    }
1682
1683    /// Admission pressure clears as existing binds settle, so its refusal must
1684    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1685    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1686    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1687    /// cannot currently reach its target; `module_timeout` would falsely claim
1688    /// that a wait expired. A new, cleaner code would be terminal to deployed
1689    /// clients, so it requires a client-tolerance rollout before daemon emission.
1690    #[allow(clippy::too_many_arguments)]
1691    fn route_open_admission_refusal_frame(
1692        &self,
1693        ctx: &RouteCtx,
1694        caller: &RouteOpenCaller,
1695        frame: &Frame,
1696        target_module_id: &str,
1697        reason: &'static str,
1698        (in_flight, limit): (usize, usize),
1699        message: impl Into<String>,
1700    ) -> Result<Frame, RouterError> {
1701        let code = error_codes::TARGET_UNAVAILABLE;
1702        self.counters.increment_route_open_refused(code);
1703        info!(
1704            target: "control",
1705            code,
1706            reason,
1707            module_id = ?target_module_id,
1708            connection_id = ctx.connection_id.get(),
1709            principal = %caller.principal,
1710            scope = %caller.scope,
1711            harness = %caller.harness,
1712            in_flight,
1713            limit,
1714            "route.open refused"
1715        );
1716        control_error_frame(frame, code, message.into())
1717    }
1718
1719    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1720    ///
1721    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1722    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1723    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1724    #[cfg(test)]
1725    pub fn handle_control(
1726        &self,
1727        connection_id: ConnectionId,
1728        frame: Frame,
1729    ) -> Result<Vec<Frame>, RouterError> {
1730        match frame.header.ty {
1731            FrameType::Ping => Ok(vec![pong(&frame)?]),
1732            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1733            FrameType::Goodbye => self.handle_goodbye(connection_id),
1734            ty => Ok(vec![control_error_frame(
1735                &frame,
1736                "unsupported_control_frame",
1737                format!("unsupported channel-0 frame {ty:?}"),
1738            )?]),
1739        }
1740    }
1741
1742    pub async fn handle_control_frame(
1743        &self,
1744        ctx: &RouteCtx,
1745        frame: Frame,
1746    ) -> Result<Vec<Frame>, RouterError> {
1747        self.handle_control_frame_timed(ctx, frame, None).await
1748    }
1749
1750    pub(crate) fn record_caller_endpoints(
1751        &self,
1752        connection: ConnectionId,
1753        endpoints: Option<(SocketAddr, SocketAddr)>,
1754    ) {
1755        let mut callers = self
1756            .caller_endpoints
1757            .lock()
1758            .unwrap_or_else(|p| p.into_inner());
1759        callers.remove(&connection);
1760        if let Some(endpoints) = endpoints {
1761            callers.insert(connection, endpoints);
1762        }
1763    }
1764
1765    #[cfg(test)]
1766    pub(crate) fn with_caller_lookup_for_test(
1767        mut self,
1768        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1769    ) -> Self {
1770        self.caller_lookup = Some(Arc::new(lookup));
1771        self
1772    }
1773
1774    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1775        let endpoints = self
1776            .caller_endpoints
1777            .lock()
1778            .unwrap_or_else(|p| p.into_inner())
1779            .get(&connection)
1780            .copied();
1781        #[cfg(test)]
1782        let lookup = self.caller_lookup.clone();
1783        tokio::task::spawn_blocking(move || {
1784            #[cfg(test)]
1785            if let Some(lookup) = lookup {
1786                return lookup(endpoints);
1787            }
1788            endpoints
1789                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1790                .unwrap_or_default()
1791        })
1792        .await
1793        .unwrap_or_default()
1794    }
1795
1796    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1797        frame.header.channel == 0
1798            && frame.header.ty == FrameType::Request
1799            && self
1800                .forwarding
1801                .module_endpoint_for_connection(ctx.connection_id)
1802                .is_ok_and(|endpoint| endpoint.is_none())
1803            && parse_client_control_request(&frame.body)
1804                .ok()
1805                .is_some_and(|request| privileged_operation(&request).is_some())
1806    }
1807
1808    pub(crate) async fn handle_control_frame_timed(
1809        &self,
1810        ctx: &RouteCtx,
1811        frame: Frame,
1812        dispatch_started_at: Option<StdInstant>,
1813    ) -> Result<Vec<Frame>, RouterError> {
1814        match frame.header.ty {
1815            FrameType::Ping => Ok(vec![pong(&frame)?]),
1816            FrameType::Hello => {
1817                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1818            }
1819            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1820            FrameType::Cancel => {
1821                // A Cancel on channel 0 names a waiting operator.confirm or a
1822                // subscription. Successful cancellation has no ordinary reply.
1823                if self
1824                    .forwarding
1825                    .operator_confirms()
1826                    .cancel(ctx.connection_id, frame.header.corr)
1827                    || self
1828                        .supervisor
1829                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1830                    || self.cancel_activity_subscription(ctx, &frame)?
1831                {
1832                    Ok(Vec::new())
1833                } else {
1834                    Ok(vec![control_error_frame(
1835                        &frame,
1836                        "unknown_subscription",
1837                        "no subscription has this correlation id",
1838                    )?])
1839                }
1840            }
1841            FrameType::Request => {
1842                // This additive operation is not part of the existing exhaustive
1843                // module-control enum. Probe the op before decoding that enum.
1844                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1845                if op
1846                    .as_ref()
1847                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1848                {
1849                    return self.handle_activity_record(ctx, frame).await;
1850                }
1851                if op.as_ref().is_some_and(|probe| {
1852                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1853                }) {
1854                    return self.handle_activity_read(ctx, frame).await;
1855                }
1856                if op
1857                    .as_ref()
1858                    .is_some_and(|probe| probe.op == "operator.confirm")
1859                {
1860                    return self.handle_operator_confirm(ctx, frame);
1861                }
1862                if op
1863                    .as_ref()
1864                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1865                {
1866                    return self.handle_scope_served_by(ctx.connection_id, frame);
1867                }
1868                if self
1869                    .forwarding
1870                    .module_endpoint_for_connection(ctx.connection_id)
1871                    .map_err(RouterError::Forwarding)?
1872                    .is_some()
1873                {
1874                    if !is_known_module_request_op(&frame.body) {
1875                        return Ok(vec![control_error_frame(
1876                            &frame,
1877                            "unsupported_control_frame",
1878                            "module-originated channel-0 REQUEST is not supported",
1879                        )?]);
1880                    }
1881                    let request = match parse_module_control_request_from_module(&frame.body) {
1882                        Ok(request) => request,
1883                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1884                            return Ok(vec![control_error_frame(
1885                                &frame,
1886                                "unsupported_control_frame",
1887                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1888                            )?])
1889                        }
1890                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1891                            return Ok(vec![control_error_frame(
1892                                &frame,
1893                                "invalid_control_body",
1894                                format!("malformed module control body: {err}"),
1895                            )?])
1896                        }
1897                    };
1898                    let op = module_control_request_op(&request);
1899                    let corr = frame.header.corr;
1900                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1901                    let result =
1902                        self.handle_module_control_request(ctx.connection_id, frame, request);
1903                    log_slow_control_dispatch(
1904                        dispatch_started_at,
1905                        op,
1906                        ctx.connection_id,
1907                        corr,
1908                        None,
1909                    );
1910                    return result;
1911                }
1912
1913                if is_known_module_request_op(&frame.body) {
1914                    return Ok(vec![control_error_frame(
1915                        &frame,
1916                        "not_registered",
1917                        "catalog.update requires an active module registration owned by this connection",
1918                    )?]);
1919                }
1920
1921                let request = match parse_client_control_request(&frame.body) {
1922                    Ok(request) => request,
1923                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1924                        return Ok(vec![control_error_frame(
1925                            &frame,
1926                            "unknown_control_op",
1927                            format!("unknown client control op: {err}"),
1928                        )?])
1929                    }
1930                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1931                        return Ok(vec![control_error_frame(
1932                            &frame,
1933                            "invalid_control_body",
1934                            format!("malformed client control body: {err}"),
1935                        )?])
1936                    }
1937                };
1938                let op = client_control_request_op(&request);
1939                let corr = frame.header.corr;
1940                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1941                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1942                #[cfg(test)]
1943                if let Some(delay) = self.control_dispatch_delay {
1944                    tokio::time::sleep(delay).await;
1945                }
1946                let result = self
1947                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1948                    .await;
1949                if let Some(timings) = route_open_timings.as_mut() {
1950                    timings.finish();
1951                }
1952                log_slow_control_dispatch(
1953                    dispatch_started_at,
1954                    op,
1955                    ctx.connection_id,
1956                    corr,
1957                    route_open_timings.as_ref(),
1958                );
1959                result
1960            }
1961            FrameType::Push => {
1962                let Some(endpoint) = self
1963                    .forwarding
1964                    .module_endpoint_for_connection(ctx.connection_id)
1965                    .map_err(RouterError::Forwarding)?
1966                else {
1967                    return Ok(vec![control_error_frame(
1968                        &frame,
1969                        "unsupported_control_frame",
1970                        "client-originated channel-0 PUSH is not supported",
1971                    )?]);
1972                };
1973                self.handle_status_update(endpoint, frame)
1974            }
1975            FrameType::Response | FrameType::Error
1976                if self
1977                    .forwarding
1978                    .module_endpoint_for_connection(ctx.connection_id)
1979                    .map_err(RouterError::Forwarding)?
1980                    .is_some() =>
1981            {
1982                self.handle_module_relay_response(ctx.connection_id, frame)
1983            }
1984            ty => Ok(vec![control_error_frame(
1985                &frame,
1986                "unsupported_control_frame",
1987                format!("unsupported channel-0 frame {ty:?}"),
1988            )?]),
1989        }
1990    }
1991
1992    pub fn cleanup_connection(
1993        &self,
1994        connection_id: ConnectionId,
1995    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1996        self.caller_endpoints
1997            .lock()
1998            .unwrap_or_else(|p| p.into_inner())
1999            .remove(&connection_id);
2000        self.forwarding
2001            .operator_confirms()
2002            .connection_closed(connection_id);
2003        self.cleanup_connection_with_end_reason(
2004            connection_id,
2005            RegistrationEndReason::ConnectionClosed,
2006        )
2007    }
2008
2009    fn cleanup_connection_with_end_reason(
2010        &self,
2011        connection_id: ConnectionId,
2012        requested_reason: RegistrationEndReason,
2013    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2014        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2015            self.registry
2016                .get_module_by_connection(connection_id)?
2017                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2018                .and_then(|module| module.registration_end_reason().ok().flatten())
2019                .unwrap_or(requested_reason)
2020        } else {
2021            requested_reason
2022        };
2023        let crash_closed = self
2024            .registry
2025            .get_module_by_connection(connection_id)?
2026            .and_then(|registration| {
2027                self.forwarding
2028                    .module_endpoint_for_connection(connection_id)
2029                    .ok()
2030                    .flatten()
2031                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2032                    .map(|routes| (registration.manifest.module_id, routes))
2033            });
2034        let crash_closed = crash_closed.map(|(module_id, routes)| {
2035            let terminal = match self.supervisor.get(&module_id) {
2036                None => false,
2037                Some(module) => match module.will_recover_after_connection_loss() {
2038                    Ok(will_recover) => !will_recover,
2039                    Err(err) => {
2040                        warn!(
2041                            %module_id,
2042                            error = %err,
2043                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2044                        );
2045                        false
2046                    }
2047                },
2048            };
2049            // The forwarding table gates all providers at the start of daemon
2050            // shutdown, before their connections are closed. An ordinary
2051            // module disconnect still reports crash if that gate is not set.
2052            let (reason, drained) = match self
2053                .forwarding
2054                .daemon_drain_result_for_connection(connection_id)
2055            {
2056                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2057                Ok(None) => (RouteCloseReason::Crash, false),
2058                Err(err) => {
2059                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2060                    (RouteCloseReason::Crash, false)
2061                }
2062            };
2063            (module_id, routes, reason, drained, terminal)
2064        });
2065        let registrations = self.deregister_connection(connection_id, end_reason);
2066        let cleanup = if crash_closed.is_some() {
2067            self.forwarding.cleanup_connection_counted(connection_id)
2068        } else {
2069            // A module connection's teardown needs the count of abandoned
2070            // route.bind relays for its route.closed notice below. Any other
2071            // connection, such as a client's, sends no such notice and needs
2072            // only its routes released, so it uses the route-only wrapper and
2073            // reports zero.
2074            self.forwarding
2075                .cleanup_connection(connection_id)
2076                .map(|released| crate::forwarding::ConnectionCleanup {
2077                    released,
2078                    abandoned_relays: 0,
2079                })
2080        };
2081        // The route.closed push waits for forwarding teardown because only
2082        // teardown knows how many pending route.bind relays it aborted. It still
2083        // goes out before the GOODBYEs for the released routes, and its targets
2084        // were captured above, before teardown removed those routes.
2085        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2086            let abandoned = cleanup
2087                .as_ref()
2088                .map_or(0, |cleanup| cleanup.abandoned_relays);
2089            send_route_control_pushes(
2090                &self.forwarding,
2091                routes,
2092                ClientControlPush::RouteClosed {
2093                    module_id,
2094                    channels: Vec::new(),
2095                    reason,
2096                    drained,
2097                    abandoned,
2098                    excluded_subscriptions: 0,
2099                    terminal: Some(terminal),
2100                },
2101            );
2102        }
2103        if let Ok(cleanup) = cleanup {
2104            self.emit_route_goodbyes(cleanup.released);
2105        }
2106        // Signal the registration-release watch only now that BOTH registry and
2107        // forwarding teardown are done, so a supervisor waiting to spawn a
2108        // replacement never observes release while old routes still exist.
2109        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2110            crate::supervise::notify_registration_release();
2111            self.capability_evaluator.wake_deadline_loop();
2112            self.refresh_capability_requirements();
2113        }
2114        self.supervisor.remove_spawn_subscribers(connection_id);
2115        self.activity_streams
2116            .lock()
2117            .unwrap_or_else(|p| p.into_inner())
2118            .retain(|(id, _), task| {
2119                if *id == connection_id {
2120                    task.abort();
2121                    false
2122                } else {
2123                    true
2124                }
2125            });
2126        // Sync authority dies with its connection, so the owner's next
2127        // connection can take it; the owner's scopes stay as they are.
2128        self.hello_launch_nonces
2129            .lock()
2130            .unwrap_or_else(|poisoned| poisoned.into_inner())
2131            .forget(connection_id);
2132        self.scopes
2133            .write()
2134            .unwrap_or_else(|poisoned| poisoned.into_inner())
2135            .release_connection(connection_id);
2136        registrations
2137    }
2138
2139    pub(crate) fn handle_route_goodbye(
2140        &self,
2141        connection_id: ConnectionId,
2142        route_channel: u16,
2143        route_epoch: u32,
2144    ) -> Result<bool, RouterError> {
2145        debug!(
2146            connection_id = connection_id.get(),
2147            route_channel, route_epoch, "handling route GOODBYE"
2148        );
2149        let RouteRelease::Removed(released_route) = self
2150            .forwarding
2151            .release_client_route(connection_id, route_channel, route_epoch)
2152            .map_err(RouterError::Forwarding)?
2153        else {
2154            return Ok(false);
2155        };
2156        self.emit_route_goodbyes(vec![released_route]);
2157        Ok(true)
2158    }
2159
2160    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2161        for released in released_routes {
2162            let frame = match Frame::build_with_version(
2163                released.negotiated_ver,
2164                FrameType::Goodbye,
2165                control_flags(),
2166                released.channel,
2167                released.epoch,
2168                0,
2169                Vec::new(),
2170            ) {
2171                Ok(frame) => frame,
2172                Err(err) => {
2173                    warn!(
2174                        route_channel = released.channel,
2175                        error = %err,
2176                        "failed to build route GOODBYE frame"
2177                    );
2178                    continue;
2179                }
2180            };
2181            if !released.close_on_delivery_failure() {
2182                crate::forwarding::send_module_route_goodbye(
2183                    &self.counters,
2184                    &released.sink,
2185                    frame,
2186                    released.module_id.as_deref(),
2187                    "client route released",
2188                );
2189                continue;
2190            }
2191            if let Err(err) = released.sink.try_send(frame) {
2192                warn!(
2193                    target_connection_id = released.connection_id.get(),
2194                    route_channel = released.channel,
2195                    error = %err,
2196                    "route GOODBYE was not delivered to client; closing target connection"
2197                );
2198                if self
2199                    .forwarding
2200                    .escalate_client_delivery_failure(
2201                        released.connection_id,
2202                        released.channel,
2203                        released.epoch,
2204                        CloseReason::new(
2205                            "route_goodbye_delivery_failed",
2206                            format!(
2207                                "failed to enqueue route GOODBYE for channel {}: {err}",
2208                                released.channel
2209                            ),
2210                        ),
2211                        crate::forwarding::UndeliveredFrame {
2212                            module_id: released.module_id.as_deref(),
2213                            sink: &released.sink,
2214                        },
2215                    )
2216                    .unwrap_or(false)
2217                {
2218                    self.counters.increment_goodbye_relay_client_failed();
2219                }
2220            }
2221        }
2222    }
2223
2224    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2225    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2226    /// own commit failed after the module had already accepted). Without this, a
2227    /// module that accepts late keeps a binding subc has torn down, so a later
2228    /// frame on that module channel could misdeliver if the channel is reused.
2229    ///
2230    /// Never closes the shared module connection on failure: a dropped notification
2231    /// only wastes a bounded amount of warm module-side state, which the module's
2232    /// own idle reaper reclaims. Only call this once the route.bind relay was
2233    /// actually enqueued to the module — if the relay send itself failed, the
2234    /// module never created a binding and there is nothing to tear down.
2235    fn send_abandoned_route_bind_goodbye(
2236        &self,
2237        module_sink: &crate::FrameSink,
2238        negotiated_ver: u8,
2239        module_channel: u16,
2240        module_epoch: u32,
2241    ) {
2242        let frame = match Frame::build_with_version(
2243            negotiated_ver,
2244            FrameType::Goodbye,
2245            control_flags(),
2246            module_channel,
2247            module_epoch,
2248            0,
2249            Vec::new(),
2250        ) {
2251            Ok(frame) => frame,
2252            Err(err) => {
2253                warn!(
2254                    route_channel = module_channel,
2255                    error = %err,
2256                    "failed to build GOODBYE for abandoned route.bind"
2257                );
2258                return;
2259            }
2260        };
2261        crate::forwarding::send_module_route_goodbye(
2262            &self.counters,
2263            module_sink,
2264            frame,
2265            None,
2266            "abandoned route.bind",
2267        );
2268    }
2269
2270    fn handle_hello(
2271        &self,
2272        connection_id: ConnectionId,
2273        sink: Option<crate::FrameSink>,
2274        frame: Frame,
2275    ) -> Result<Vec<Frame>, RouterError> {
2276        debug!(
2277            connection_id = connection_id.get(),
2278            corr = frame.header.corr,
2279            "handling HELLO"
2280        );
2281        // A module connection has one identity for its entire lifetime. A second
2282        // registration would leave the old registry owner behind while replacing
2283        // its forwarding endpoint and launch nonce.
2284        if self
2285            .registry
2286            .get_module_by_connection(connection_id)
2287            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2288            .is_some()
2289        {
2290            return Ok(vec![control_error_frame(
2291                &frame,
2292                "invalid_hello",
2293                "connection is already registered as a module",
2294            )?]);
2295        }
2296        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2297            Ok(value) => value,
2298            Err(err) => {
2299                return Ok(vec![control_error_frame(
2300                    &frame,
2301                    "invalid_hello",
2302                    format!("malformed HELLO body: {err}"),
2303                )?])
2304            }
2305        };
2306        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2307            return Ok(vec![control_error_frame(
2308                &frame,
2309                "invalid_capability_grammar",
2310                err.to_string(),
2311            )?]);
2312        }
2313        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2314            return Ok(vec![control_error_frame(
2315                &frame,
2316                "invalid_manifest",
2317                err.to_string(),
2318            )?]);
2319        }
2320        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2321            return Ok(vec![control_error_frame(
2322                &frame,
2323                "invalid_event_declaration",
2324                err.to_string(),
2325            )?]);
2326        }
2327        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2328            return Ok(vec![control_error_frame(
2329                &frame,
2330                "invalid_manifest",
2331                err.to_string(),
2332            )?]);
2333        }
2334        if let Some(provenance) = hello_value
2335            .get("manifest")
2336            .and_then(|manifest| manifest.get("provenance"))
2337        {
2338            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2339                return Ok(vec![control_error_frame(
2340                    &frame,
2341                    "invalid_manifest",
2342                    format!("malformed manifest provenance: {err}"),
2343                )?]);
2344            }
2345        }
2346        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2347            let module_id = hello_value["manifest"]["module_id"]
2348                .as_str()
2349                .unwrap_or("<unknown>");
2350            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2351        }
2352        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2353            Ok(hello) => hello,
2354            Err(err) => {
2355                return Ok(vec![control_error_frame(
2356                    &frame,
2357                    "invalid_hello",
2358                    format!("malformed HELLO body: {err}"),
2359                )?])
2360            }
2361        };
2362
2363        if hello.protocol_ver != hello.manifest.protocol_ver {
2364            return Ok(vec![control_error_frame(
2365                &frame,
2366                "invalid_manifest",
2367                format!(
2368                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2369                    hello.protocol_ver, hello.manifest.protocol_ver
2370                ),
2371            )?]);
2372        }
2373
2374        if hello.manifest.module_id.trim().is_empty() {
2375            return Ok(vec![control_error_frame(
2376                &frame,
2377                "invalid_manifest",
2378                "manifest module_id must not be empty",
2379            )?]);
2380        }
2381
2382        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2383            Ok(negotiated_ver) => negotiated_ver,
2384            Err(message) => {
2385                return Ok(vec![control_error_frame(
2386                    &frame,
2387                    "version_unsupported",
2388                    message,
2389                )?])
2390            }
2391        };
2392
2393        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2394        // swap is open for this id, the only HELLO admitted as a second process
2395        // is the one carrying the candidate's launch nonce (the swap token), and
2396        // it registers into the candidate slot rather than being refused as a
2397        // duplicate. Run after the reserved gate, a reserved module's candidate
2398        // would be refused `reserved_module` for presenting a nonce that gate
2399        // does not know. See `SupervisorHandle::swap_hello_admission`.
2400        let swap_admission = self
2401            .supervisor
2402            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2403        if swap_admission == SwapHelloAdmission::Refused {
2404            warn!(
2405                module_id = %hello.manifest.module_id,
2406                connection_id = connection_id.get(),
2407                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2408            );
2409            return Ok(vec![control_error_frame(
2410                &frame,
2411                "swap_token_invalid",
2412                format!(
2413                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2414                    hello.manifest.module_id
2415                ),
2416            )?]);
2417        }
2418        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2419
2420        // A swap candidate proves its identity with the candidate nonce above.
2421        // Other registrations receive this catalog fact only when reserved-module
2422        // policy required and accepted a nonce; unreserved registrations require
2423        // no nonce check.
2424        let nonce_verified = if swap_candidate {
2425            true
2426        } else {
2427            match self.supervisor.reserved_hello_verification(
2428                &hello.manifest.module_id,
2429                hello.launch_nonce.as_deref(),
2430            ) {
2431                Ok(verified) => verified,
2432                Err(rejection) => {
2433                    let message = match rejection {
2434                        ReservedHelloRejection::Exact { module_id } => format!(
2435                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2436                        ),
2437                        ReservedHelloRejection::Prefix {
2438                            prefix,
2439                            owner_module_id,
2440                        } => format!(
2441                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2442                            hello.manifest.module_id
2443                        ),
2444                    };
2445                    return Ok(vec![control_error_frame(
2446                        &frame,
2447                        "reserved_module",
2448                        message,
2449                    )?]);
2450                }
2451            }
2452        };
2453
2454        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2455            &hello.manifest.module_id,
2456            hello.manifest.capabilities.as_ref(),
2457        );
2458        if let Some(refusal) = reserved_capability_refusals.first() {
2459            let capability = refusal.capability.clone();
2460            let bound_module = refusal.claimants[0].clone();
2461            log_duplicate_claim_events(reserved_capability_refusals);
2462            return Ok(vec![control_error_frame(
2463                &frame,
2464                "reserved_capability",
2465                format!(
2466                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2467                    capability, bound_module, hello.manifest.module_id
2468                ),
2469            )?]);
2470        }
2471
2472        // A connection that already opened client routes must not also register as
2473        // a module: cleanup would then release only one side and leak the other.
2474        if self
2475            .forwarding
2476            .connection_has_client_routes(connection_id)
2477            .map_err(RouterError::Forwarding)?
2478        {
2479            return Ok(vec![control_error_frame(
2480                &frame,
2481                "invalid_hello",
2482                "connection has open client routes and cannot also register as a module",
2483            )?]);
2484        }
2485
2486        // Kept for scope sync authority, which goes only to the connection that
2487        // presented the module's current launch nonce. Recorded before the
2488        // registration is attempted: a connection whose registration then fails
2489        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2490        self.hello_launch_nonces
2491            .lock()
2492            .unwrap_or_else(|poisoned| poisoned.into_inner())
2493            .record(connection_id, hello.launch_nonce.as_deref());
2494        let control_ops = effective_module_control_ops(hello.control_ops);
2495        // Built before anything is registered so an encoding failure leaves no
2496        // registry or forwarding state behind.
2497        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2498        if swap_candidate {
2499            return self.register_swap_candidate(
2500                connection_id,
2501                sink,
2502                &frame,
2503                hello.manifest,
2504                negotiated_ver,
2505                control_ops,
2506                hello_ack,
2507            );
2508        }
2509        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2510        // is refused, but `catalog.list` does not show the module and the
2511        // catalog generation does not move until it is published below, once
2512        // its forwarding endpoint exists. Listing it now would advertise a
2513        // module that `route.open` still refuses for having no forwarding
2514        // connection.
2515        let registration = match self
2516            .registry
2517            .register_unpublished_verified_with_control_ops(
2518                hello.manifest,
2519                negotiated_ver,
2520                connection_id,
2521                control_ops,
2522                nonce_verified,
2523            ) {
2524            Ok(registration) => registration,
2525            Err(RegistryError::DuplicateModuleId { module_id }) => {
2526                return Ok(vec![control_error_frame(
2527                    &frame,
2528                    "duplicate_module_id",
2529                    format!(
2530                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2531                    ),
2532                )?])
2533            }
2534            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2535                return Ok(vec![control_error_frame(
2536                    &frame,
2537                    "invalid_module_id",
2538                    err.to_string(),
2539                )?])
2540            }
2541            Err(err) => {
2542                return Ok(vec![control_error_frame(
2543                    &frame,
2544                    "registry_error",
2545                    err.to_string(),
2546                )?])
2547            }
2548        };
2549        #[cfg(test)]
2550        self.hold_between_registry_and_forwarding();
2551
2552        let reply = if let Some(sink) = sink {
2553            // The forwarding table's module store is also the daemon-to-module
2554            // control-RPC lane, so every HELLO gets a live endpoint even when the
2555            // manifest has no routable provider role. Non-routable modules still
2556            // cannot receive route.bind in production: `handle_route_open` checks
2557            // the registry manifest with `target_has_required_role` before the
2558            // only production call to `begin_route_bind_relay_for` below that
2559            // route.open path. The remaining direct relay callers are unit tests
2560            // and benchmark harnesses that construct forwarding state explicitly.
2561            //
2562            // The HELLO_ACK is queued by the forwarding table itself, before the
2563            // endpoint becomes visible, and is NOT returned as a reply. A module
2564            // reads HELLO_ACK first and exits on anything else; a reply is only
2565            // written after this handler returns, by which time a route.open on
2566            // another connection could already have queued a route.bind request
2567            // for this module ahead of it.
2568            let concurrency = manifest_concurrency(&registration.manifest);
2569            if let Err(err) = self.forwarding.register_module_connection_acked(
2570                connection_id,
2571                registration.manifest.module_id.clone(),
2572                negotiated_ver,
2573                concurrency,
2574                sink,
2575                hello_ack,
2576            ) {
2577                // Forwarding registration failed, so there is no forwarding
2578                // state to tear down. Remove the registry entry, which was never
2579                // published and so never listed or counted in the catalog
2580                // generation, and signal the release watch directly.
2581                if matches!(
2582                    self.deregister_connection(
2583                        connection_id,
2584                        RegistrationEndReason::RegistrationFailed,
2585                    ),
2586                    Ok(r) if !r.is_empty()
2587                ) {
2588                    crate::supervise::notify_registration_release();
2589                }
2590                return Ok(vec![control_error_frame(
2591                    &frame,
2592                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2593                        "invalid_hello"
2594                    } else {
2595                        forwarding_error_code(&err)
2596                    },
2597                    err.to_string(),
2598                )?]);
2599            }
2600            Vec::new()
2601        } else {
2602            // No sink means no forwarding endpoint, so nothing can be routed
2603            // ahead of the ack; it goes out as the reply.
2604            vec![hello_ack]
2605        };
2606
2607        // The module is routable now (or, without a sink, as routable as it
2608        // will ever be), so the catalog may list it. Publishing bumps the
2609        // catalog generation in the same registry critical section that makes
2610        // the module listed.
2611        if !self
2612            .registry
2613            .publish_registration(connection_id)
2614            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2615        {
2616            debug!(
2617                module_id = %registration.manifest.module_id,
2618                connection_id = connection_id.get(),
2619                "registration was removed before it could be published"
2620            );
2621        }
2622
2623        // Exposure over assumption: Concurrency's serde default is pinned to the
2624        // pre-field behavior (ModuleManaged), so a management surface that is
2625        // genuinely Serial and just never declared it inherits concurrent
2626        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2627        // turns "no module has been bitten yet" into the checkable claim "no
2628        // module is exposed" -- one read of the boot log instead of a fleet
2629        // audit. Detected from the raw HELLO bytes because the serde default
2630        // deliberately erases the absent/declared distinction from the type.
2631        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2632            info!(
2633                module_id = %registration.manifest.module_id,
2634                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2635            );
2636        }
2637
2638        self.apply_registration_capabilities(&registration);
2639
2640        self.supervisor
2641            .record_activity_registration(&registration.manifest.module_id);
2642
2643        info!(
2644            module_id = %registration.manifest.module_id,
2645            module_version = %registration.manifest.module_version,
2646            negotiated_ver,
2647            routable_provider = manifest_provides_routable_role(&registration.manifest),
2648            connection_id = connection_id.get(),
2649            "module registered"
2650        );
2651
2652        Ok(reply)
2653    }
2654
2655    /// Register a HELLO the swap gate admitted into the candidate slot of the
2656    /// registry and of forwarding, where it is reachable over its own
2657    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2658    /// nothing routes to it until the supervisor cuts over.
2659    ///
2660    /// Registry first (unpublished), then forwarding, then publication, the
2661    /// same order as an ordinary HELLO; a forwarding failure removes the
2662    /// registry entry again. The swap supervisor treats a published, ready
2663    /// candidate as registered and promotes it, and promotion needs the
2664    /// candidate's forwarding endpoint, so the candidate is published only
2665    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2666    /// recompute capability verdicts or run the capability deny census
2667    /// (`enforce_capability_denies`): both are about the module in the active
2668    /// slot, which `route.open` can target. A candidate sits in the candidate
2669    /// slot, so it neither provides capabilities to other modules nor can be
2670    /// routed to until promotion moves it into the active slot.
2671    #[allow(clippy::too_many_arguments)]
2672    fn register_swap_candidate(
2673        &self,
2674        connection_id: ConnectionId,
2675        sink: Option<crate::FrameSink>,
2676        frame: &Frame,
2677        manifest: ModuleManifest,
2678        negotiated_ver: u8,
2679        control_ops: Vec<String>,
2680        hello_ack: Frame,
2681    ) -> Result<Vec<Frame>, RouterError> {
2682        let module_id = manifest.module_id.clone();
2683        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2684            manifest,
2685            negotiated_ver,
2686            connection_id,
2687            control_ops,
2688        ) {
2689            Ok(registration) => registration,
2690            Err(RegistryError::DuplicateModuleId { module_id }) => {
2691                return Ok(vec![control_error_frame(
2692                    frame,
2693                    "duplicate_module_id",
2694                    format!(
2695                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2696                    ),
2697                )?])
2698            }
2699            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2700                return Ok(vec![control_error_frame(
2701                    frame,
2702                    "invalid_module_id",
2703                    err.to_string(),
2704                )?])
2705            }
2706            Err(err) => {
2707                return Ok(vec![control_error_frame(
2708                    frame,
2709                    "registry_error",
2710                    err.to_string(),
2711                )?])
2712            }
2713        };
2714        #[cfg(test)]
2715        self.hold_between_registry_and_forwarding();
2716        let reply = if let Some(sink) = sink {
2717            // Same ordering as an ordinary HELLO: the forwarding table queues
2718            // the HELLO_ACK before the candidate endpoint is inserted, because
2719            // a module exits if its first frame after HELLO is anything else.
2720            let concurrency = manifest_concurrency(&registration.manifest);
2721            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2722                connection_id,
2723                module_id.clone(),
2724                negotiated_ver,
2725                concurrency,
2726                sink,
2727                hello_ack,
2728            ) {
2729                if matches!(
2730                    self.deregister_connection(
2731                        connection_id,
2732                        RegistrationEndReason::RegistrationFailed,
2733                    ),
2734                    Ok(r) if !r.is_empty()
2735                ) {
2736                    crate::supervise::notify_registration_release();
2737                }
2738                return Ok(vec![control_error_frame(
2739                    frame,
2740                    forwarding_error_code(&err),
2741                    err.to_string(),
2742                )?]);
2743            }
2744            Vec::new()
2745        } else {
2746            vec![hello_ack]
2747        };
2748        if !self
2749            .registry
2750            .publish_registration(connection_id)
2751            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2752        {
2753            debug!(
2754                module_id = %module_id,
2755                connection_id = connection_id.get(),
2756                "swap candidate registration was removed before it could be published"
2757            );
2758        }
2759        self.supervisor.mark_swap_candidate_admitted(&module_id);
2760        info!(
2761            module_id = %module_id,
2762            module_version = %registration.manifest.module_version,
2763            negotiated_ver,
2764            ready = registration.ready,
2765            connection_id = connection_id.get(),
2766            "swap candidate registered; not routable until cutover"
2767        );
2768        Ok(reply)
2769    }
2770
2771    fn build_hello_ack(
2772        &self,
2773        frame: &Frame,
2774        negotiated_ver: u8,
2775        module_id: &str,
2776    ) -> Result<Frame, RouterError> {
2777        let ack = ModuleHelloAckBody {
2778            negotiated_ver,
2779            subc_ops: module_subc_ops(),
2780            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2781            storage: self
2782                .storage_config
2783                .as_ref()
2784                .map(|cfg| cfg.descriptor_for(module_id)),
2785            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2786        };
2787        let body = serde_json::to_vec(&ack).map_err(|err| {
2788            RouterError::backend(
2789                0,
2790                frame.header.corr,
2791                format!("failed to encode HELLO_ACK: {err}"),
2792            )
2793        })?;
2794
2795        Frame::build_with_version(
2796            negotiated_ver,
2797            FrameType::HelloAck,
2798            control_flags(),
2799            0,
2800            0,
2801            frame.header.corr,
2802            body,
2803        )
2804        .map_err(RouterError::FrameBuild)
2805    }
2806
2807    async fn handle_client_control_request(
2808        &self,
2809        ctx: &RouteCtx,
2810        frame: Frame,
2811        request: ClientControlRequest,
2812        route_open_timings: &mut Option<RouteOpenTimings>,
2813    ) -> Result<Vec<Frame>, RouterError> {
2814        if let Some(operation) = privileged_operation(&request) {
2815            let caller = self.resolve_caller(ctx.connection_id).await;
2816            let op = client_control_request_op(&request);
2817            let text = prompt_text(operation, &caller);
2818            match self.privileged_operator_prompt {
2819                PrivilegedOperatorPrompt::Log => {
2820                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2821                        target = ?operation, caller = %caller.description(), summary = %text,
2822                        "privileged request would prompt the operator");
2823                }
2824                PrivilegedOperatorPrompt::Enforce => {
2825                    let key =
2826                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2827                    let outcome = self
2828                        .forwarding
2829                        .operator_confirms()
2830                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2831                        .await;
2832                    if outcome.code != "confirmed" {
2833                        return Ok(vec![outcome.frame(&frame)]);
2834                    }
2835                }
2836            }
2837        }
2838        match request {
2839            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2840            ClientControlRequest::CatalogList { module_id } => {
2841                self.handle_catalog_list(frame, module_id)
2842            }
2843            ClientControlRequest::RouteOpen {
2844                target,
2845                identity,
2846                consumer_identity,
2847                consumer_capabilities,
2848                role_versions,
2849                admission_facts,
2850                scope,
2851            } => {
2852                self.handle_route_open(
2853                    ctx,
2854                    frame,
2855                    RouteOpenRequest {
2856                        target,
2857                        identity,
2858                        consumer_identity,
2859                        consumer_capabilities,
2860                        role_versions,
2861                        admission_facts,
2862                        scope,
2863                    },
2864                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2865                )
2866                .await
2867            }
2868            ClientControlRequest::RoutePoll {
2869                route_channel,
2870                route_epoch,
2871                kind,
2872            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2873            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2874            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2875                self.handle_supervisor_spawn_snapshot(frame)
2876            }
2877            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2878                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2879            }
2880            ClientControlRequest::SupervisorRestart {
2881                module_id,
2882                drain_timeout_ms,
2883            } => {
2884                self.log_supervisor_request_received(
2885                    ctx,
2886                    frame.header.corr,
2887                    ops::SUPERVISOR_RESTART,
2888                    Some(&module_id),
2889                    None,
2890                )?;
2891                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2892                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2893                    .await
2894            }
2895            ClientControlRequest::SupervisorSwap {
2896                module_id,
2897                ready_timeout_ms,
2898            } => {
2899                self.log_supervisor_request_received(
2900                    ctx,
2901                    frame.header.corr,
2902                    ops::SUPERVISOR_SWAP,
2903                    Some(&module_id),
2904                    None,
2905                )?;
2906                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2907                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2908                    .await
2909            }
2910            ClientControlRequest::SupervisorReload { module_id } => {
2911                self.log_supervisor_request_received(
2912                    ctx,
2913                    frame.header.corr,
2914                    ops::SUPERVISOR_RELOAD,
2915                    Some(&module_id),
2916                    None,
2917                )?;
2918                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2919                self.handle_supervisor_reload(frame, module_id, requester)
2920                    .await
2921            }
2922            ClientControlRequest::SupervisorRescan { preview } => {
2923                if !preview {
2924                    self.log_supervisor_request_received(
2925                        ctx,
2926                        frame.header.corr,
2927                        ops::SUPERVISOR_RESCAN,
2928                        None,
2929                        None,
2930                    )?;
2931                }
2932                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2933                self.handle_supervisor_rescan(
2934                    frame,
2935                    if preview {
2936                        RescanMode::Preview
2937                    } else {
2938                        RescanMode::Execute
2939                    },
2940                    requester,
2941                )
2942                .await
2943            }
2944            ClientControlRequest::SupervisorRescanChecked {
2945                preview,
2946                candidate,
2947                expect,
2948            } => {
2949                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2950                    .await
2951            }
2952            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2953                self.log_supervisor_request_received(
2954                    ctx,
2955                    frame.header.corr,
2956                    ops::SUPERVISOR_RELEASE_RESERVED,
2957                    Some(&module_id),
2958                    None,
2959                )?;
2960                self.handle_supervisor_release_reserved(frame, module_id)
2961                    .await
2962            }
2963            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2964                self.log_supervisor_request_received(
2965                    ctx,
2966                    frame.header.corr,
2967                    ops::SUPERVISOR_SET_ENABLED,
2968                    Some(&module_id),
2969                    Some(enabled),
2970                )?;
2971                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2972                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2973                    .await
2974            }
2975            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2976                self.handle_supervisor_health_probe(frame, module_id).await
2977            }
2978            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2979            ClientControlRequest::SupervisorRoutes { module_id } => {
2980                self.handle_supervisor_routes(frame, module_id)
2981            }
2982            ClientControlRequest::SupervisorProvenance { module_id } => {
2983                self.handle_supervisor_provenance(frame, module_id).await
2984            }
2985            ClientControlRequest::SupervisorStderrTail {
2986                module_id,
2987                max_lines,
2988                max_bytes,
2989            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2990            ClientControlRequest::SupervisorTerminals { module_id } => {
2991                self.handle_supervisor_terminals(frame, module_id).await
2992            }
2993        }
2994    }
2995
2996    fn log_supervisor_request_received(
2997        &self,
2998        ctx: &RouteCtx,
2999        corr: u64,
3000        op: &'static str,
3001        module_id: Option<&str>,
3002        enabled: Option<bool>,
3003    ) -> Result<(), RouterError> {
3004        let caller = self.supervisor_requester(ctx, corr)?;
3005        let caller = principal_label(&caller);
3006        // `Option` fields are recorded only when present, so a request with no
3007        // module id (rescan) or no enabled flag simply omits that field.
3008        info!(
3009            target: "control",
3010            op,
3011            module_id,
3012            enabled,
3013            connection_id = ctx.connection_id.get(),
3014            caller = %caller,
3015            "supervisor request received"
3016        );
3017        Ok(())
3018    }
3019
3020    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3021        Ok(self
3022            .registry
3023            .get_module_by_connection(ctx.connection_id)
3024            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3025            .map(|registration| Principal::Reserved {
3026                module_id: registration.manifest.module_id,
3027            })
3028            .unwrap_or(Principal::Direct))
3029    }
3030
3031    fn handle_module_control_request(
3032        &self,
3033        connection_id: ConnectionId,
3034        frame: Frame,
3035        request: ModuleControlRequestFromModule,
3036    ) -> Result<Vec<Frame>, RouterError> {
3037        match request {
3038            ModuleControlRequestFromModule::CatalogUpdate {
3039                provides,
3040                capabilities,
3041                ready,
3042            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3043            ModuleControlRequestFromModule::LiveRoots {} => {
3044                let registered = self
3045                    .registry
3046                    .get_module_by_connection(connection_id)
3047                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3048                let Some(registration) = registered else {
3049                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3050                };
3051                let response = self
3052                    .forwarding
3053                    .live_roots(&registration.manifest.module_id)
3054                    .map_err(RouterError::Forwarding)?;
3055                Ok(vec![control_response_body_frame(
3056                    &frame,
3057                    &response,
3058                    "ModuleControlResponseToModule::LiveRoots",
3059                )?])
3060            }
3061            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3062                self.handle_scope_sync(connection_id, frame, generation, scopes)
3063            }
3064            ModuleControlRequestFromModule::ScopeApply {
3065                generation,
3066                upsert,
3067                end,
3068            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3069            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3070                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3071            }
3072        }
3073    }
3074
3075    fn handle_scope_served_by(
3076        &self,
3077        connection_id: ConnectionId,
3078        frame: Frame,
3079    ) -> Result<Vec<Frame>, RouterError> {
3080        let registered = self
3081            .registry
3082            .get_module_by_connection(connection_id)
3083            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3084        let Some(registered) = registered else {
3085            return Ok(vec![control_error_frame(
3086                &frame,
3087                "not_registered",
3088                "scope.served_by requires an active module registration owned by this connection",
3089            )?]);
3090        };
3091        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3092        {
3093            Ok(request) => request,
3094            Err(err) => {
3095                return Ok(vec![control_error_frame(
3096                    &frame,
3097                    "invalid_control_body",
3098                    format!("malformed scope.served_by body: {err}"),
3099                )?])
3100            }
3101        };
3102        let answer = self
3103            .scopes
3104            .read()
3105            .unwrap_or_else(|poisoned| poisoned.into_inner())
3106            .served_by(
3107                &registered.manifest.module_id,
3108                &request.scope_ref,
3109                request.scope_epoch,
3110                &request.module_id,
3111            );
3112        let reply = subc_control::ScopeServedByReply::new(
3113            answer,
3114            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3115        );
3116        Ok(vec![control_response_body_frame(
3117            &frame,
3118            &reply,
3119            "ScopeServedByReply",
3120        )?])
3121    }
3122
3123    fn handle_operator_confirm(
3124        &self,
3125        ctx: &RouteCtx,
3126        frame: Frame,
3127    ) -> Result<Vec<Frame>, RouterError> {
3128        use crate::operator_confirm::{audit, Outcome};
3129        let registration = self
3130            .registry
3131            .get_module_by_connection(ctx.connection_id)
3132            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3133        let Some(registration) = registration else {
3134            let outcome = Outcome::refusal("not_registered");
3135            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3136            return Ok(vec![outcome.frame(&frame)]);
3137        };
3138        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3139            Ok(request) => request,
3140            Err(_) => {
3141                let outcome = Outcome::refusal("invalid_control_body");
3142                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3143                return Ok(vec![outcome.frame(&frame)]);
3144            }
3145        };
3146        let module_id = registration.manifest.module_id;
3147        // Read the launch nonce (under its own lock) before taking the forwarding
3148        // table's lock below: holding forwarding while waiting on another daemon
3149        // lock risks a lock-order deadlock with paths that take them the other way.
3150        let nonce = self
3151            .hello_launch_nonces
3152            .lock()
3153            .unwrap_or_else(|p| p.into_inner())
3154            .nonce(ctx.connection_id)
3155            .map(str::to_owned);
3156        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3157            self.supervisor
3158                .spawned_consumer_authorized(&module_id, nonce)
3159        });
3160        let confirms = self.forwarding.operator_confirms();
3161        self.forwarding
3162            .with_operator_route(
3163                ctx.connection_id,
3164                request.route_channel,
3165                request.route_epoch,
3166                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3167            )
3168            .map_err(RouterError::Forwarding)
3169    }
3170
3171    /// Resolve folders before taking the scope-table lock: a slow filesystem
3172    /// must not hold up admissions for unrelated scoped routes.
3173    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3174        let mut scopes = ScopeRecords::from(scopes);
3175        for (index, record) in scopes.records.iter_mut().enumerate() {
3176            if let Some(refusal) = Self::canonicalize_workdir(record) {
3177                scopes.refusals.insert(index, refusal);
3178            }
3179        }
3180        scopes
3181    }
3182
3183    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3184        let workdir = record.workdir.as_mut()?;
3185        let path = std::path::Path::new(workdir);
3186        if !path.is_absolute() {
3187            return Some((
3188                error_codes::INVALID_WORKDIR,
3189                "workdir must be absolute".to_string(),
3190            ));
3191        }
3192        match ProjectRootId::from_path_allowing_missing(path) {
3193            Ok(root) => match root.as_path().to_str() {
3194                Some(canonical) => {
3195                    *workdir = canonical.to_string();
3196                    None
3197                }
3198                None => Some((
3199                    error_codes::INVALID_WORKDIR,
3200                    "canonical workdir is not UTF-8".to_string(),
3201                )),
3202            },
3203            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3204        }
3205    }
3206
3207    /// `scope.sync`: the owner is the module registered on this connection.
3208    /// A connection with no registration (every client connection, `direct`
3209    /// included) is refused `not_registered` before the table is consulted.
3210    fn handle_scope_sync(
3211        &self,
3212        connection_id: ConnectionId,
3213        frame: Frame,
3214        generation: u64,
3215        scopes: Vec<ScopeRecord>,
3216    ) -> Result<Vec<Frame>, RouterError> {
3217        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3218    }
3219
3220    fn handle_scope_change(
3221        &self,
3222        connection_id: ConnectionId,
3223        frame: Frame,
3224        generation: u64,
3225        scopes: Vec<ScopeRecord>,
3226        end: Option<Vec<ScopeEnd>>,
3227    ) -> Result<Vec<Frame>, RouterError> {
3228        let is_apply = end.is_some();
3229        let Some(registration) = self
3230            .registry
3231            .get_module_by_connection(connection_id)
3232            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3233        else {
3234            return Ok(vec![control_error_frame(
3235                &frame,
3236                "not_registered",
3237                "scope.sync and scope.apply require an active module registration owned by this connection",
3238            )?]);
3239        };
3240        let owner = registration.manifest.module_id;
3241        let scopes = Self::prepare_scope_records(scopes);
3242        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3243        let is_current_launch = |connection: ConnectionId| {
3244            self.hello_launch_nonces
3245                .lock()
3246                .unwrap_or_else(|poisoned| poisoned.into_inner())
3247                .presented(connection, current_nonce.as_deref())
3248        };
3249        // Lock order is the scope table, then the forwarding table: the new
3250        // tags are published, and the routes the change closes are selected,
3251        // while the scope table is still write-locked, so no admission can read
3252        // a record whose tag is not yet published.
3253        let mut table = self
3254            .scopes
3255            .write()
3256            .unwrap_or_else(|poisoned| poisoned.into_inner());
3257        let outcome = match end {
3258            Some(end) => table.apply(
3259                &owner,
3260                connection_id,
3261                is_current_launch,
3262                generation,
3263                scopes,
3264                end,
3265            ),
3266            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3267        };
3268        let drained = match &outcome {
3269            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3270            Err(_) => Vec::new(),
3271        };
3272        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3273        drop(table);
3274        if has_deadlines {
3275            self.scope_deadline_added.notify_one();
3276        }
3277        match outcome {
3278            Ok(applied) => {
3279                let counts = ScopeOutcomeCounts::of(&applied.results);
3280                info!(
3281                    owner = %owner,
3282                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3283                    generation,
3284                    records = applied.results.len(),
3285                    created = counts.created,
3286                    replaced = counts.replaced,
3287                    updated = counts.updated,
3288                    unchanged = counts.unchanged,
3289                    refused = counts.refused,
3290                    ended = applied.ended.len(),
3291                    tag_changes = applied.tag_changes.len(),
3292                    routes_closed = drained.len(),
3293                    "scope change accepted"
3294                );
3295                // An accepted scope change can still refuse individual records, and the
3296                // owner is the only party that sees the reply. Name them here so
3297                // an operator can tell a refused session from a missing one
3298                // without the owner's logs. Capped so a call that refuses
3299                // thousands cannot flood the log; the count above is complete.
3300                for refused in applied
3301                    .results
3302                    .iter()
3303                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3304                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3305                {
3306                    warn!(
3307                        owner = %owner,
3308                        generation,
3309                        scope_ref = %refused.scope_ref,
3310                        scope_epoch = refused.scope_epoch,
3311                        code = refused.code.as_deref().unwrap_or(""),
3312                        "scope record refused"
3313                    );
3314                }
3315                self.close_scope_drained_routes(drained);
3316                let response = if is_apply {
3317                    ModuleControlResponseToModule::ScopeApply {
3318                        generation,
3319                        results: applied.results,
3320                        end_results: applied.end_results,
3321                        ended: applied.ended,
3322                    }
3323                } else {
3324                    ModuleControlResponseToModule::ScopeSync {
3325                        generation,
3326                        results: applied.results,
3327                        ended: applied.ended,
3328                    }
3329                };
3330                Ok(vec![control_response_body_frame(
3331                    &frame,
3332                    &response,
3333                    "scope change response",
3334                )?])
3335            }
3336            Err(refusal) => {
3337                info!(
3338                    owner = %owner,
3339                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3340                    generation,
3341                    code = refusal.code,
3342                    "scope change refused"
3343                );
3344                Ok(vec![control_error_frame(
3345                    &frame,
3346                    refusal.code,
3347                    refusal.message,
3348                )?])
3349            }
3350        }
3351    }
3352
3353    /// Tell both ends of each route a scope change closed. The module gets a
3354    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3355    /// the client's route handle. The client also gets `route.closed` with the
3356    /// scope reason, one push per module and reason, so it can tell a revoked
3357    /// route from an ordinary close and not reopen it.
3358    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3359        if drained.is_empty() {
3360            return;
3361        }
3362        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3363            BTreeMap::new();
3364        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3365        for route in drained {
3366            warn!(
3367                module_id = %route.module_id,
3368                reason = ?route.reason,
3369                client_connection_id = route.client.connection_id.get(),
3370                route_channel = route.client.channel,
3371                "closing route because its scope changed"
3372            );
3373            pushes
3374                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3375                .or_insert_with(|| (route.reason, Vec::new()))
3376                .1
3377                .push(EndpointRoute {
3378                    goodbye_target: route.client.clone(),
3379                    principal: Principal::Unverified,
3380                    bound_at: Instant::now(),
3381                    draining: false,
3382                    drain_reason: None,
3383                });
3384            goodbyes.push(route.module);
3385            goodbyes.push(route.client);
3386        }
3387        for ((module_id, _), (reason, routes)) in pushes {
3388            send_route_control_pushes(
3389                &self.forwarding,
3390                routes,
3391                ClientControlPush::RouteClosed {
3392                    module_id,
3393                    channels: Vec::new(),
3394                    reason,
3395                    drained: false,
3396                    abandoned: 0,
3397                    excluded_subscriptions: 0,
3398                    terminal: Some(false),
3399                },
3400            );
3401        }
3402        self.emit_route_goodbyes(goodbyes);
3403    }
3404
3405    /// `scope.describe`: any registered module may read any scope, because a
3406    /// provider must read the scope a route it serves is stamped with.
3407    fn handle_scope_describe(
3408        &self,
3409        connection_id: ConnectionId,
3410        frame: Frame,
3411        owner: Principal,
3412        scope_ref: String,
3413    ) -> Result<Vec<Frame>, RouterError> {
3414        let registered = self
3415            .registry
3416            .get_module_by_connection(connection_id)
3417            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3418        let Some(registered) = registered else {
3419            return Ok(vec![control_error_frame(
3420                &frame,
3421                "not_registered",
3422                "scope.describe requires an active module registration owned by this connection",
3423            )?]);
3424        };
3425        let mut description = self
3426            .scopes
3427            .read()
3428            .unwrap_or_else(|poisoned| poisoned.into_inner())
3429            .describe(&owner, &scope_ref);
3430        // Describe readers also decode strict ScopeAttributes. Attribution is
3431        // safe to omit unless the requesting module declares it understands it.
3432        if !registered
3433            .manifest
3434            .capabilities
3435            .as_ref()
3436            .is_some_and(|caps| {
3437                caps.provides
3438                    .iter()
3439                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3440            })
3441        {
3442            if let Some(stamp) = description.stamp.as_mut() {
3443                stamp.attributes.on_behalf_of_agent = None;
3444            }
3445        }
3446        let owner_configured = match &owner {
3447            // Ask whether the owner is configured (`is_configured`), not
3448            // whether it is on the roster (`get(..).is_some()`): a supervised
3449            // module's process can register and describe a scope before the
3450            // supervisor has put it on the roster.
3451            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3452            _ => false,
3453        };
3454        let response = ModuleControlResponseToModule::ScopeDescribe {
3455            status: description.status,
3456            scope_epoch: description.scope_epoch,
3457            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3458            owner_synced: description.owner_synced,
3459            owner_configured,
3460            scope: description.stamp,
3461        };
3462        Ok(vec![control_response_body_frame(
3463            &frame,
3464            &response,
3465            "ModuleControlResponseToModule::ScopeDescribe",
3466        )?])
3467    }
3468
3469    fn handle_catalog_update(
3470        &self,
3471        connection_id: ConnectionId,
3472        frame: Frame,
3473        provides: Vec<ProviderRole>,
3474        capabilities: Option<CapabilityDeclarations>,
3475        ready: Option<bool>,
3476    ) -> Result<Vec<Frame>, RouterError> {
3477        self.refresh_capability_requirements();
3478        let Some(registration) = self
3479            .registry
3480            .get_module_by_connection(connection_id)
3481            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3482        else {
3483            return Ok(vec![control_error_frame(
3484                &frame,
3485                "not_registered",
3486                "catalog.update requires an active module registration owned by this connection",
3487            )?]);
3488        };
3489
3490        if let Some(message) =
3491            catalog_update_frozen_field_message(&registration.manifest, &provides)
3492        {
3493            return Ok(vec![control_error_frame(
3494                &frame,
3495                "catalog_update_frozen_field",
3496                message,
3497            )?]);
3498        }
3499
3500        let mut candidate = registration.manifest.clone();
3501        candidate.provides = provides.clone();
3502        candidate.capabilities = capabilities
3503            .clone()
3504            .or_else(|| registration.manifest.capabilities.clone());
3505        if let Some(rich_ui) = &candidate.rich_ui {
3506            if let Err(err) = rich_ui.validate() {
3507                return Ok(vec![control_error_frame(
3508                    &frame,
3509                    "invalid_manifest",
3510                    err.to_string(),
3511                )?]);
3512            }
3513        }
3514        if let Err(err) = candidate.validate_capability_grammar() {
3515            return Ok(vec![control_error_frame(
3516                &frame,
3517                "invalid_capability_grammar",
3518                err.to_string(),
3519            )?]);
3520        }
3521
3522        // Updates must honor the same reserved owner as initial registration;
3523        // otherwise an empty HELLO could acquire the claim after admission.
3524        let mut conflicts = self
3525            .capability_evaluator
3526            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3527        if let Some(conflict) = conflicts.first() {
3528            let message = format!(
3529                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3530                conflict.capability, conflict.claimants[0], candidate.module_id
3531            );
3532            for conflict in &mut conflicts {
3533                conflict.source = DuplicateClaimSource::CatalogUpdate;
3534            }
3535            log_duplicate_claim_events(conflicts);
3536            return Ok(vec![control_error_frame(
3537                &frame,
3538                "reserved_capability",
3539                message,
3540            )?]);
3541        }
3542
3543        let updated = self
3544            .registry
3545            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3546            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3547        if updated.is_none() {
3548            return Ok(vec![control_error_frame(
3549                &frame,
3550                "not_registered",
3551                "catalog.update requires an active module registration owned by this connection",
3552            )?]);
3553        }
3554        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3555            log_duplicate_claim_events(
3556                self.capability_evaluator
3557                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3558            );
3559        }
3560        if capability_census_trigger(
3561            registration.manifest.capabilities.as_ref(),
3562            updated
3563                .as_ref()
3564                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3565        ) {
3566            self.enforce_capability_denies();
3567        }
3568        self.refresh_capability_requirements();
3569
3570        let response = ModuleControlResponseToModule::CatalogUpdate {};
3571        control_response_body_frame(
3572            &frame,
3573            &response,
3574            "ModuleControlResponseToModule::CatalogUpdate",
3575        )
3576        .map(|frame| vec![frame])
3577    }
3578
3579    pub(crate) fn with_activity_ledger_config(
3580        mut self,
3581        run_dir: &Path,
3582        incarnation: String,
3583        config: crate::activity::ActivityConfig,
3584    ) -> Self {
3585        self.activity = Some(crate::activity::ActivityLedger::open_config(
3586            run_dir.join("activity.jsonl"),
3587            incarnation,
3588            config,
3589        ));
3590        self.supervisor.set_activity(self.activity.clone());
3591        self
3592    }
3593
3594    pub(crate) async fn configure_activity(
3595        &self,
3596        config: crate::activity::ActivityConfig,
3597    ) -> Result<(), &'static str> {
3598        match &self.activity {
3599            Some(ledger) => ledger.configure(config).await,
3600            None => Ok(()),
3601        }
3602    }
3603
3604    fn cancel_activity_subscription(
3605        &self,
3606        ctx: &RouteCtx,
3607        frame: &Frame,
3608    ) -> Result<bool, RouterError> {
3609        let task = self
3610            .activity_streams
3611            .lock()
3612            .unwrap_or_else(|p| p.into_inner())
3613            .remove(&(ctx.connection_id, frame.header.corr));
3614        let Some(task) = task else {
3615            return Ok(false);
3616        };
3617        task.abort();
3618        let end = Frame::build_with_version(
3619            response_version(frame),
3620            FrameType::StreamEnd,
3621            control_flags(),
3622            0,
3623            0,
3624            frame.header.corr,
3625            Vec::new(),
3626        )
3627        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3628        let sink = ctx.egress.clone();
3629        tokio::spawn(async move {
3630            let _ = sink.send(end).await;
3631        });
3632        Ok(true)
3633    }
3634
3635    async fn handle_activity_read(
3636        &self,
3637        ctx: &RouteCtx,
3638        frame: Frame,
3639    ) -> Result<Vec<Frame>, RouterError> {
3640        use subc_control::{
3641            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3642            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3643        };
3644        #[derive(Deserialize)]
3645        struct Identity {
3646            #[serde(default)]
3647            consumer_identity: Option<ConsumerIdentity>,
3648        }
3649        // A registered module connection already identifies its caller. Ignore
3650        // consumer_identity there so a module cannot borrow a mirror's identity.
3651        // Only separate consumer connections use the Direct/nonce trust boundary.
3652        let principal = match self
3653            .forwarding
3654            .module_id_for_connection(ctx.connection_id)
3655            .map_err(RouterError::Forwarding)?
3656        {
3657            Some(module_id) => Ok(Principal::Reserved { module_id }),
3658            None => {
3659                let identity: Identity = match serde_json::from_slice(&frame.body) {
3660                    Ok(identity) => identity,
3661                    Err(_) => {
3662                        return Ok(vec![control_error_frame(
3663                            &frame,
3664                            ACTIVITY_MALFORMED,
3665                            "malformed activity read",
3666                        )?])
3667                    }
3668                };
3669                self.route_open_principal(&frame, identity.consumer_identity)?
3670            }
3671        };
3672        let admitted = matches!(principal, Ok(Principal::Direct))
3673            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3674        if !admitted {
3675            return Ok(vec![control_error_frame(
3676                &frame,
3677                ACTIVITY_READ_FORBIDDEN,
3678                "activity reads require an operator or trusted mirror principal",
3679            )?]);
3680        }
3681        let Some(ledger) = &self.activity else {
3682            return Ok(vec![control_error_frame(
3683                &frame,
3684                ACTIVITY_UNAVAILABLE,
3685                "activity ledger unavailable",
3686            )?]);
3687        };
3688        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3689            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3690        {
3691            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3692                Ok(request) => ledger.query(request).await,
3693                Err(_) => Err(ACTIVITY_MALFORMED),
3694            };
3695            return match result {
3696                Ok(reply) => Ok(vec![control_response_body_frame(
3697                    &frame,
3698                    &reply,
3699                    ops::ACTIVITY_QUERY,
3700                )?]),
3701                Err(code) => Ok(vec![control_error_frame(
3702                    &frame,
3703                    code,
3704                    "activity query refused",
3705                )?]),
3706            };
3707        }
3708        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3709            Ok(request) => request,
3710            Err(_) => {
3711                return Ok(vec![control_error_frame(
3712                    &frame,
3713                    ACTIVITY_MALFORMED,
3714                    "malformed activity subscription",
3715                )?])
3716            }
3717        };
3718        let mut subscription = match ledger.subscribe(request.since).await {
3719            Ok(subscription) => subscription,
3720            Err(code) => {
3721                return Ok(vec![control_error_frame(
3722                    &frame,
3723                    code,
3724                    "activity subscription refused",
3725                )?])
3726            }
3727        };
3728        let key = (ctx.connection_id, frame.header.corr);
3729        let sink = ctx.egress.clone();
3730        let mut streams = self
3731            .activity_streams
3732            .lock()
3733            .unwrap_or_else(|p| p.into_inner());
3734        streams.retain(|_, task| !task.is_finished());
3735        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3736            return Ok(vec![control_error_frame(
3737                &frame,
3738                "duplicate_subscription",
3739                "activity subscription correlation id already in use",
3740            )?]);
3741        }
3742        let task = tokio::spawn(async move {
3743            loop {
3744                let entry = if let Some(entry) = subscription.replay.pop_front() {
3745                    Some(entry)
3746                } else {
3747                    subscription.receiver.recv().await
3748                };
3749                let Some(entry) = entry else {
3750                    break;
3751                };
3752                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3753                    return;
3754                };
3755                let Ok(next) = Frame::build_with_version(
3756                    response_version(&frame),
3757                    FrameType::StreamData,
3758                    control_flags(),
3759                    0,
3760                    0,
3761                    frame.header.corr,
3762                    body,
3763                ) else {
3764                    return;
3765                };
3766                if sink.send(next).await.is_err() {
3767                    return;
3768                }
3769            }
3770            // Dropping a full subscriber's sender drains its accepted entries
3771            // before the terminal error, preserving their order on the wire.
3772            if let Some(code) = subscription.terminal_code() {
3773                if let Ok(end) =
3774                    control_error_frame(&frame, code, "activity subscriber fell behind")
3775                {
3776                    let _ = sink.send(end).await;
3777                }
3778            }
3779        });
3780        streams.insert(key, task.abort_handle());
3781        Ok(Vec::new())
3782    }
3783
3784    async fn handle_activity_record(
3785        &self,
3786        ctx: &RouteCtx,
3787        frame: Frame,
3788    ) -> Result<Vec<Frame>, RouterError> {
3789        use subc_control::{
3790            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3791            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3792        };
3793        let result = async {
3794            let module = self
3795                .forwarding
3796                .module_id_for_connection(ctx.connection_id)
3797                .ok()
3798                .flatten()
3799                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3800            let request: ActivityRecordRequest =
3801                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3802            crate::activity::validate(&module, &request)?;
3803            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3804            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3805            let at_ms = std::time::SystemTime::now()
3806                .duration_since(std::time::UNIX_EPOCH)
3807                .unwrap_or_default()
3808                .as_millis() as u64;
3809            ledger.record(module, request, actor, at_ms).await
3810        }
3811        .await;
3812        match result {
3813            Ok(entry) => Ok(vec![control_response_body_frame(
3814                &frame,
3815                &ActivityRecordResponse { seq: entry.seq },
3816                "activity.record",
3817            )?]),
3818            Err(code) => Ok(vec![control_error_frame(
3819                &frame,
3820                code,
3821                "activity recording refused",
3822            )?]),
3823        }
3824    }
3825
3826    fn activity_actor(
3827        &self,
3828        connection: ConnectionId,
3829        module: &str,
3830        on: Option<&subc_control::ActivityOn>,
3831    ) -> Result<subc_control::ActivityActor, &'static str> {
3832        use crate::forwarding::{DataRoute, DataRouteState};
3833        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3834        let principal = |id: &str| Principal::Reserved {
3835            module_id: id.to_owned(),
3836        };
3837        let mut actor = ActivityActor {
3838            principal: principal(module),
3839            scope_owner: None,
3840            agent_id: None,
3841            run_id: None,
3842            flow_id: None,
3843            project_root: None,
3844            workdir: None,
3845            project_id: None,
3846        };
3847        let scope = match on {
3848            None => return Ok(actor),
3849            Some(ActivityOn::Scope(scope)) => {
3850                let selector = subc_protocol::scope::ScopeSelector::new(
3851                    principal(module),
3852                    scope.scope_ref.clone(),
3853                    Some(scope.epoch),
3854                );
3855                self.scopes
3856                    .read()
3857                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3858                    .attribute(&principal(module), module, &selector, true)
3859                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3860            }
3861            Some(ActivityOn::Route(route)) => {
3862                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3863                    .forwarding
3864                    .lookup_data_route(connection, route.channel, route.epoch)
3865                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3866                else {
3867                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3868                };
3869                if binding.module_id != module {
3870                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3871                }
3872                actor.principal = binding.principal.clone();
3873                actor.project_root = binding
3874                    .project_root
3875                    .as_ref()
3876                    .map(|root| root.as_path().to_path_buf());
3877                let Some(bound) = &binding.scope else {
3878                    return Ok(actor);
3879                };
3880                let owner = principal(&bound.owner);
3881                let selector = subc_protocol::scope::ScopeSelector::new(
3882                    owner.clone(),
3883                    bound.scope_ref.clone(),
3884                    Some(bound.tag.scope_epoch),
3885                );
3886                let admitted = self
3887                    .scopes
3888                    .read()
3889                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3890                    .attribute(&owner, module, &selector, true)
3891                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3892                if admitted.tag != bound.tag {
3893                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3894                }
3895                admitted
3896            }
3897        };
3898        actor.scope_owner = Some(scope.stamp.owner);
3899        actor.agent_id = scope.stamp.attributes.agent_id;
3900        actor.run_id = scope.stamp.attributes.run_id;
3901        actor.flow_id = scope.stamp.attributes.flow_id;
3902        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3903        actor.project_id = scope.stamp.project_id;
3904        Ok(actor)
3905    }
3906
3907    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3908        self.refresh_capability_requirements();
3909        // A bare connection count is ambiguous between many clients holding a
3910        // route each and one client accumulating hundreds, so publish the
3911        // concentration alongside it. Route state is best-effort here: a
3912        // diagnostic endpoint must still answer if the forwarding lock is
3913        // contended.
3914        let mut counters = self.counters.snapshot();
3915        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3916            obj.insert(
3917                "activity_unparseable_lines".into(),
3918                ledger.read_errors().into(),
3919            );
3920            obj.insert(
3921                "activity_lifecycle_append_failures".into(),
3922                ledger.lifecycle_failures().into(),
3923            );
3924            obj.insert(
3925                "activity_started_digest_skips".into(),
3926                ledger.lifecycle_digest_skips().into(),
3927            );
3928        }
3929        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3930            self.forwarding.client_route_concentration(),
3931            counters.as_object_mut(),
3932        ) {
3933            obj.insert(
3934                "client_connections_with_routes".into(),
3935                connections_with_routes.into(),
3936            );
3937            obj.insert("max_routes_on_one_connection".into(), max.into());
3938        }
3939        // A module that is being fast-refused and a module that is fine look
3940        // identical from a client that retries and succeeds, so name the open
3941        // breakers here. This rides the existing free-form counters object
3942        // rather than a new wire field, so no sibling that deserializes
3943        // `ServerDescribe` has to be rebuilt to keep reading it.
3944        if let (Some(open_breakers), Some(obj)) = (
3945            self.route_bind_breakers.open_snapshot(),
3946            counters.as_object_mut(),
3947        ) {
3948            obj.insert("route_bind_breakers_open".into(), open_breakers);
3949        }
3950        let response = ClientControlResponse::ServerDescribe {
3951            protocol_ver: PROTOCOL_VERSION,
3952            subc_ops: subc_ops(),
3953            capabilities: self.describe_capabilities(),
3954            connected_clients: self.connected_clients.count(),
3955            counters: Some(counters),
3956            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3957            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3958            capability_requirements: self.capability_requirement_statuses(),
3959            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3960            config_path: self
3961                .rescan
3962                .as_ref()
3963                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3964        };
3965        Ok(vec![control_response_body_frame(
3966            &frame,
3967            &response,
3968            "ClientControlResponse::ServerDescribe",
3969        )?])
3970    }
3971
3972    fn handle_catalog_list(
3973        &self,
3974        frame: Frame,
3975        module_id: Option<String>,
3976    ) -> Result<Vec<Frame>, RouterError> {
3977        let (generation, modules) = self
3978            .registry
3979            .list_modules_with_nonce_verification()
3980            .map_err(|err| {
3981                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3982            })?;
3983        let entries = modules
3984            .into_iter()
3985            .filter(|(registration, _)| {
3986                module_id
3987                    .as_deref()
3988                    .map(|wanted| registration.manifest.module_id == wanted)
3989                    .unwrap_or(true)
3990            })
3991            .map(|(registration, nonce_verified)| {
3992                let not_ready = self.not_ready_reason(&registration);
3993                let roles = registration.manifest.provides;
3994                CatalogEntry::new(
3995                    registration.manifest.module_id,
3996                    roles,
3997                    registration.control_ops,
3998                )
3999                .with_ready(not_ready.is_none())
4000                .with_not_ready(not_ready)
4001                .with_module_version(Some(registration.manifest.module_version))
4002                .with_capabilities(registration.manifest.capabilities)
4003                .with_self_signals(registration.manifest.self_signals)
4004                .with_events(registration.manifest.events)
4005                .with_operation_delivery(registration.manifest.operation_delivery)
4006                .with_operation_payload(registration.manifest.operation_payload)
4007                .with_rich_ui(registration.manifest.rich_ui)
4008                .with_role_declarations(registration.manifest.role_declarations)
4009                .with_nonce_verified(nonce_verified)
4010            })
4011            .collect();
4012        let response = ClientControlResponse::CatalogList {
4013            generation,
4014            modules: entries,
4015            subc_ops: subc_ops(),
4016        };
4017        Ok(vec![control_response_body_frame(
4018            &frame,
4019            &response,
4020            "ClientControlResponse::CatalogList",
4021        )?])
4022    }
4023
4024    /// The principal a route.open's `consumer_identity` attests: `Direct`
4025    /// when it presents none, `Reserved` when its launch nonce verifies, and
4026    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4027    /// can run before any refusal check without changing which check refuses.
4028    fn attest_route_open_consumer(
4029        &self,
4030        consumer_identity: Option<&ConsumerIdentity>,
4031    ) -> Option<Principal> {
4032        let Some(consumer_identity) = consumer_identity else {
4033            return Some(Principal::Direct);
4034        };
4035        self.supervisor
4036            .spawned_consumer_authorized(
4037                &consumer_identity.module_id,
4038                &consumer_identity.launch_nonce,
4039            )
4040            .then(|| Principal::Reserved {
4041                module_id: consumer_identity.module_id.clone(),
4042            })
4043    }
4044
4045    /// The caller fields for a refusal that only has the raw request frame,
4046    /// such as the connection-level admission refusal sent before the request
4047    /// reaches [`Self::handle_route_open`].
4048    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4049        let Ok(ClientControlRequest::RouteOpen {
4050            identity,
4051            consumer_identity,
4052            scope,
4053            ..
4054        }) = parse_client_control_request(&frame.body)
4055        else {
4056            return RouteOpenCaller::unknown();
4057        };
4058        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4059        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4060    }
4061
4062    fn route_open_principal(
4063        &self,
4064        frame: &Frame,
4065        consumer_identity: Option<ConsumerIdentity>,
4066    ) -> Result<Result<Principal, Frame>, RouterError> {
4067        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4068        self.route_open_principal_attested(frame, consumer_identity, attested)
4069    }
4070
4071    /// [`Self::route_open_principal`] with `attested` already taken by
4072    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4073    /// open checks the launch nonce once.
4074    fn route_open_principal_attested(
4075        &self,
4076        frame: &Frame,
4077        consumer_identity: Option<ConsumerIdentity>,
4078        attested: Option<Principal>,
4079    ) -> Result<Result<Principal, Frame>, RouterError> {
4080        if let Some(principal) = attested {
4081            return Ok(Ok(principal));
4082        }
4083        let Some(consumer_identity) = consumer_identity else {
4084            return Ok(Ok(Principal::Direct));
4085        };
4086
4087        Ok(Err(control_error_frame(
4088            frame,
4089            "bad_consumer_identity",
4090            format!(
4091                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4092                consumer_identity.module_id
4093            ),
4094        )?))
4095    }
4096
4097    /// Ordinary `route.open` refusals go through here; admission and breaker
4098    /// refusals log separately with their capacity or breaker state. The daemon can
4099    /// attest which code it sent: without the event, a client's "the daemon
4100    /// refused me" and the daemon's own view could only be reconciled by
4101    /// argument. Malformed input (`invalid_project_root`) does not come here;
4102    /// rejecting a request that was never a valid open is not a refusal of one.
4103    #[allow(clippy::too_many_arguments)]
4104    fn route_open_refusal_frame(
4105        &self,
4106        ctx: &RouteCtx,
4107        caller: &RouteOpenCaller,
4108        frame: &Frame,
4109        module_id: &str,
4110        reason: &'static str,
4111        code: &'static str,
4112        message: impl Into<String>,
4113    ) -> Result<Frame, RouterError> {
4114        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4115        control_error_frame(frame, code, message.into())
4116    }
4117
4118    /// Refuse a `route.open` because the target module's bind-relay breaker is
4119    /// open, without attempting the relay.
4120    ///
4121    /// The wire code is `module_timeout`, which is the truth (the module has
4122    /// not been answering binds) and which both SDKs already classify as
4123    /// retryable with capped backoff. Reusing it is what keeps this change out
4124    /// of both SDKs; the daemon-side distinction lives in the counter key
4125    /// instead.
4126    ///
4127    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4128    /// While a breaker is open this fires on every open to that module, and the
4129    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4130    /// produced 261 lines about a single module inside 3000 lines of daemon
4131    /// log. The rare transitions are logged at warn/info instead and the volume
4132    /// is carried by the counter, so the evidence survives without the flood.
4133    /// The debug line keeps a per-refusal record reachable for whoever turns
4134    /// the level up.
4135    #[allow(clippy::too_many_arguments)]
4136    fn route_open_breaker_refusal_frame(
4137        &self,
4138        ctx: &RouteCtx,
4139        caller: &RouteOpenCaller,
4140        frame: &Frame,
4141        module_id: &str,
4142        consecutive_timeouts: u32,
4143        retry_in: Duration,
4144        probe_in_flight: bool,
4145    ) -> Result<Frame, RouterError> {
4146        self.counters
4147            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4148        debug!(
4149            target: "control",
4150            code = "module_timeout",
4151            module_id = ?module_id,
4152            connection_id = ctx.connection_id.get(),
4153            principal = %caller.principal,
4154            scope = %caller.scope,
4155            harness = %caller.harness,
4156            consecutive_timeouts,
4157            retry_in_ms = retry_in.as_millis() as u64,
4158            probe_in_flight,
4159            "route.open refused by open bind-relay breaker"
4160        );
4161        // Say what a caller can act on. An open bind-relay breaker means the
4162        // module timed out accepting several new routes in a row. The module
4163        // is still running and its established routes keep working; only new
4164        // route.open requests are refused until the cooldown ends and one
4165        // test route (the probe) gets through. A message that only counts
4166        // failed relays reads as "the module is down" to a worker that sees it.
4167        let detail = if probe_in_flight {
4168            "one test route is already being tried; retry once it settles".to_string()
4169        } else {
4170            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4171        };
4172        control_error_frame(
4173            frame,
4174            "module_timeout",
4175            format!(
4176                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4177                 timed out in a row); {detail}; its established routes are unaffected"
4178            ),
4179        )
4180    }
4181
4182    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4183    /// requester's bytes (an unknown target is whatever the client sent) and
4184    /// is Debug-formatted so control characters land in the log escaped
4185    /// rather than as terminal sequences for whoever tails it.
4186    ///
4187    /// `reason` names the check that refused, because one wire code has
4188    /// several senders: after a module registers, `target_unavailable` can
4189    /// come from a missing role, an inactive registration, a supervisor that
4190    /// has not marked the process live, a missing forwarding connection, or a
4191    /// failed relay, and a log that records only the code cannot say which of
4192    /// them fired. It is a static, daemon-chosen label per branch, so it is
4193    /// safe to print plainly and stays a closed set.
4194    fn observe_route_open_refusal(
4195        &self,
4196        ctx: &RouteCtx,
4197        caller: &RouteOpenCaller,
4198        module_id: &str,
4199        reason: &'static str,
4200        code: &'static str,
4201    ) {
4202        self.counters.increment_route_open_refused(code);
4203        info!(
4204            target: "control",
4205            code,
4206            reason,
4207            module_id = ?module_id,
4208            connection_id = ctx.connection_id.get(),
4209            principal = %caller.principal,
4210            scope = %caller.scope,
4211            harness = %caller.harness,
4212            "route.open refused"
4213        );
4214        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4215            self.route_outages.record_not_serving(module_id, reason);
4216        }
4217    }
4218
4219    /// Record an ACCEPTED route.open.
4220    ///
4221    /// Refusals have been logged and counted since the attestation work; accepts
4222    /// were invisible, so the daemon knew every principal it stamped and wrote
4223    /// none of them down. The party that attests the identity was the only party
4224    /// not recording it, which left a credential vault unable to name the sender
4225    /// of a call that reached it (claustrum #43) and left the launch-nonce
4226    /// concurrency question unanswerable from the outside.
4227    ///
4228    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4229    /// `code`/`module_id`/`connection_id` returns both directions of the same
4230    /// decision rather than two shapes a reader has to join by hand.
4231    ///
4232    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4233    /// and the difference carries information rather than being an
4234    /// inconsistency. This line is only reachable after a successful bind to a
4235    /// REGISTERED module, so the value has already passed HELLO validation
4236    /// including the path-hazard refusal and cannot contain control bytes. A
4237    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4238    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4239    ///
4240    /// Bare is also what every other daemon line already emits (`module
4241    /// registered`, `configured module supervised`). Shipping `?module_id` here
4242    /// made this instrument the only one in the file whose ids did not answer
4243    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4244    /// line whose whole purpose is being grepped beside its sibling.
4245    ///
4246    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4247    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4248    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4249    /// one are recorded identically. A test written against that harness passes
4250    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4251    /// green assertion that cannot fail. The same limit applies to the escaping
4252    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4253    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4254    /// Fencing either needs the real formatter, not the capture layer.
4255    ///
4256    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4257    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4258    /// to record. The ephemeral port would decay within minutes and answer only a
4259    /// live question. The identity question is instead answered by counting
4260    /// distinct live connections presenting one module's `consumer_identity` --
4261    /// "is anyone else holding this secret" rather than "is this the right
4262    /// process".
4263    fn observe_route_open_accept(
4264        &self,
4265        ctx: &RouteCtx,
4266        module_id: &str,
4267        principal: &str,
4268        scope: &str,
4269    ) {
4270        self.route_outages.record_accepted(module_id);
4271        self.counters.increment_route_open_accepted(principal);
4272        info!(
4273            target: "control",
4274            principal,
4275            module_id,
4276            connection_id = ctx.connection_id.get(),
4277            scope = %scope,
4278            "route.open accepted"
4279        );
4280    }
4281
4282    fn supervised_absent_route_open_refusal_frame(
4283        &self,
4284        ctx: &RouteCtx,
4285        caller: &RouteOpenCaller,
4286        frame: &Frame,
4287        module_id: &str,
4288        code: &'static str,
4289        status: &crate::supervise::ModuleStatus,
4290    ) -> Result<Frame, RouterError> {
4291        self.counters.increment_route_open_refused(code);
4292        info!(
4293            target: "control",
4294            code,
4295            reason = "supervised_not_registered",
4296            module_id = ?module_id,
4297            connection_id = ctx.connection_id.get(),
4298            principal = %caller.principal,
4299            scope = %caller.scope,
4300            harness = %caller.harness,
4301            state = %status.state,
4302            enabled = status.enabled,
4303            live = status.live,
4304            "route.open refused"
4305        );
4306        // A supervised module whose process has not registered is not
4307        // serving, whatever the reason; the supervisor knows this id, so it is
4308        // safe to track.
4309        self.route_outages
4310            .record_not_serving(module_id, "supervised_not_registered");
4311        let message = if let Some(pid) = status.waiting_for_previous_pid {
4312            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4313                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4314            })
4315        } else {
4316            format!(
4317                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4318                status.state, status.enabled, status.live
4319            )
4320        };
4321        control_error_frame(frame, code, message)
4322    }
4323
4324    async fn handle_route_open(
4325        &self,
4326        ctx: &RouteCtx,
4327        frame: Frame,
4328        request: RouteOpenRequest,
4329        timings: &mut RouteOpenTimings,
4330    ) -> Result<Vec<Frame>, RouterError> {
4331        let RouteOpenRequest {
4332            target,
4333            mut identity,
4334            consumer_identity,
4335            consumer_capabilities,
4336            role_versions,
4337            admission_facts,
4338            scope,
4339        } = request;
4340        let target_module_id = target_module_id(&target).to_string();
4341        timings.module_id.clone_from(&target_module_id);
4342        if self
4343            .registry
4344            .get_module_by_connection(ctx.connection_id)
4345            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4346            .is_some()
4347        {
4348            return Ok(vec![control_error_frame(
4349                &frame,
4350                "invalid_request",
4351                "module connections cannot open client routes",
4352            )?]);
4353        }
4354        debug!(
4355            connection_id = ctx.connection_id.get(),
4356            corr = frame.header.corr,
4357            module_id = %target_module_id,
4358            "handling route.open"
4359        );
4360
4361        // The caller's attested principal, requested scope and harness, for
4362        // every `route.open refused` line below. The launch nonce is checked
4363        // here, once, and the answer is reused where the principal is
4364        // enforced further down, so the order of refusals is unchanged.
4365        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4366        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4367
4368        // A malformed declaration is refused first, before anything about the
4369        // target is looked up: the same body would be refused against any
4370        // module, so the caller learns nothing by retrying or waiting. An empty
4371        // map declares nothing and travels as no field at all, so a provider
4372        // only ever sees a missing field or a non-empty one.
4373        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4374        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4375            self.observe_route_open_refusal(
4376                ctx,
4377                &caller,
4378                &target_module_id,
4379                "invalid_role_versions",
4380                error_codes::INVALID_REQUEST,
4381            );
4382            return Ok(vec![control_error_body_frame(
4383                &frame,
4384                ErrorBody {
4385                    code: error_codes::INVALID_REQUEST.to_string(),
4386                    message: error.to_string(),
4387                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4388                },
4389            )?]);
4390        }
4391
4392        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4393            if status.waiting_for_previous_pid.is_some() {
4394                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4395                    ctx,
4396                    &caller,
4397                    &frame,
4398                    &target_module_id,
4399                    "module_warming",
4400                    &status,
4401                )?]);
4402            }
4403        }
4404
4405        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4406        // opposite. Below, a caller learns whether a module is unregistered,
4407        // supervised-but-down (with state/enabled/live), or registered without the
4408        // requested role. Elsewhere that is an enumeration leak: a probe learning
4409        // the shape of a fleet it cannot otherwise see.
4410        //
4411        // It is not one here, and the reason is the ACCESS MODEL rather than
4412        // anything about these errors. Reaching route.open requires the
4413        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4414        // connection file, so any caller who completes it already runs as this
4415        // user -- and can read subc.jsonc for the module list and `ck module
4416        // status` for live state. The reply discloses nothing the caller cannot
4417        // read more easily from disk, while the precision is load-bearing:
4418        // `unknown_module` is retryable and a missing role is not.
4419        //
4420        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4421        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4422        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4423        let Some(registration) = self
4424            .registry
4425            .get_module(&target_module_id)
4426            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4427        else {
4428            if let Some((status, warming)) =
4429                self.supervisor_status(&target_module_id, frame.header.corr)?
4430            {
4431                // BEFORE the two availability codes below, because for a module
4432                // that speaks no subc wire both of them are false comfort: they
4433                // say "not right now" and are retried, and this module will
4434                // never register no matter how long the caller waits. The
4435                // absence here is the declaration being honoured, not a module
4436                // that is late.
4437                if status.protocol == ModuleProtocol::None {
4438                    return Ok(vec![self.route_open_refusal_frame(
4439                        ctx,
4440                        &caller,
4441                        &frame,
4442                        &target_module_id,
4443                        "protocol_none",
4444                        error_codes::MODULE_NO_PROTOCOL,
4445                        format!(
4446                            "module_id '{target_module_id}' is declared protocol: none; \
4447                             it speaks no subc wire and serves no routes"
4448                        ),
4449                    )?]);
4450                }
4451                let code = if warming {
4452                    "module_warming"
4453                } else {
4454                    "target_unavailable"
4455                };
4456                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4457                    ctx,
4458                    &caller,
4459                    &frame,
4460                    &target_module_id,
4461                    code,
4462                    &status,
4463                )?]);
4464            }
4465            if let Some(removed_ago_ms) =
4466                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4467            {
4468                return Ok(vec![self.route_open_refusal_frame(
4469                    ctx,
4470                    &caller,
4471                    &frame,
4472                    &target_module_id,
4473                    "removed",
4474                    error_codes::MODULE_REMOVED,
4475                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4476                )?]);
4477            }
4478            return Ok(vec![self.route_open_refusal_frame(
4479                ctx,
4480                &caller,
4481                &frame,
4482                &target_module_id,
4483                "not_registered",
4484                error_codes::UNKNOWN_MODULE,
4485                format!("module_id '{target_module_id}' is not registered"),
4486            )?]);
4487        };
4488
4489        // Best-effort only: registry readiness and forwarding reservation use
4490        // different locks, so a module can flip readiness between this read and
4491        // the relay. Modules must still tolerate an `on_bind` while not ready.
4492        if !registration.ready {
4493            self.counters
4494                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4495            info!(
4496                target: "control",
4497                code = error_codes::MODULE_WARMING,
4498                module_id = ?target_module_id,
4499                connection_id = ctx.connection_id.get(),
4500                principal = %caller.principal,
4501                scope = %caller.scope,
4502                harness = %caller.harness,
4503                reason = "declared_not_ready",
4504                "route.open refused"
4505            );
4506            // The module is registered but says it cannot take work, which is
4507            // an outage from the caller's side even though its process is up.
4508            self.route_outages
4509                .record_not_serving(&target_module_id, "declared_not_ready");
4510            return Ok(vec![control_error_body_frame(
4511                &frame,
4512                ErrorBody {
4513                    code: error_codes::MODULE_WARMING.to_string(),
4514                    message: format!(
4515                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4516                    ),
4517                    detail: Some(serde_json::json!({
4518                        "reason": "declared_not_ready"
4519                    })),
4520                },
4521            )?]);
4522        }
4523
4524        // Effective readiness, second half: a module that declares a capability
4525        // `need: required` is not routable while that capability has no
4526        // registered provider. It is enforced HERE, as a retryable routing
4527        // refusal, and deliberately not as spawn ordering or a boot block. The
4528        // module is still started and registered and can make its own calls;
4529        // spawn ordering is a promise that cannot be kept once a provider
4530        // crashes at runtime, and refusing to boot would stop the whole
4531        // machine, including the tools needed to fix its configuration.
4532        //
4533        // "Provided" is the evaluator's verdict, which counts a provider as
4534        // soon as it has REGISTERED, not once it is ready. Two modules that
4535        // require each other's capabilities are therefore both routable once
4536        // both register; counting readiness instead would deadlock them.
4537        //
4538        // Only new opens are refused. Routes already bound when a provider
4539        // goes away stay bound: nothing here tears them down, and the module
4540        // answers them as it can. Like the readiness read above this is
4541        // best-effort against a provider registering or leaving concurrently.
4542        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4543            self.counters
4544                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4545            info!(
4546                target: "control",
4547                code = error_codes::MODULE_WARMING,
4548                module_id = ?target_module_id,
4549                connection_id = ctx.connection_id.get(),
4550                principal = %caller.principal,
4551                scope = %caller.scope,
4552                harness = %caller.harness,
4553                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4554                capability = %capability,
4555                "route.open refused"
4556            );
4557            return Ok(vec![control_error_body_frame(
4558                &frame,
4559                ErrorBody {
4560                    code: error_codes::MODULE_WARMING.to_string(),
4561                    message: format!(
4562                        "module_id '{target_module_id}' requires capability '{capability}', \
4563                         which no registered module provides; retry"
4564                    ),
4565                    detail: Some(serde_json::json!({
4566                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4567                        "capability": capability,
4568                    })),
4569                },
4570            )?]);
4571        }
4572
4573        if !target_has_required_role(&target, &registration.manifest.provides) {
4574            return Ok(vec![self.route_open_refusal_frame(
4575                ctx,
4576                &caller,
4577                &frame,
4578                &target_module_id,
4579                "role_not_provided",
4580                "target_unavailable",
4581                format!("module_id '{target_module_id}' does not provide the requested target"),
4582            )?]);
4583        }
4584
4585        if registration.state != ChannelState::Active {
4586            return Ok(vec![self.route_open_refusal_frame(
4587                ctx,
4588                &caller,
4589                &frame,
4590                &target_module_id,
4591                "registration_not_active",
4592                "target_unavailable",
4593                format!("module_id '{target_module_id}' is not active"),
4594            )?]);
4595        }
4596
4597        if self
4598            .forwarding
4599            .module_is_draining(&target_module_id)
4600            .map_err(RouterError::Forwarding)?
4601        {
4602            return Ok(vec![self.route_open_refusal_frame(
4603                ctx,
4604                &caller,
4605                &frame,
4606                &target_module_id,
4607                "reloading",
4608                "module_reloading",
4609                format!("module_id '{target_module_id}' is reloading"),
4610            )?]);
4611        }
4612
4613        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4614            process_liveness.process_live(&target_module_id) == Some(false)
4615        }) {
4616            // A module the supervisor is restarting or reloading can still hold
4617            // a registration: the old process before its connection closes, or
4618            // a new one that registered while the supervisor was draining. The
4619            // forwarding table does not see that as draining, but the consumer
4620            // should still be told to retry soon, exactly as for the drain
4621            // above, rather than that the target is unavailable.
4622            if process_liveness.process_replacing(&target_module_id) {
4623                return Ok(vec![self.route_open_refusal_frame(
4624                    ctx,
4625                    &caller,
4626                    &frame,
4627                    &target_module_id,
4628                    "reloading",
4629                    "module_reloading",
4630                    format!("module_id '{target_module_id}' is reloading"),
4631                )?]);
4632            }
4633            return Ok(vec![self.route_open_refusal_frame(
4634                ctx,
4635                &caller,
4636                &frame,
4637                &target_module_id,
4638                "supervisor_not_live",
4639                "target_unavailable",
4640                format!("module_id '{target_module_id}' is not live"),
4641            )?]);
4642        }
4643
4644        if !self
4645            .forwarding
4646            .has_live_module_connection(&target_module_id)
4647            .map_err(RouterError::Forwarding)?
4648        {
4649            return Ok(vec![self.route_open_refusal_frame(
4650                ctx,
4651                &caller,
4652                &frame,
4653                &target_module_id,
4654                "no_forwarding_connection",
4655                "target_unavailable",
4656                format!("module_id '{target_module_id}' has no live forwarding connection"),
4657            )?]);
4658        }
4659
4660        if let Some(error) =
4661            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4662        {
4663            self.observe_route_open_refusal(
4664                ctx,
4665                &caller,
4666                &target_module_id,
4667                "op_not_allowed",
4668                "op_not_allowed",
4669            );
4670            return Ok(vec![error]);
4671        }
4672
4673        let principal =
4674            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4675                Ok(principal) => principal,
4676                Err(error) => {
4677                    self.observe_route_open_refusal(
4678                        ctx,
4679                        &caller,
4680                        &target_module_id,
4681                        "bad_consumer_identity",
4682                        "bad_consumer_identity",
4683                    );
4684                    return Ok(vec![error]);
4685                }
4686            };
4687        timings.principal = principal_label(&principal);
4688
4689        // This is attested, control-plane policy for supervised module origins.
4690        // Keep it before route reservation and out of the opaque forwarding hot
4691        // path: data frames must never acquire a per-frame capability check.
4692        if let Principal::Reserved {
4693            module_id: opening_module_id,
4694        } = &principal
4695        {
4696            if let Some(opening_registration) = self
4697                .registry
4698                .get_module(opening_module_id)
4699                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4700            {
4701                if let Some(capability) =
4702                    denied_capability(&opening_registration.manifest, &registration.manifest)
4703                {
4704                    warn!(
4705                        opening_module_id,
4706                        target_module_id,
4707                        capability,
4708                        "refusing route.open because an attested capability deny edge matches"
4709                    );
4710                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4711                        &frame,
4712                        &target_module_id,
4713                        "capability_deny_edge",
4714                        "capability_forbidden",
4715                        format!(
4716                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4717                        ),
4718                    )?]);
4719                }
4720            }
4721        }
4722
4723        if admission_facts.is_some() {
4724            let carrier_matches = matches!(
4725                &principal,
4726                Principal::Reserved { module_id }
4727                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4728            );
4729            if !carrier_matches {
4730                return Ok(vec![self.route_open_refusal_frame(
4731                    ctx,
4732                    &caller,
4733                    &frame,
4734                    &target_module_id,
4735                    "admission_facts_carrier_not_permitted",
4736                    "admission_facts_not_permitted",
4737                    "admission facts may only be carried by the configured reserved module",
4738                )?]);
4739            }
4740
4741            let target_allowed = self
4742                .admission_facts_targets
4743                .as_ref()
4744                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4745            if !target_allowed {
4746                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4747                    &frame,
4748                    &target_module_id,
4749                    "admission_facts_target_not_listed",
4750                    "admission_facts_target_not_allowed",
4751                    format!(
4752                        "admission facts are not permitted for target module_id '{target_module_id}'"
4753                    ),
4754                )?]);
4755            }
4756
4757            // Keep the value opaque to subc. The downstream admission validator owns
4758            // schema and semantic checks; this daemon only enforces carrier authority
4759            // and the configured destination allowlist.
4760        }
4761
4762        // Scope admission, on the attested principal above and never on the
4763        // request body. The tag read here travels with the pending bind and is
4764        // compared with the published one at commit, so a sync between here
4765        // and the module's ack refuses the open instead of binding a stamp
4766        // that is no longer true.
4767        let (bound_scope, mut scope_stamp) = match &scope {
4768            None => (None, None),
4769            Some(selector) => {
4770                let owner_configured = match &selector.owner {
4771                    // A reserved owner counts as configured from before its
4772                    // process is spawned (see `SupervisorHandle::is_configured`).
4773                    // So an owner that has not synced its scopes yet is refused
4774                    // as retryable (`scope_not_synced`), not as one that will
4775                    // never sync.
4776                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4777                    _ => false,
4778                };
4779                let mut scopes = self
4780                    .scopes
4781                    .write()
4782                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4783                let admitted = scopes
4784                    .admit(&principal, &target_module_id, selector, owner_configured)
4785                    .and_then(|admission| {
4786                        crate::scopes::check_target_flow_support(
4787                            &admission.stamp,
4788                            &target_module_id,
4789                            registration.manifest.capabilities.as_ref(),
4790                        )?;
4791                        crate::scopes::check_target_agent_run_support(
4792                            &admission.stamp,
4793                            &target_module_id,
4794                            registration.manifest.capabilities.as_ref(),
4795                        )?;
4796                        let bound = scopes.reserve_bind(
4797                            &admission,
4798                            &principal,
4799                            ctx.connection_id,
4800                            selector.hold_presence,
4801                        );
4802                        Ok((admission, bound))
4803                    });
4804                match admitted {
4805                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4806                    Err(refusal) => {
4807                        return Ok(vec![self.route_open_refusal_frame(
4808                            ctx,
4809                            &caller,
4810                            &frame,
4811                            &target_module_id,
4812                            refusal.code,
4813                            refusal.code,
4814                            refusal.message,
4815                        )?]);
4816                    }
4817                }
4818            }
4819        };
4820        // From here on a refusal names the scope by the stamp admission
4821        // issued, exactly as the `route.open accepted` line does.
4822        if let Some(stamp) = &scope_stamp {
4823            caller.scope = route_open_scope_label(stamp);
4824        }
4825
4826        // Bind admits a root that no longer exists on disk, because refusing here
4827        // closes the only exit from a paused run: cancel needs a bound route, and a
4828        // renamed or reclaimed directory makes that route unopenable forever. The
4829        // run itself is intact and still addressable by its recorded identity.
4830        //
4831        // This does NOT relax the rule the strict constructor protects. That rule is
4832        // that no root is ever aliased into NEW durable state -- a missing component
4833        // can reappear as a symlink elsewhere, which would move the identity and
4834        // split a session's history across two of them. The engine now refuses the
4835        // two operations that create such state (send and import) at admission,
4836        // which is a narrower way to hold the same invariant: reads and terminations
4837        // are admitted, writes are not. That refusal had to ship before this line
4838        // changed, or there is an interval where a send commits under a provisional
4839        // identity -- the exact failure the original policy existed to prevent.
4840        //
4841        // Resolution follows realpath rather than lexical cleanup: the longest
4842        // existing ancestor is canonicalized and the missing tail re-appended, so a
4843        // live root is unchanged and a vanished leaf keeps the identity it was
4844        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4845        // same caller the moment the directory vanished, which strands the run more
4846        // quietly than refusing it.
4847        timings.enter(RouteOpenPhase::ProjectRoot);
4848        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4849            Ok(project_root) => project_root,
4850            Err(err) => {
4851                return Ok(vec![control_error_frame(
4852                    &frame,
4853                    "invalid_project_root",
4854                    err.to_string(),
4855                )?])
4856            }
4857        };
4858        timings.enter(RouteOpenPhase::Admission);
4859        identity.project_root = project_root.as_path().to_path_buf();
4860
4861        // Last gate before any relay work, and deliberately after the cheap
4862        // registry and availability checks above: those name a more precise
4863        // condition (unknown, removed, reloading) and a caller is better served
4864        // by the precise code than by this one.
4865        //
4866        // Everything below this point costs an egress permit, a reserved handle
4867        // pair and, if the module does not answer, the whole relay budget. The
4868        // reader no longer waits for that budget, so cap each target explicitly;
4869        // serial dispatch used to provide the accidental cap of one relay per
4870        // connection. Admission is a mutex-protected count and never waits.
4871        let _concurrency_guard = match self
4872            .route_bind_concurrency
4873            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4874        {
4875            Ok(guard) => guard,
4876            Err(in_flight) => {
4877                return Ok(vec![self.route_open_target_capacity_refusal(
4878                    ctx,
4879                    &caller,
4880                    &frame,
4881                    &target_module_id,
4882                    in_flight,
4883                )?]);
4884            }
4885        };
4886
4887        // A module that has already burned the whole budget `threshold` times
4888        // in a row does not get to charge it again until a probe says it recovered.
4889        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4890            RouteBindAdmission::Admitted { guard, probe } => {
4891                if probe {
4892                    info!(
4893                        module_id = %target_module_id,
4894                        connection_id = ctx.connection_id.get(),
4895                        "route.bind breaker half-open: admitting one probe"
4896                    );
4897                }
4898                guard
4899            }
4900            RouteBindAdmission::Refused {
4901                consecutive_timeouts,
4902                retry_in,
4903                probe_in_flight,
4904            } => {
4905                return Ok(vec![self.route_open_breaker_refusal_frame(
4906                    ctx,
4907                    &caller,
4908                    &frame,
4909                    &target_module_id,
4910                    consecutive_timeouts,
4911                    retry_in,
4912                    probe_in_flight,
4913                )?]);
4914            }
4915        };
4916
4917        // Resolve the per-module budget here so the wait matches the operator's
4918        // intent for this specific target. A per-module override in
4919        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4920        // daemons) wins over the daemon-wide default.
4921        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4922        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4923        timings.enter(RouteOpenPhase::EgressReservation);
4924        let pending = match self
4925            .forwarding
4926            .begin_route_bind_relay_for(
4927                ctx.connection_id,
4928                ctx.egress.clone(),
4929                response_version(&frame),
4930                frame.header.corr,
4931                &target_module_id,
4932                principal.clone(),
4933                bound_scope,
4934                Some(project_root),
4935                relay_deadline,
4936            )
4937            .await
4938        {
4939            Ok(pending) => pending,
4940            Err(err) => {
4941                return Ok(vec![self.route_open_refusal_frame(
4942                    ctx,
4943                    &caller,
4944                    &frame,
4945                    &target_module_id,
4946                    "relay_reservation_failed",
4947                    forwarding_error_code(&err),
4948                    err.to_string(),
4949                )?])
4950            }
4951        };
4952        timings.enter(RouteOpenPhase::Admission);
4953        let crate::forwarding::PendingRouteBindRelay {
4954            endpoint,
4955            module_sink,
4956            negotiated_ver,
4957            client_channel,
4958            client_epoch,
4959            module_channel,
4960            module_epoch,
4961            corr: relay_corr,
4962            receiver,
4963        } = pending;
4964        let mut reservation =
4965            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4966
4967        // Reserving egress can wait while a module reconnects or a swap cuts
4968        // over. Check the relay's captured connection, not the earlier by-id
4969        // lookup: the original module's flow/run capabilities cannot authorize
4970        // its replacement. The captured sink stays bound to that connection.
4971        let relay_registration = self
4972            .registry
4973            .get_module_by_connection(endpoint.connection_id)
4974            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4975        let relay_capabilities = relay_registration
4976            .as_ref()
4977            .and_then(|registration| registration.manifest.capabilities.as_ref());
4978        if let Some(selector) = &scope {
4979            // Egress reservation may wait across a scope update. Recheck the
4980            // proof against the current record before sending the captured bind.
4981            if let Err(refusal) = self
4982                .scopes
4983                .read()
4984                .unwrap_or_else(|poisoned| poisoned.into_inner())
4985                .check_bind_proof(&principal, selector)
4986            {
4987                reservation.release_and_disarm();
4988                return Ok(vec![self.route_open_refusal_frame(
4989                    ctx,
4990                    &caller,
4991                    &frame,
4992                    &target_module_id,
4993                    refusal.code,
4994                    refusal.code,
4995                    refusal.message,
4996                )?]);
4997            }
4998        }
4999        if let Some(stamp) = scope_stamp
5000            .as_ref()
5001            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5002        {
5003            if let Err(refusal) = crate::scopes::check_target_flow_support(
5004                stamp,
5005                &target_module_id,
5006                relay_capabilities,
5007            )
5008            .and_then(|()| {
5009                crate::scopes::check_target_agent_run_support(
5010                    stamp,
5011                    &target_module_id,
5012                    relay_capabilities,
5013                )
5014            }) {
5015                reservation.release_and_disarm();
5016                return Ok(vec![self.route_open_refusal_frame(
5017                    ctx,
5018                    &caller,
5019                    &frame,
5020                    &target_module_id,
5021                    refusal.code,
5022                    refusal.code,
5023                    refusal.message,
5024                )?]);
5025            }
5026        }
5027
5028        // Older readers may reject unknown fields. Select optional fields from
5029        // the captured endpoint's declaration, including on unscoped binds.
5030        if !relay_capabilities.is_some_and(|caps| {
5031            caps.provides
5032                .iter()
5033                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5034        }) {
5035            if let Some(stamp) = scope_stamp.as_mut() {
5036                stamp.workdir = None;
5037                stamp.project_id = None;
5038            }
5039        }
5040        if !relay_capabilities.is_some_and(|caps| {
5041            caps.provides
5042                .iter()
5043                .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5044        }) {
5045            if let Some(stamp) = scope_stamp.as_mut() {
5046                stamp.attributes.on_behalf_of_agent = None;
5047            }
5048        }
5049        let call_mode = if relay_capabilities
5050            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5051        {
5052            Some(derive_call_mode(scope_stamp.as_ref()))
5053        } else {
5054            if let Some(stamp) = scope_stamp.as_mut() {
5055                stamp.expires_at_ms = None;
5056            }
5057            None
5058        };
5059
5060        debug!(
5061            connection_id = ctx.connection_id.get(),
5062            client_channel,
5063            client_epoch,
5064            module_channel,
5065            module_epoch,
5066            "reserved route handle pair"
5067        );
5068        // Rendered BEFORE the move into the relay, because the accept arm below
5069        // is where it is logged and the principal is gone by then.
5070        let route_principal_label = principal_label(&principal);
5071        let relay = ModuleControlRequest::RouteBind {
5072            route_channel: module_channel,
5073            epoch: module_epoch,
5074            target,
5075            identity,
5076            principal: Some(principal),
5077            consumer_capabilities,
5078            role_versions,
5079            admission_facts,
5080            scope: scope_stamp,
5081            call_mode,
5082        };
5083        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5084            RouterError::backend(
5085                0,
5086                frame.header.corr,
5087                format!("failed to encode route.bind request: {err}"),
5088            )
5089        })?;
5090        let relay_frame = Frame::build_with_version(
5091            negotiated_ver,
5092            FrameType::Request,
5093            control_flags(),
5094            0,
5095            0,
5096            relay_corr,
5097            relay_body,
5098        )
5099        .map_err(RouterError::FrameBuild)?;
5100
5101        timings.enter(RouteOpenPhase::BindSend);
5102        let sent = module_sink.send(relay_frame).await;
5103        timings.enter(RouteOpenPhase::Admission);
5104        if let Err(err) = sent {
5105            reservation.release_and_disarm();
5106            return Ok(vec![self.route_open_refusal_frame(
5107                ctx,
5108                &caller,
5109                &frame,
5110                &target_module_id,
5111                "relay_send_failed",
5112                "target_unavailable",
5113                err.to_string(),
5114            )?]);
5115        }
5116
5117        #[cfg(test)]
5118        if let Some(hold) = &self.route_bind_relay_sent_hold {
5119            // First wait: tell the test the bind is queued. Second wait: stay
5120            // here until the test has driven whatever it wants to happen
5121            // between queueing the bind and marking it enqueued.
5122            hold.wait().await;
5123            hold.wait().await;
5124        }
5125
5126        // A missing relay is not by itself an abandoned bind: the module may
5127        // already have acked it and the route may be committed. Only a relay
5128        // dropped without any module reply needs this GOODBYE; one a reply
5129        // settled has already had its GOODBYE decided by the reply path.
5130        if self
5131            .forwarding
5132            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5133            .map_err(RouterError::Forwarding)?
5134            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5135        {
5136            self.send_abandoned_route_bind_goodbye(
5137                &module_sink,
5138                negotiated_ver,
5139                module_channel,
5140                module_epoch,
5141            );
5142        }
5143
5144        timings.enter(RouteOpenPhase::BindAckWait);
5145        let outcome = timeout_at(relay_deadline, receiver).await;
5146        timings.enter(RouteOpenPhase::Admission);
5147        match outcome {
5148            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5149                reservation.disarm();
5150                if breaker.record_accepted() {
5151                    info!(
5152                        module_id = %target_module_id,
5153                        "route.bind breaker closed: the probe was accepted"
5154                    );
5155                }
5156                self.observe_route_open_accept(
5157                    ctx,
5158                    &target_module_id,
5159                    &route_principal_label,
5160                    &caller.scope,
5161                );
5162                Ok(Vec::new())
5163            }
5164            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5165                reservation.release_and_disarm();
5166                // A module that says no in microseconds is healthy. Rejection
5167                // is a different condition with its own refusal and must not
5168                // move the breaker.
5169                breaker.record_inconclusive();
5170                // The daemon's own commit re-check refused the bind because the
5171                // scope ended or changed after admission. The module accepted;
5172                // counting it as a module rejection would blame the module.
5173                let scope_code = match body.code.as_str() {
5174                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5175                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5176                    _ => None,
5177                };
5178                if let Some(code) = scope_code {
5179                    self.observe_route_open_refusal(
5180                        ctx,
5181                        &caller,
5182                        &target_module_id,
5183                        "scope_changed_before_commit",
5184                        code,
5185                    );
5186                    return Ok(vec![control_error_body_frame(&frame, body)?]);
5187                }
5188                self.counters
5189                    .increment_route_open_refused("module_rejected");
5190                info!(
5191                    target: "control",
5192                    code = "module_rejected",
5193                    module_code = ?body.code,
5194                    module_id = ?target_module_id,
5195                    connection_id = ctx.connection_id.get(),
5196                    principal = %caller.principal,
5197                    scope = %caller.scope,
5198                    harness = %caller.harness,
5199                    "route.open refused"
5200                );
5201                Ok(vec![control_error_body_frame(&frame, body)?])
5202            }
5203            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5204                reservation.release_and_disarm();
5205                breaker.record_inconclusive();
5206                // Fires when the module's connection closes while a relayed
5207                // bind is pending -- typically a caller racing a module restart
5208                // whose bind was relayed BEFORE the drain mark went up. Logged
5209                // because the caller sees only its own error and the fleet has
5210                // already spent one diagnosis round unable to tell this arm
5211                // from a relay timeout without daemon-side evidence.
5212                tracing::warn!(
5213                    module_id = %target_module_id,
5214                    "route.bind relay abandoned: {message}"
5215                );
5216                Ok(vec![self.route_open_refusal_frame(
5217                    ctx,
5218                    &caller,
5219                    &frame,
5220                    &target_module_id,
5221                    "relay_abandoned",
5222                    "target_unavailable",
5223                    message,
5224                )?])
5225            }
5226            Ok(Err(_)) => {
5227                reservation.release_and_disarm();
5228                breaker.record_inconclusive();
5229                Ok(vec![self.route_open_refusal_frame(
5230                    ctx,
5231                    &caller,
5232                    &frame,
5233                    &target_module_id,
5234                    "relay_waiter_canceled",
5235                    "target_unavailable",
5236                    "route.bind relay waiter was canceled before the module responded",
5237                )?])
5238            }
5239            Err(_) => {
5240                reservation.release_and_disarm();
5241                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5242                // answer at all is the one condition a fast refusal can
5243                // usefully stand in for; every other arm already answered.
5244                if let Some(opened) = breaker.record_timeout(
5245                    self.route_bind_breaker_threshold,
5246                    self.route_bind_breaker_cooldown,
5247                ) {
5248                    warn!(
5249                        module_id = %target_module_id,
5250                        consecutive_timeouts = opened.consecutive_timeouts,
5251                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5252                        reopened_after_probe = opened.reopened_after_probe,
5253                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5254                    );
5255                }
5256                // The generous budget just burned to no answer: the module is
5257                // registered and its connection is up, but its bind handler sat
5258                // on the ack for the full budget (warm-on-bind, cold configure,
5259                // or a wedged handler). Every earlier unavailability shape
5260                // fast-refuses BEFORE the relay, so this arm firing means the
5261                // slowness is module-side -- log it so the per-module timeline
5262                // is reconstructable without client audit rows.
5263                tracing::warn!(
5264                    module_id = %target_module_id,
5265                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5266                    "route.bind relay timed out: module did not ack within budget"
5267                );
5268                Ok(vec![self.route_open_refusal_frame(
5269                    ctx,
5270                    &caller,
5271                    &frame,
5272                    &target_module_id,
5273                    "relay_timed_out",
5274                    "module_timeout",
5275                    format!(
5276                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5277                        route_bind_relay_timeout
5278                    ),
5279                )?])
5280            }
5281        }
5282    }
5283
5284    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5285        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5286            snapshot: self.supervisor.spawn_snapshot(),
5287        };
5288        Ok(vec![control_response_body_frame(
5289            &frame,
5290            &response,
5291            "ClientControlResponse::SupervisorSpawnSnapshot",
5292        )?])
5293    }
5294
5295    fn handle_supervisor_spawn_subscribe(
5296        &self,
5297        ctx: &RouteCtx,
5298        frame: Frame,
5299        since: Option<SpawnCursor>,
5300    ) -> Result<Vec<Frame>, RouterError> {
5301        match self.supervisor.subscribe_spawns(
5302            ctx.connection_id,
5303            frame.header.corr,
5304            response_version(&frame),
5305            since,
5306            ctx.egress.clone(),
5307        ) {
5308            Ok(()) => Ok(Vec::new()),
5309            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5310                Ok(vec![control_error_body_frame(
5311                    &frame,
5312                    ErrorBody {
5313                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5314                        message: "spawn cursor belongs to a different daemon incarnation"
5315                            .to_string(),
5316                        detail: Some(serde_json::json!({
5317                            "current_daemon_incarnation": current
5318                        })),
5319                    },
5320                )?])
5321            }
5322            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5323                &frame,
5324                ErrorBody {
5325                    code: "spawn_cursor_too_old".to_string(),
5326                    message: "spawn cursor predates the retained event ring".to_string(),
5327                    detail: Some(serde_json::json!({
5328                        "oldest_retained_cursor": oldest
5329                    })),
5330                },
5331            )?]),
5332            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5333                0,
5334                frame.header.corr,
5335                format!("failed to open supervisor spawn subscription: {error}"),
5336            )),
5337        }
5338    }
5339
5340    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5341        let generation = self
5342            .registry
5343            .generation()
5344            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5345        let mut modules = Vec::new();
5346        for module in self.supervisor.list() {
5347            let status = module.status_for_control("list").map_err(|err| {
5348                RouterError::backend(
5349                    0,
5350                    frame.header.corr,
5351                    format!("failed to read supervisor status: {err}"),
5352                )
5353            })?;
5354            let (configured, _) = module.configuration().map_err(|err| {
5355                RouterError::backend(
5356                    0,
5357                    frame.header.corr,
5358                    format!("failed to read module configuration: {err}"),
5359                )
5360            })?;
5361            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5362                RouterError::backend(
5363                    0,
5364                    frame.header.corr,
5365                    format!("failed to read retry time: {err}"),
5366                )
5367            })?;
5368            // Status and configuration snapshots release their locks before the image probe awaits.
5369            let image = module.running_image_agreement().await;
5370            // Read per request so the figure is current when the operator asks;
5371            // the daemon samples nothing in between.
5372            let resources = Some(module.child_resource_usage());
5373            let mut verdict =
5374                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5375            verdict.spec = Some(launch_spec_agreement(
5376                &configured,
5377                status.process_alive,
5378                status.launch_spec_digest.as_ref(),
5379            ));
5380            let pending_reload = Some(verdict);
5381            let configured_enabled = self
5382                .capability_evaluator
5383                .configured_enabled(&status.module_id);
5384            // Keep the retired policy field on the wire for one release so
5385            // existing status consumers still receive the platform policy.
5386            modules.push(
5387                SupervisorEntry::new(
5388                    status.module_id,
5389                    status.state.to_string(),
5390                    status.enabled,
5391                    status.live,
5392                    status.health.status,
5393                )
5394                .with_configured_enabled(configured_enabled)
5395                .with_next_retry_at_ms(next_retry_at_ms)
5396                .with_launch_nonce_env(Some(!cfg!(unix)))
5397                .with_launch_nonce_source(status.launch_nonce_source)
5398                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5399                .with_protocol(status.protocol)
5400                .with_pending_reload(pending_reload)
5401                .with_last_probe_ms(status.health.last_probe_ms)
5402                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5403                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5404                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5405                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5406                .with_restart_count(Some(status.restart_count))
5407                .with_max_restarts(Some(status.max_restarts))
5408                .with_lifetime_restarts(Some(status.lifetime_restarts))
5409                .with_spawn_generation(Some(status.spawn_generation))
5410                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5411                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5412                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5413                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5414                .with_resources(resources),
5415            );
5416        }
5417        let response = ClientControlResponse::SupervisorList {
5418            generation,
5419            modules,
5420        };
5421        Ok(vec![control_response_body_frame(
5422            &frame,
5423            &response,
5424            "ClientControlResponse::SupervisorList",
5425        )?])
5426    }
5427
5428    fn handle_supervisor_stderr_tail(
5429        &self,
5430        frame: Frame,
5431        module_id: String,
5432        max_lines: Option<u32>,
5433        max_bytes: Option<u32>,
5434    ) -> Result<Vec<Frame>, RouterError> {
5435        let Some(module) = self.supervisor.get(&module_id) else {
5436            return Ok(vec![control_error_frame(
5437                &frame,
5438                "unknown_module",
5439                format!("module_id '{module_id}' is not supervised"),
5440            )?]);
5441        };
5442
5443        let snapshot = module.stderr_tail(
5444            max_lines.map(|value| value as usize),
5445            max_bytes.map(|value| value as usize),
5446        );
5447
5448        let response = ClientControlResponse::SupervisorStderrTail {
5449            module_id,
5450            tail: StderrTail {
5451                capture: match snapshot.capture {
5452                    CaptureState::Captured => StderrCaptureState::Captured,
5453                    CaptureState::Incomplete { reason } => {
5454                        StderrCaptureState::Incomplete { reason }
5455                    }
5456                    CaptureState::NotCaptured { reason } => {
5457                        StderrCaptureState::NotCaptured { reason }
5458                    }
5459                },
5460                entries: snapshot
5461                    .entries
5462                    .into_iter()
5463                    .map(|entry| match entry {
5464                        TailEntry::Line {
5465                            text,
5466                            truncated,
5467                            at_ms,
5468                        } => StderrTailEntry::Line {
5469                            text,
5470                            truncated,
5471                            at_ms,
5472                        },
5473                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5474                    })
5475                    .collect(),
5476                dropped_lines: snapshot.dropped_lines,
5477            },
5478        };
5479        Ok(vec![control_response_body_frame(
5480            &frame,
5481            &response,
5482            "ClientControlResponse::SupervisorStderrTail",
5483        )?])
5484    }
5485
5486    async fn handle_supervisor_terminals(
5487        &self,
5488        frame: Frame,
5489        module_id: String,
5490    ) -> Result<Vec<Frame>, RouterError> {
5491        let Some(module) = self.supervisor.get(&module_id) else {
5492            return Ok(vec![control_error_frame(
5493                &frame,
5494                "unknown_module",
5495                format!("module_id '{module_id}' is not supervised"),
5496            )?]);
5497        };
5498
5499        // The journal read runs on a blocking thread: it can be megabytes of
5500        // file I/O and must not occupy a runtime worker.
5501        let terminals = module
5502            .read_durable_terminal_history()
5503            .await
5504            .map_err(|error| {
5505                RouterError::backend(
5506                    0,
5507                    frame.header.corr,
5508                    format!("failed to read terminal history: {error}"),
5509                )
5510            })?;
5511        let response = ClientControlResponse::SupervisorTerminals {
5512            module_id,
5513            terminals,
5514        };
5515        Ok(vec![control_response_body_frame(
5516            &frame,
5517            &response,
5518            "ClientControlResponse::SupervisorTerminals",
5519        )?])
5520    }
5521
5522    fn handle_supervisor_routes(
5523        &self,
5524        frame: Frame,
5525        module_id: Option<String>,
5526    ) -> Result<Vec<Frame>, RouterError> {
5527        let modules = self
5528            .forwarding
5529            .route_census(module_id.as_deref())
5530            .map_err(RouterError::Forwarding)?
5531            .into_iter()
5532            .map(|(module_id, routes)| SupervisorRouteModule {
5533                module_id,
5534                routes: routes
5535                    .into_iter()
5536                    .map(|route| SupervisorRoute {
5537                        consumer: match route.principal {
5538                            Principal::Reserved { module_id } => {
5539                                SupervisorRouteConsumer::Reserved { module_id }
5540                            }
5541                            Principal::Direct | Principal::Unverified => {
5542                                SupervisorRouteConsumer::Direct {
5543                                    connection_id: route.goodbye_target.connection_id.get(),
5544                                }
5545                            }
5546                        },
5547                        age_ms: Instant::now()
5548                            .saturating_duration_since(route.bound_at)
5549                            .as_millis()
5550                            .try_into()
5551                            .unwrap_or(u64::MAX),
5552                        draining: route.draining,
5553                        drain_reason: route.drain_reason,
5554                    })
5555                    .collect(),
5556            })
5557            .collect();
5558        let response = ClientControlResponse::SupervisorRoutes { modules };
5559        Ok(vec![control_response_body_frame(
5560            &frame,
5561            &response,
5562            "ClientControlResponse::SupervisorRoutes",
5563        )?])
5564    }
5565
5566    async fn handle_supervisor_provenance(
5567        &self,
5568        frame: Frame,
5569        module_id: Option<String>,
5570    ) -> Result<Vec<Frame>, RouterError> {
5571        let mut selected = if let Some(module_id) = module_id {
5572            let Some(module) = self.supervisor.get(&module_id) else {
5573                return Ok(vec![control_error_frame(
5574                    &frame,
5575                    "unknown_module",
5576                    format!("module_id '{module_id}' is not supervised"),
5577                )?]);
5578            };
5579            vec![module]
5580        } else {
5581            self.supervisor.list()
5582        };
5583
5584        let mut modules = Vec::with_capacity(selected.len());
5585        for module in selected.drain(..) {
5586            let (status, observed_image) = module
5587                .status_and_running_image_agreement()
5588                .await
5589                .map_err(|err| {
5590                    RouterError::backend(
5591                        0,
5592                        frame.header.corr,
5593                        format!("failed to read supervisor status: {err}"),
5594                    )
5595                })?;
5596            let module_declared = self
5597                .registry
5598                .get_module(&status.module_id)
5599                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5600                .and_then(|registration| registration.manifest.provenance)
5601                .map(|build| ModuleDeclaredProvenance::Reported { build })
5602                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5603            #[cfg(test)]
5604            let running_image = match &self.provenance_probe_override {
5605                Some(result) => result.clone(),
5606                None => observed_image,
5607            };
5608            #[cfg(not(test))]
5609            let running_image = observed_image;
5610            modules.push(SupervisorModuleProvenance {
5611                module_id: status.module_id,
5612                module_declared,
5613                daemon_observed: SupervisorObservedProcess {
5614                    launch_nonce_source: status.launch_nonce_source,
5615                    pid: status.pid,
5616                    spawned_at_ms: status.spawned_at_ms,
5617                    spawned_from: status.spawned_from,
5618                    running_image,
5619                },
5620            });
5621        }
5622        let daemon = SupervisorDaemonProvenance {
5623            daemon_build: self.daemon_provenance.build.clone(),
5624            daemon_observed: DaemonObservedProcess {
5625                pid: self.daemon_provenance.pid,
5626                started_at_ms: self
5627                    .daemon_provenance
5628                    .start_clock
5629                    .map(|clock| clock.started_at_ms())
5630                    .or(self.daemon_provenance.started_at_ms),
5631                running_image: self
5632                    .daemon_provenance
5633                    .probe
5634                    .observe(
5635                        self.daemon_provenance.pid,
5636                        self.daemon_provenance.executable_path.as_deref(),
5637                        self.daemon_provenance.executable_identity.clone(),
5638                        self.daemon_provenance.process_start_time,
5639                    )
5640                    .await,
5641            },
5642        };
5643        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5644        Ok(vec![control_response_body_frame(
5645            &frame,
5646            &response,
5647            "ClientControlResponse::SupervisorProvenance",
5648        )?])
5649    }
5650
5651    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5652        self.refresh_capability_requirements();
5653        let generation = self
5654            .registry
5655            .generation()
5656            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5657        let modules = self
5658            .supervisor
5659            .list()
5660            .into_iter()
5661            .map(|module| {
5662                let status = module.status_for_control("health").map_err(|err| {
5663                    RouterError::backend(
5664                        0,
5665                        frame.header.corr,
5666                        format!("failed to read supervisor health: {err}"),
5667                    )
5668                })?;
5669                let module_id = status.module_id;
5670                let capability_detail = self
5671                    .capability_evaluator
5672                    .required_problem_detail(&module_id);
5673                Ok(SupervisorHealthEntry {
5674                    module_id,
5675                    status: status.health.status,
5676                    detail: append_capability_problem_detail(
5677                        status.health.detail,
5678                        capability_detail,
5679                    ),
5680                    metrics: status.health.metrics,
5681                    consecutive_failures: status.health.consecutive_failures,
5682                    late_answer_count: status.health.late_answer_count,
5683                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5684                    last_action: status.health.last_action,
5685                    last_action_ms: status.health.last_action_ms,
5686                    last_probe_ms: status.health.last_probe_ms,
5687                })
5688            })
5689            .collect::<Result<Vec<_>, RouterError>>()?;
5690        let response = ClientControlResponse::SupervisorHealth {
5691            generation,
5692            modules,
5693        };
5694        Ok(vec![control_response_body_frame(
5695            &frame,
5696            &response,
5697            "ClientControlResponse::SupervisorHealth",
5698        )?])
5699    }
5700
5701    async fn handle_supervisor_restart(
5702        &self,
5703        frame: Frame,
5704        module_id: String,
5705        drain_timeout_ms: Option<u64>,
5706        requester: Principal,
5707    ) -> Result<Vec<Frame>, RouterError> {
5708        let operation_lock = self.supervisor.operation_lock();
5709        let _operation_guard = operation_lock.lock().await;
5710        let Some(module) = self.supervisor.get(&module_id) else {
5711            return Ok(vec![control_error_frame(
5712                &frame,
5713                "unknown_module",
5714                format!("module_id '{module_id}' is not supervised"),
5715            )?]);
5716        };
5717
5718        self.route_outages.mark_operator_action(&module_id);
5719        if let Err(err) = module
5720            .restart_requested(drain_timeout_ms, Some(requester))
5721            .await
5722        {
5723            self.route_outages
5724                .operator_action_ended_unrefused(&module_id);
5725            let (code, message) = match err {
5726                crate::supervise::SuperviseError::Disabled { .. } => {
5727                    ("module_disabled", err.to_string())
5728                }
5729                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5730                    ("swap_in_progress", err.to_string())
5731                }
5732                _ => (
5733                    "target_unavailable",
5734                    format!("failed to restart module_id '{module_id}': {err}"),
5735                ),
5736            };
5737            return Ok(vec![control_error_frame(&frame, code, message)?]);
5738        }
5739
5740        let response = ClientControlResponse::SupervisorAck {
5741            module_id,
5742            applied: true,
5743        };
5744        Ok(vec![control_response_body_frame(
5745            &frame,
5746            &response,
5747            "ClientControlResponse::SupervisorAck",
5748        )?])
5749    }
5750
5751    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5752    /// when the old process has finished draining: a caller whose own lane
5753    /// rides the old process must get its reply before that drain waits on it.
5754    async fn handle_supervisor_swap(
5755        &self,
5756        frame: Frame,
5757        module_id: String,
5758        ready_timeout_ms: Option<u64>,
5759        requester: Principal,
5760    ) -> Result<Vec<Frame>, RouterError> {
5761        // The daemon-wide operation lock is held only to resolve the handle,
5762        // not across the swap. The swap can take its whole readiness budget,
5763        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5764        // holding it here would park an operator's stop behind the swap it is
5765        // meant to abort. A rescan or stop that reaches the module during the
5766        // swap is served by the swap itself (see `supervise_swap`).
5767        let module = {
5768            let operation_lock = self.supervisor.operation_lock();
5769            let _operation_guard = operation_lock.lock().await;
5770            self.supervisor.get(&module_id)
5771        };
5772        let Some(module) = module else {
5773            return Ok(vec![control_error_frame(
5774                &frame,
5775                "unknown_module",
5776                format!("module_id '{module_id}' is not supervised"),
5777            )?]);
5778        };
5779
5780        self.route_outages.mark_operator_action(&module_id);
5781        if let Err(err) = module
5782            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5783            .await
5784        {
5785            self.route_outages
5786                .operator_action_ended_unrefused(&module_id);
5787            use crate::supervise::SuperviseError;
5788            let message = err.to_string();
5789            let error = match err {
5790                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5791                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5792                    code: "swap_refused".to_string(),
5793                    message,
5794                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5795                },
5796                SuperviseError::SwapFailed {
5797                    arm,
5798                    candidate_exit,
5799                    ..
5800                } => ErrorBody {
5801                    code: "swap_failed".to_string(),
5802                    message,
5803                    detail: Some(serde_json::json!({
5804                        "arm": arm.as_str(),
5805                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5806                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5807                    })),
5808                },
5809                _ => ErrorBody::new(
5810                    "target_unavailable",
5811                    format!("failed to swap module_id '{module_id}': {message}"),
5812                ),
5813            };
5814            return Ok(vec![control_error_body_frame(&frame, error)?]);
5815        }
5816        // A completed swap kept the incumbent serving until cutover, so it
5817        // usually opened no outage; a mark left behind would make the next,
5818        // unrelated outage read as requested.
5819        self.route_outages
5820            .operator_action_ended_unrefused(&module_id);
5821
5822        let response = ClientControlResponse::SupervisorAck {
5823            module_id,
5824            applied: true,
5825        };
5826        Ok(vec![control_response_body_frame(
5827            &frame,
5828            &response,
5829            "ClientControlResponse::SupervisorAck",
5830        )?])
5831    }
5832
5833    async fn handle_supervisor_reload(
5834        &self,
5835        frame: Frame,
5836        module_id: String,
5837        requester: Principal,
5838    ) -> Result<Vec<Frame>, RouterError> {
5839        let operation_lock = self.supervisor.operation_lock();
5840        let _operation_guard = operation_lock.lock().await;
5841        let Some(module) = self.supervisor.get(&module_id) else {
5842            return Ok(vec![control_error_frame(
5843                &frame,
5844                "unknown_module",
5845                format!("module_id '{module_id}' is not supervised"),
5846            )?]);
5847        };
5848
5849        self.route_outages.mark_operator_action(&module_id);
5850        if let Err(err) = module.reload_requested(Some(requester)).await {
5851            self.route_outages
5852                .operator_action_ended_unrefused(&module_id);
5853            let (code, message) = match err {
5854                crate::supervise::SuperviseError::Disabled { .. } => {
5855                    ("module_disabled", err.to_string())
5856                }
5857                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5858                    ("swap_in_progress", err.to_string())
5859                }
5860                _ => (
5861                    "reload_failed",
5862                    format!("failed to reload module_id '{module_id}': {err}"),
5863                ),
5864            };
5865            return Ok(vec![control_error_frame(&frame, code, message)?]);
5866        }
5867
5868        let response = ClientControlResponse::SupervisorAck {
5869            module_id,
5870            applied: true,
5871        };
5872        Ok(vec![control_response_body_frame(
5873            &frame,
5874            &response,
5875            "ClientControlResponse::SupervisorAck",
5876        )?])
5877    }
5878
5879    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5880    /// the configuration comes from and whether to apply it; every mode then
5881    /// runs the same comparison, so a preview describes the apply it previews.
5882    async fn handle_supervisor_rescan(
5883        &self,
5884        frame: Frame,
5885        mode: RescanMode,
5886        requester: Principal,
5887    ) -> Result<Vec<Frame>, RouterError> {
5888        let preview = mode.previews();
5889        let Some(context) = self.rescan.clone() else {
5890            return Ok(vec![control_error_frame(
5891                &frame,
5892                "rescan_unavailable",
5893                "the daemon was not started with a reloadable config path".to_string(),
5894            )?]);
5895        };
5896
5897        let operation_lock = self.supervisor.operation_lock();
5898        let _operation_guard = operation_lock.lock().await;
5899        // Loaded under the operation lock, so no other rescan can change the
5900        // staged configuration between a checked apply's comparison and the
5901        // apply itself.
5902        let loaded = match self.load_rescan_config(&mode, &context) {
5903            Ok(config) => config,
5904            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5905        };
5906        // `load` reports a missing file as Ok(None), which is correct at boot
5907        // (no config, nothing to supervise) and catastrophic here: rescan treats
5908        // "not in the config" as "remove it", so an absent file would read as an
5909        // empty module list and retire the entire running fleet. An editor
5910        // writing via write-new-then-rename, or a half-finished edit, is enough
5911        // to open that window. Refuse instead: a config that cannot be read
5912        // carries no instruction to remove anything.
5913        let Some((config, activity_config)) = loaded else {
5914            return Ok(vec![control_error_frame(
5915                &frame,
5916                "invalid_daemon_config",
5917                format!(
5918                    "daemon config not found at {}; refusing to rescan (an absent config would \
5919                     retire every supervised module)",
5920                    context.config_path.display()
5921                ),
5922            )?]);
5923        };
5924        let (
5925            configured_port,
5926            storage_config,
5927            admission_facts_carrier_module_id,
5928            admission_facts_targets,
5929            scope_authority_owners,
5930            privileged_operator_prompt,
5931            modules,
5932            reserved_capabilities,
5933        ) = (
5934            config.port,
5935            config.storage,
5936            config.admission_facts_carrier_module_id,
5937            config.admission_facts_targets,
5938            config.scope_authority_owners,
5939            config.privileged_operator_prompt,
5940            config.modules,
5941            config.reserved_capabilities,
5942        );
5943
5944        // Collect the sections rescan cannot apply, so the REPLY carries them.
5945        //
5946        // The warning below has always been correct and has always gone only to
5947        // the journal -- addressed to whoever reads logs, while the person who
5948        // just edited the config is looking at the CLI. Naming each section
5949        // individually rather than setting a flag: "something outside modules
5950        // changed" sends the operator back to diffing their own file, which is
5951        // the work this is meant to save.
5952        let mut restart_required = Vec::new();
5953        for section in RestartRequiredSection::ALL {
5954            let changed = match section {
5955                RestartRequiredSection::Port => configured_port != context.configured_port,
5956                RestartRequiredSection::Storage => storage_config != context.storage_config,
5957                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5958                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5959                }
5960                RestartRequiredSection::AdmissionFactsTargets => {
5961                    admission_facts_targets != context.admission_facts_targets
5962                }
5963                RestartRequiredSection::ScopeAuthorityOwners => {
5964                    scope_authority_owners != context.scope_authority_owners
5965                }
5966                RestartRequiredSection::PrivilegedOperatorPrompt => {
5967                    privileged_operator_prompt != context.privileged_operator_prompt
5968                }
5969            };
5970            if changed {
5971                restart_required.push(section.label().to_string());
5972            }
5973        }
5974        if !restart_required.is_empty() {
5975            warn!(
5976                config_path = %context.config_path.display(),
5977                sections = %restart_required.join(", "),
5978                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5979            );
5980        }
5981
5982        for configured in &modules {
5983            if let Err(err) = validate_spec(&configured.module_spec()) {
5984                return Ok(vec![control_error_frame(
5985                    &frame,
5986                    "invalid_daemon_config",
5987                    format!("supervisor rescan rejected daemon config: {err}"),
5988                )?]);
5989            }
5990        }
5991
5992        let configured_capabilities = modules
5993            .iter()
5994            .map(|module| (module.module_id.clone(), module.enabled))
5995            .collect::<Vec<_>>();
5996        let preview_capability_warnings = if preview {
5997            let (_, registrations) = self.runtime_capability_snapshot()?;
5998            let current_modules = self
5999                .supervisor
6000                .list()
6001                .into_iter()
6002                .map(|module| module.module_id().to_string())
6003                .collect::<BTreeSet<_>>();
6004            let resulting_modules = configured_capabilities.clone();
6005            let removed = current_modules
6006                .into_iter()
6007                .filter(|module_id| {
6008                    !resulting_modules
6009                        .iter()
6010                        .any(|(configured_id, _)| configured_id == module_id)
6011                })
6012                .collect::<Vec<_>>();
6013            // A candidate preview also names providers the candidate disables.
6014            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6015            let disabled = match &mode {
6016                RescanMode::Candidate(_) => resulting_modules
6017                    .iter()
6018                    .filter(|(_, enabled)| !enabled)
6019                    .map(|(module_id, _)| module_id.clone())
6020                    .collect::<Vec<_>>(),
6021                _ => Vec::new(),
6022            };
6023            self.capability_evaluator.preview_warnings(
6024                resulting_modules,
6025                &removed,
6026                &disabled,
6027                &registrations,
6028            )
6029        } else {
6030            Vec::new()
6031        };
6032        let reconciled = match &mode {
6033            RescanMode::Candidate(candidate) => self
6034                .preview_candidate(&context, modules, candidate)
6035                .map(|result| (result, Vec::new())),
6036            _ => {
6037                self.reconcile_supervised_modules(
6038                    &context.supervisor,
6039                    modules,
6040                    preview,
6041                    requester.clone(),
6042                )
6043                .await
6044            }
6045        };
6046        let (result, changed) = match reconciled {
6047            Ok(result) => result,
6048            Err(message) => {
6049                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6050            }
6051        };
6052        if !preview {
6053            if let Some(ledger) = &self.activity {
6054                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6055                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6056            }
6057            if let Err(code) = self.configure_activity(activity_config).await {
6058                warn!(
6059                    code,
6060                    "activity retention could not be applied during rescan"
6061                );
6062            }
6063            self.capability_evaluator
6064                .configure(configured_capabilities, reserved_capabilities);
6065            self.capability_evaluator.wake_deadline_loop();
6066            self.refresh_capability_requirements();
6067        }
6068        let mut result = result;
6069        result.restart_required = restart_required;
6070        result.capability_warnings = preview_capability_warnings;
6071        let response = ClientControlResponse::SupervisorRescan { result };
6072        Ok(vec![control_response_body_frame(
6073            &frame,
6074            &response,
6075            "ClientControlResponse::SupervisorRescan",
6076        )?])
6077    }
6078
6079    async fn handle_supervisor_release_reserved(
6080        &self,
6081        frame: Frame,
6082        module_id: String,
6083    ) -> Result<Vec<Frame>, RouterError> {
6084        let Some(context) = self.rescan.clone() else {
6085            return Ok(vec![control_error_frame(
6086                &frame,
6087                "release_unavailable",
6088                "reserved-id release requires a daemon started with a reloadable config path",
6089            )?]);
6090        };
6091        let operation_lock = self.supervisor.operation_lock();
6092        let _operation_guard = operation_lock.lock().await;
6093        let loaded = match crate::daemon_config::load(&context.config_path) {
6094            Ok(Some(config)) => config,
6095            Ok(None) => {
6096                return Ok(vec![control_error_frame(
6097                    &frame,
6098                    "invalid_daemon_config",
6099                    format!(
6100                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6101                        context.config_path.display()
6102                    ),
6103                )?])
6104            }
6105            Err(err) => {
6106                return Ok(vec![control_error_frame(
6107                    &frame,
6108                    "invalid_daemon_config",
6109                    format!("unable to verify reserved-id release against daemon config: {err}"),
6110                )?])
6111            }
6112        };
6113        if loaded
6114            .modules
6115            .iter()
6116            .any(|configured| configured.module_id == module_id)
6117        {
6118            return Ok(vec![control_error_frame(
6119                &frame,
6120                "reserved_module_configured",
6121                format!(
6122                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6123                ),
6124            )?]);
6125        }
6126        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6127            return Ok(vec![control_error_frame(
6128                &frame,
6129                "reserved_gate_not_retained",
6130                format!(
6131                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6132                ),
6133            )?]);
6134        }
6135
6136        let response = ClientControlResponse::SupervisorAck {
6137            module_id,
6138            applied: true,
6139        };
6140        Ok(vec![control_response_body_frame(
6141            &frame,
6142            &response,
6143            "ClientControlResponse::SupervisorAck",
6144        )?])
6145    }
6146
6147    /// Compare the configured module set with the supervised one, changing
6148    /// nothing. The staged spec each supervised module holds is what the
6149    /// configured entry is compared with.
6150    fn plan_rescan(
6151        &self,
6152        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6153    ) -> Result<RescanPlan, String> {
6154        let mut current = BTreeMap::new();
6155        for module in self.supervisor.list() {
6156            let (spec, health) = module.configuration().map_err(|err| {
6157                format!(
6158                    "failed to read configuration for module_id '{}': {err}",
6159                    module.module_id()
6160                )
6161            })?;
6162            let enabled = module
6163                .status()
6164                .map_err(|err| {
6165                    format!(
6166                        "failed to read status for module_id '{}': {err}",
6167                        module.module_id()
6168                    )
6169                })?
6170                .enabled;
6171            current.insert(
6172                module.module_id().to_string(),
6173                (module, spec, health, enabled),
6174            );
6175        }
6176        let configured = configured_modules
6177            .into_iter()
6178            .map(|module| (module.module_id.clone(), module))
6179            .collect::<BTreeMap<_, _>>();
6180
6181        let added = configured
6182            .keys()
6183            .filter(|module_id| !current.contains_key(*module_id))
6184            .cloned()
6185            .collect::<Vec<_>>();
6186        let removed = current
6187            .keys()
6188            .filter(|module_id| !configured.contains_key(*module_id))
6189            .cloned()
6190            .collect::<Vec<_>>();
6191        let mut changed_pending_reload = Vec::new();
6192        let mut configuration_changes = BTreeSet::new();
6193        let mut enabled_changes = BTreeSet::new();
6194        let mut unchanged = 0_u32;
6195
6196        for (module_id, configured_module) in &configured {
6197            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6198            else {
6199                continue;
6200            };
6201            // Compare the whole launch spec so a future launch field cannot
6202            // accidentally become a live-only policy change. Health is stored
6203            // separately and applies live without replacing the process.
6204            let launch_changed = *current_spec != configured_module.module_spec();
6205            let configuration_changed =
6206                launch_changed || *current_health != configured_module.health;
6207            let enabled_changed = *current_enabled != configured_module.enabled;
6208            if configuration_changed {
6209                configuration_changes.insert(module_id.clone());
6210            }
6211            if launch_changed {
6212                changed_pending_reload.push(module_id.clone());
6213            }
6214            if enabled_changed {
6215                enabled_changes.insert(module_id.clone());
6216            }
6217            if !configuration_changed && !enabled_changed {
6218                unchanged = unchanged.saturating_add(1);
6219            }
6220        }
6221        Ok(RescanPlan {
6222            current,
6223            configured,
6224            added,
6225            removed,
6226            changed_pending_reload,
6227            configuration_changes,
6228            enabled_changes,
6229            unchanged,
6230        })
6231    }
6232
6233    /// Reconcile the running module set against the configured one.
6234    ///
6235    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6236    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6237    /// deliberately shares this function with the executing path rather than
6238    /// computing the same diff somewhere else -- two implementations of one
6239    /// decision agree until they do not, and the whole value of a preview is that
6240    /// it describes the operation that will actually run.
6241    async fn reconcile_supervised_modules(
6242        &self,
6243        supervisor: &Supervisor,
6244        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6245        preview: bool,
6246        requester: Principal,
6247    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6248        let plan = self.plan_rescan(configured_modules)?;
6249
6250        // Everything above this point is pure computation over two snapshots.
6251        // Everything below MUTATES. The preview returns here so the boundary is a
6252        // single early return rather than a condition repeated at each mutation
6253        // site, where one missed guard would apply part of a change the caller was
6254        // told would not happen.
6255        if preview {
6256            return Ok((plan.preview_result(), Vec::new()));
6257        }
6258        let changed = plan
6259            .configuration_changes
6260            .union(&plan.enabled_changes)
6261            .cloned()
6262            .collect::<Vec<_>>();
6263        let RescanPlan {
6264            current,
6265            configured,
6266            added,
6267            removed,
6268            changed_pending_reload,
6269            configuration_changes,
6270            enabled_changes,
6271            unchanged,
6272        } = plan;
6273        // The staged generation moves before the first change rather than
6274        // after the last, so a rescan that fails partway still invalidates
6275        // every earlier `supervisor.rescan_checked` preview: part of its diff
6276        // may have run.
6277        if !(added.is_empty()
6278            && removed.is_empty()
6279            && configuration_changes.is_empty()
6280            && enabled_changes.is_empty())
6281        {
6282            if let Some(context) = &self.rescan {
6283                context
6284                    .generation
6285                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6286            }
6287        }
6288
6289        for module_id in &removed {
6290            let module = &current
6291                .get(module_id)
6292                .expect("removed module came from current supervisor state")
6293                .0;
6294            module
6295                .retire_requested(Some(requester.clone()))
6296                .await
6297                .map_err(|err| {
6298                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6299                })?;
6300            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6301            //
6302            // `handle_route_open` resolves an absent module in three steps:
6303            // registry, then supervisor status, then tombstone. Retiring first
6304            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6305            // went with the teardown above, the supervisor entry went with
6306            // `retire`, and the tombstone does not exist yet -- so a route.open
6307            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6308            // it") for a module that was deliberately removed and whose caller
6309            // should get `module_removed` (TERMINAL, carrying a removal age).
6310            //
6311            // Writing the tombstone first closes it: during the window the
6312            // supervisor entry still answers, so the caller gets
6313            // `target_unavailable` -- retryable, and TRUE, because the module
6314            // is mid-teardown. After both statements it is `module_removed`.
6315            // No instant remains where a removed module reads as one that
6316            // never existed.
6317            //
6318            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6319            // the absence of a test beside a fix invites deletion: these are
6320            // two sync statements with no await between them, so reaching the
6321            // window needs a second worker thread to land exactly between them
6322            // and there is no hook to force it. MEASURED: the 25 daemon_config
6323            // tests pass identically with the old order and the new one, so
6324            // the existing suite cannot see this and a green run is not
6325            // evidence either way. What the suite does hold is the
6326            // post-condition -- a removed module answers `module_removed` --
6327            // which this preserves.
6328            //
6329            // Found by an Athena panel reading the shipped tree against a
6330            // design note (2026-09-19), as the one concrete instance of that
6331            // note's class that survived contact with source. Direction is
6332            // benign: retryable where terminal was intended, never the reverse.
6333            self.supervisor.record_rescan_removal(module_id);
6334            self.supervisor.retire(module_id);
6335            self.route_outages.forget(module_id);
6336        }
6337
6338        for module_id in configured.keys() {
6339            let Some((module, _, _, _)) = current.get(module_id) else {
6340                continue;
6341            };
6342            let configured_module = configured
6343                .get(module_id)
6344                .expect("configured module id came from configured map");
6345            if configuration_changes.contains(module_id) {
6346                module
6347                    .update_configuration(
6348                        configured_module.module_spec(),
6349                        configured_module.health.clone(),
6350                        configured_module.drain_timeout_ms,
6351                    )
6352                    .await
6353                    .map_err(|err| {
6354                        format!(
6355                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6356                        )
6357                    })?;
6358            }
6359            if enabled_changes.contains(module_id) {
6360                // A rescan that starts or stops a module applies an operator's
6361                // edit to the config, so the resulting outage was asked for.
6362                self.route_outages.mark_operator_action(module_id);
6363                module
6364                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6365                    .await
6366                    .map_err(|err| {
6367                        self.route_outages.operator_action_ended_unrefused(module_id);
6368                        format!(
6369                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6370                            configured_module.enabled
6371                        )
6372                    })?;
6373            }
6374        }
6375
6376        for module_id in &added {
6377            let configured_module = configured
6378                .get(module_id)
6379                .expect("added module id came from configured map");
6380            supervisor
6381                .supervise_configured_requested(
6382                    configured_module.module_spec(),
6383                    configured_module.enabled,
6384                    configured_module.health.clone(),
6385                    configured_module.drain_timeout_ms,
6386                    configured_module.restart,
6387                    Some(requester.clone()),
6388                )
6389                .map_err(|err| {
6390                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6391                })?;
6392        }
6393
6394        Ok((
6395            SupervisorRescanResult {
6396                added,
6397                removed,
6398                changed_pending_reload,
6399                enabled_changes: enabled_changes.iter().cloned().collect(),
6400                unchanged,
6401                preview: false,
6402                // Filled by the caller, which is the only layer that can see the
6403                // previous config to diff against.
6404                restart_required: Vec::new(),
6405                capability_warnings: Vec::new(),
6406                module_actions: Vec::new(),
6407                earlier_pending: Vec::new(),
6408                expect: None,
6409            },
6410            changed,
6411        ))
6412    }
6413
6414    async fn handle_supervisor_set_enabled(
6415        &self,
6416        frame: Frame,
6417        module_id: String,
6418        enabled: bool,
6419        requester: Principal,
6420    ) -> Result<Vec<Frame>, RouterError> {
6421        let operation_lock = self.supervisor.operation_lock();
6422        let _operation_guard = operation_lock.lock().await;
6423        let Some(module) = self.supervisor.get(&module_id) else {
6424            return Ok(vec![control_error_frame(
6425                &frame,
6426                "unknown_module",
6427                format!("module_id '{module_id}' is not supervised"),
6428            )?]);
6429        };
6430
6431        // Enabling counts as well as disabling: a module an operator starts
6432        // is refused until it registers, and that wait was asked for.
6433        self.route_outages.mark_operator_action(&module_id);
6434        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6435            Ok(applied) => applied,
6436            Err(err) => {
6437                self.route_outages
6438                    .operator_action_ended_unrefused(&module_id);
6439                return Ok(vec![control_error_frame(
6440                    &frame,
6441                    "target_unavailable",
6442                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6443                )?]);
6444            }
6445        };
6446        if !applied {
6447            // Already in the requested state: nothing was made unavailable,
6448            // so the mark must not outlive this request.
6449            self.route_outages
6450                .operator_action_ended_unrefused(&module_id);
6451        }
6452
6453        self.capability_evaluator.wake_deadline_loop();
6454        self.refresh_capability_requirements();
6455        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6456        Ok(vec![control_response_body_frame(
6457            &frame,
6458            &response,
6459            "ClientControlResponse::SupervisorAck",
6460        )?])
6461    }
6462
6463    async fn handle_supervisor_health_probe(
6464        &self,
6465        frame: Frame,
6466        module_id: String,
6467    ) -> Result<Vec<Frame>, RouterError> {
6468        self.refresh_capability_requirements();
6469        let Some(registration) = self
6470            .registry
6471            .get_module(&module_id)
6472            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6473        else {
6474            return Ok(vec![control_error_frame(
6475                &frame,
6476                "unknown_module",
6477                format!("module_id '{module_id}' is not registered"),
6478            )?]);
6479        };
6480
6481        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6482        // named for it. Making `module_registration_grants_op` return false
6483        // unconditionally reddens five tests, and every one is named for something
6484        // else -- capability relay, probe/bind demultiplexing, supervision-only
6485        // probing. They exercise a successful advertisement check on the way to their
6486        // own subject.
6487        //
6488        // Real protection, fragile in a specific way: narrowing any of those tests to
6489        // focus on its stated subject would silently remove coverage nobody knows
6490        // they are carrying. Recorded here rather than as a sixth test, because the
6491        // useful fact is WHICH tests hold the guard up -- a new test would add
6492        // coverage without telling the next person what the existing ones quietly do.
6493        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6494        {
6495            return Ok(vec![control_error_frame(
6496                &frame,
6497                "health_not_advertised",
6498                format!("module_id '{module_id}' did not advertise health.check"),
6499            )?]);
6500        }
6501
6502        let deadline = Instant::now() + self.health_probe_timeout;
6503        let pending = match self.forwarding.begin_module_control_rpc_for(
6504            &module_id,
6505            MODULE_CONTROL_OP_HEALTH_CHECK,
6506            deadline,
6507        ) {
6508            Ok(pending) => pending,
6509            Err(err) => {
6510                return Ok(vec![control_error_frame(
6511                    &frame,
6512                    forwarding_error_code(&err),
6513                    err.to_string(),
6514                )?])
6515            }
6516        };
6517
6518        let PendingModuleControlRpc {
6519            endpoint,
6520            module_sink,
6521            negotiated_ver,
6522            corr: probe_corr,
6523            receiver,
6524        } = pending;
6525        let mut guard =
6526            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6527        let probe_body =
6528            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6529                RouterError::backend(
6530                    0,
6531                    frame.header.corr,
6532                    format!("failed to encode health.check request: {err}"),
6533                )
6534            })?;
6535        let probe_frame = Frame::build_with_version(
6536            negotiated_ver,
6537            FrameType::Request,
6538            control_flags(),
6539            0,
6540            0,
6541            probe_corr,
6542            probe_body,
6543        )
6544        .map_err(RouterError::FrameBuild)?;
6545
6546        // Queue admission and the reply share one deadline, so a module that
6547        // stops reading cannot park the requesting connection's reader forever.
6548        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6549            Ok(Ok(())) => {}
6550            Ok(Err(err)) => {
6551                return Ok(vec![control_error_frame(
6552                    &frame,
6553                    "target_unavailable",
6554                    err.to_string(),
6555                )?]);
6556            }
6557            Err(_) => {
6558                return Ok(vec![control_error_frame(
6559                    &frame,
6560                    "module_timeout",
6561                    format!(
6562                        "module_id '{module_id}' did not answer health.check within {:?}",
6563                        self.health_probe_timeout
6564                    ),
6565                )?]);
6566            }
6567        }
6568
6569        match timeout_at(deadline, receiver).await {
6570            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6571                guard.disarm();
6572                let Some(report) = response.health_report() else {
6573                    return Ok(vec![control_error_frame(
6574                        &frame,
6575                        "invalid_control_body",
6576                        "health.check RPC returned a non-health response",
6577                    )?]);
6578                };
6579                // Metrics go out whole here. The supervisor's cached snapshot
6580                // caps this blob (see truncate_health_metrics), and this path
6581                // exists precisely to answer without that cap -- so applying it
6582                // here would leave no way to see what the cached view drops.
6583                let HealthReport {
6584                    status,
6585                    detail,
6586                    metrics,
6587                } = report;
6588                let capability_detail = self
6589                    .capability_evaluator
6590                    .required_problem_detail(&module_id);
6591                let response = ClientControlResponse::SupervisorHealthProbe {
6592                    module_id,
6593                    status,
6594                    detail: append_capability_problem_detail(detail, capability_detail),
6595                    metrics,
6596                };
6597                Ok(vec![control_response_body_frame(
6598                    &frame,
6599                    &response,
6600                    "ClientControlResponse::SupervisorHealthProbe",
6601                )?])
6602            }
6603            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6604                guard.disarm();
6605                Ok(vec![control_error_body_frame(&frame, body)?])
6606            }
6607            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6608                guard.disarm();
6609                Ok(vec![control_error_frame(
6610                    &frame,
6611                    "target_unavailable",
6612                    message,
6613                )?])
6614            }
6615            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6616                guard.disarm();
6617                Ok(vec![control_error_frame(
6618                    &frame,
6619                    "invalid_control_body",
6620                    message,
6621                )?])
6622            }
6623            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6624                guard.disarm();
6625                Ok(vec![control_error_frame(
6626                    &frame,
6627                    "invalid_control_body",
6628                    format!("expected module-control op '{expected}', got '{actual}'"),
6629                )?])
6630            }
6631            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6632                guard.disarm();
6633                Ok(vec![control_error_frame(
6634                    &frame,
6635                    "module_timeout",
6636                    format!(
6637                        "module_id '{module_id}' answered health.check after {:?}",
6638                        self.health_probe_timeout
6639                    ),
6640                )?])
6641            }
6642            Ok(Err(_)) => Ok(vec![control_error_frame(
6643                &frame,
6644                "target_unavailable",
6645                "health.check waiter was canceled before the module responded",
6646            )?]),
6647            Err(_) => Ok(vec![control_error_frame(
6648                &frame,
6649                "module_timeout",
6650                format!(
6651                    "module_id '{module_id}' did not answer health.check within {:?}",
6652                    self.health_probe_timeout
6653                ),
6654            )?]),
6655        }
6656    }
6657
6658    fn supervisor_status(
6659        &self,
6660        module_id: &str,
6661        corr: u64,
6662    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6663        self.supervisor
6664            .get(module_id)
6665            .map(|module| {
6666                let warming = module.is_warming_for_control("status").map_err(|err| {
6667                    RouterError::backend(
6668                        0,
6669                        corr,
6670                        format!(
6671                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6672                        ),
6673                    )
6674                })?;
6675                module.status_for_control("status").map_err(|err| {
6676                    RouterError::backend(
6677                        0,
6678                        corr,
6679                        format!(
6680                            "failed to read supervisor status for module_id '{module_id}': {err}"
6681                        ),
6682                    )
6683                }).map(|status| (status, warming))
6684            })
6685            .transpose()
6686    }
6687
6688    fn guard_module_control_op(
6689        &self,
6690        frame: &Frame,
6691        module_id: &str,
6692        op: &str,
6693    ) -> Result<Option<Frame>, RouterError> {
6694        if self.module_grants_op(module_id, op, frame.header.corr)? {
6695            return Ok(None);
6696        }
6697
6698        Ok(Some(control_error_frame(
6699            frame,
6700            "op_not_allowed",
6701            format!("module_id '{module_id}' did not grant control op '{op}'"),
6702        )?))
6703    }
6704
6705    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6706        let Some(registration) = self
6707            .registry
6708            .get_module(module_id)
6709            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6710        else {
6711            return Ok(false);
6712        };
6713        Ok(module_registration_grants_op(&registration.control_ops, op))
6714    }
6715
6716    fn handle_status_update(
6717        &self,
6718        endpoint: ModuleEndpointId,
6719        frame: Frame,
6720    ) -> Result<Vec<Frame>, RouterError> {
6721        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6722            Ok(update) => update,
6723            Err(err) => {
6724                // Forward-compat: a newer module may push a channel-0 op this subc
6725                // version doesn't know. The control contract says unknown push ops
6726                // are IGNORED, never answered with an error. Only a malformed body
6727                // for an op we DO know is a real error worth surfacing.
6728                if is_known_module_push_op(&frame.body) {
6729                    return Ok(vec![control_error_frame(
6730                        &frame,
6731                        "invalid_control_body",
6732                        format!("malformed module control push body: {err}"),
6733                    )?]);
6734                }
6735                return Ok(Vec::new());
6736            }
6737        };
6738
6739        match update {
6740            ModuleControlPush::RouteStatus {
6741                route_channel,
6742                route_epoch,
6743                status,
6744            } => {
6745                self.forwarding
6746                    .cache_status(endpoint, route_channel, route_epoch, status)
6747                    .map_err(RouterError::Forwarding)?;
6748            }
6749        }
6750        Ok(Vec::new())
6751    }
6752
6753    fn handle_route_poll(
6754        &self,
6755        ctx: &RouteCtx,
6756        frame: Frame,
6757        route_channel: u16,
6758        route_epoch: u32,
6759        kind: PollKind,
6760    ) -> Result<Vec<Frame>, RouterError> {
6761        let snapshot = self
6762            .forwarding
6763            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6764            .map_err(RouterError::Forwarding)?;
6765        let response = match (kind, snapshot) {
6766            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6767                ClientControlResponse::RoutePoll {
6768                    route_channel,
6769                    route_epoch,
6770                    status,
6771                    live: None,
6772                }
6773            }
6774            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6775                route_channel,
6776                route_epoch,
6777                status: None,
6778                live: None,
6779            },
6780            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6781                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6782                // is what makes reporting `true` correct rather than a
6783                // confident guess. `process_live` returns None only when the
6784                // module id has no supervisor snapshot at all -- an
6785                // externally-started module the daemon did not spawn -- and
6786                // for those the supervisor has no opinion to offer, ever. It
6787                // is never None for a supervised module in an unknown state:
6788                // a supervised module always has a snapshot, and the answer
6789                // comes from `state == Running && process_alive`.
6790                //
6791                // The route is Bound, so the module completed a HELLO on a
6792                // live connection; "the process this route points at is
6793                // running" is therefore attested by the binding rather than
6794                // assumed. Reporting `false` for an unsupervised module would
6795                // be the actual lie -- it would tell a client its healthy
6796                // route is dead because the daemon does not manage the
6797                // process.
6798                //
6799                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6800                // module whose liveness is genuinely unknown, e.g. a snapshot
6801                // that has not been populated yet -- THIS DEFAULT BECOMES
6802                // WRONG and must split: unsupervised stays true, unknown
6803                // becomes null so the client can tell the two apart. The
6804                // response field is already `Option<bool>`, so the wire can
6805                // carry that distinction today.
6806                let live = self
6807                    .process_liveness
6808                    .as_ref()
6809                    .and_then(|source| source.process_live(&module_id))
6810                    .unwrap_or(true);
6811                ClientControlResponse::RoutePoll {
6812                    route_channel,
6813                    route_epoch,
6814                    status: None,
6815                    live: Some(live),
6816                }
6817            }
6818            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6819                route_channel,
6820                route_epoch,
6821                status: None,
6822                live: Some(false),
6823            },
6824        };
6825
6826        Ok(vec![control_response_body_frame(
6827            &frame,
6828            &response,
6829            "ClientControlResponse::RoutePoll",
6830        )?])
6831    }
6832
6833    pub(crate) fn observe_module_control_completion(
6834        &self,
6835        completion: ModuleControlRpcCompletion,
6836    ) -> bool {
6837        match completion {
6838            ModuleControlRpcCompletion::Unknown => false,
6839            ModuleControlRpcCompletion::Settled => true,
6840            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6841                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6842                info!(
6843                    module_id = %module_id,
6844                    latency_ms,
6845                    "late health.check answer proves the module is alive"
6846                );
6847                match self
6848                    .supervisor
6849                    .record_late_health_answer(&module_id, latency_ms)
6850                {
6851                    Ok(true) => {}
6852                    Ok(false) => debug!(
6853                        module_id = %module_id,
6854                        latency_ms,
6855                        "late health.check answer has no active supervisor snapshot"
6856                    ),
6857                    Err(err) => warn!(
6858                        module_id = %module_id,
6859                        latency_ms,
6860                        error = %err,
6861                        "failed to record late health.check answer"
6862                    ),
6863                }
6864                true
6865            }
6866        }
6867    }
6868
6869    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6870    /// the module connection whose frame is being handled, or to the client that
6871    /// relay was opened for.
6872    ///
6873    /// This runs on the MODULE connection's frame handler, where returning `Err`
6874    /// ends that connection -- and a module connection carries every client's
6875    /// routes to that module, so ending it costs the whole fleet its tools.
6876    /// `ConnectionClosing` carries the id of the connection that is closing, and
6877    /// when that id is a CLIENT's, the condition is entirely about that one
6878    /// client's route.open. A client-scoped condition has no authority over a
6879    /// shared module connection, so it is logged and the single relay is dropped:
6880    /// the client is going away, and `complete_pending_relay` already removed the
6881    /// relay before failing, so there is nothing left to settle. Anything that
6882    /// relay still reserved is released by that client's own connection teardown,
6883    /// which is already under way -- that is what "closing" means.
6884    ///
6885    /// Every other failure is a statement about THIS connection and stays fatal:
6886    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6887    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6888    /// frames correctly.
6889    fn refuse_to_end_module_connection_for_a_client(
6890        &self,
6891        module_connection_id: ConnectionId,
6892        corr: u64,
6893        err: ForwardingError,
6894    ) -> Result<(), RouterError> {
6895        if let ForwardingError::ConnectionClosing { connection_id } = err {
6896            if connection_id != module_connection_id {
6897                warn!(
6898                    module_connection_id = module_connection_id.get(),
6899                    client_connection_id = connection_id.get(),
6900                    corr,
6901                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6902                );
6903                return Ok(());
6904            }
6905        }
6906        Err(RouterError::Forwarding(err))
6907    }
6908
6909    fn handle_module_relay_response(
6910        &self,
6911        connection_id: ConnectionId,
6912        frame: Frame,
6913    ) -> Result<Vec<Frame>, RouterError> {
6914        let mut secondary_error = None;
6915        let outcome = match frame.header.ty {
6916            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6917                Ok(probe) if probe.op == "route.bind" => {
6918                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6919                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6920                            RouteBindRelayOutcome::Accepted
6921                        }
6922                        Ok(other) => {
6923                            let message =
6924                                format!("route.bind response carried unexpected body: {other:?}");
6925                            secondary_error = Some(control_error_frame(
6926                                &frame,
6927                                "invalid_control_body",
6928                                message.clone(),
6929                            )?);
6930                            RouteBindRelayOutcome::ModuleGone(message)
6931                        }
6932                        Err(err) => {
6933                            let message = format!("malformed route.bind response body: {err}");
6934                            secondary_error = Some(control_error_frame(
6935                                &frame,
6936                                "invalid_control_body",
6937                                message.clone(),
6938                            )?);
6939                            RouteBindRelayOutcome::ModuleGone(message)
6940                        }
6941                    }
6942                }
6943                Ok(probe) => {
6944                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6945                    {
6946                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6947                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6948                            "malformed {} response body: {err}",
6949                            probe.op
6950                        )),
6951                    };
6952                    let completion = self
6953                        .forwarding
6954                        .complete_module_control_rpc(
6955                            connection_id,
6956                            frame.header.corr,
6957                            Some(&probe.op),
6958                            outcome,
6959                        )
6960                        .map_err(RouterError::Forwarding)?;
6961                    if !self.observe_module_control_completion(completion) {
6962                        debug!(
6963                            connection_id = connection_id.get(),
6964                            corr = frame.header.corr,
6965                            op = %probe.op,
6966                            "dropping late or unknown module-control RPC response"
6967                        );
6968                    }
6969                    return Ok(Vec::new());
6970                }
6971                Err(err) => {
6972                    if let Some(expected_op) = self
6973                        .forwarding
6974                        .pending_module_control_op(connection_id, frame.header.corr)
6975                        .map_err(RouterError::Forwarding)?
6976                    {
6977                        let completion = self
6978                            .forwarding
6979                            .complete_module_control_rpc(
6980                                connection_id,
6981                                frame.header.corr,
6982                                None,
6983                                ModuleControlRpcOutcome::MalformedResponse(format!(
6984                                    "malformed {expected_op} response body: {err}"
6985                                )),
6986                            )
6987                            .map_err(RouterError::Forwarding)?;
6988                        if !self.observe_module_control_completion(completion) {
6989                            debug!(
6990                                connection_id = connection_id.get(),
6991                                corr = frame.header.corr,
6992                                "dropping late malformed module-control RPC response"
6993                            );
6994                        }
6995                        return Ok(Vec::new());
6996                    }
6997                    let message = format!("malformed route.bind response body: {err}");
6998                    secondary_error = Some(control_error_frame(
6999                        &frame,
7000                        "invalid_control_body",
7001                        message.clone(),
7002                    )?);
7003                    RouteBindRelayOutcome::ModuleGone(message)
7004                }
7005            },
7006            FrameType::Error => {
7007                if self
7008                    .forwarding
7009                    .pending_module_control_op(connection_id, frame.header.corr)
7010                    .map_err(RouterError::Forwarding)?
7011                    .is_some()
7012                {
7013                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7014                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7015                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7016                            "malformed module-control ERROR body: {err}"
7017                        )),
7018                    };
7019                    let completion = self
7020                        .forwarding
7021                        .complete_module_control_rpc(
7022                            connection_id,
7023                            frame.header.corr,
7024                            None,
7025                            outcome,
7026                        )
7027                        .map_err(RouterError::Forwarding)?;
7028                    if !self.observe_module_control_completion(completion) {
7029                        debug!(
7030                            connection_id = connection_id.get(),
7031                            corr = frame.header.corr,
7032                            "dropping late or unknown module-control RPC error"
7033                        );
7034                    }
7035                    return Ok(Vec::new());
7036                }
7037                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7038                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7039                    Err(err) => {
7040                        let message = format!("malformed route.bind ERROR body: {err}");
7041                        secondary_error = Some(control_error_frame(
7042                            &frame,
7043                            "invalid_control_body",
7044                            message.clone(),
7045                        )?);
7046                        RouteBindRelayOutcome::ModuleGone(message)
7047                    }
7048                }
7049            }
7050            ty => {
7051                return Ok(vec![control_error_frame(
7052                    &frame,
7053                    "unsupported_control_frame",
7054                    format!("unsupported module channel-0 frame {ty:?}"),
7055                )?])
7056            }
7057        };
7058
7059        let settled =
7060            self.forwarding
7061                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7062        let completion = match settled {
7063            Ok(completion) => completion,
7064            Err(err) => {
7065                self.refuse_to_end_module_connection_for_a_client(
7066                    connection_id,
7067                    frame.header.corr,
7068                    err,
7069                )?;
7070                return Ok(secondary_error.into_iter().collect());
7071            }
7072        };
7073        if let Some(target) = completion.abandoned.as_ref() {
7074            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7075        }
7076        if !completion.settled {
7077            debug!(
7078                connection_id = connection_id.get(),
7079                corr = frame.header.corr,
7080                frame_type = ?frame.header.ty,
7081                "dropping late or unknown route.bind relay response"
7082            );
7083        }
7084        Ok(secondary_error.into_iter().collect())
7085    }
7086
7087    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7088        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7089        // GOODBYE ends the connection's logical session even when its socket
7090        // stays open. Use disconnect teardown so verdicts, client notices and
7091        // scope authority are released at the same lifecycle boundary.
7092        self.cleanup_connection_with_end_reason(
7093            connection_id,
7094            RegistrationEndReason::ExplicitGoodbye,
7095        )
7096        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7097        Ok(Vec::new())
7098    }
7099}
7100
7101impl Default for ControlHandler {
7102    fn default() -> Self {
7103        Self::new(Arc::new(Registry::default()))
7104    }
7105}
7106
7107impl crate::supervise::SwapPromotionObserver for ControlHandler {
7108    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7109        self.apply_registration_capabilities(registration);
7110    }
7111}
7112
7113fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7114    CapabilityRequirementStatus {
7115        consumer: status.consumer,
7116        capability: status.capability,
7117        need: match status.need {
7118            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7119            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7120        },
7121        verdict: status.verdict.as_str().to_string(),
7122        episode_seq: status.episode_seq,
7123        config_satisfiable: status.config_satisfiable,
7124        runtime_available: status.runtime_available,
7125        detail: status.detail,
7126    }
7127}
7128
7129fn append_capability_problem_detail(
7130    detail: Option<String>,
7131    capability_detail: Option<String>,
7132) -> Option<String> {
7133    match (detail, capability_detail) {
7134        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7135        (Some(detail), None) => Some(detail),
7136        (None, Some(capability_detail)) => Some(capability_detail),
7137        (None, None) => None,
7138    }
7139}
7140
7141fn subc_ops() -> Vec<String> {
7142    SUBC_CONTROL_OPS
7143        .iter()
7144        .map(|op| (*op).to_string())
7145        .collect()
7146}
7147
7148fn module_subc_ops() -> Vec<String> {
7149    SUBC_CONTROL_OPS
7150        .iter()
7151        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7152        .map(|op| (*op).to_string())
7153        .collect()
7154}
7155
7156#[cfg(test)]
7157fn module_baseline_control_ops() -> Vec<String> {
7158    MODULE_BASELINE_CONTROL_OPS
7159        .iter()
7160        .map(|op| (*op).to_string())
7161        .collect()
7162}
7163
7164fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7165    let mut seen = HashSet::new();
7166    let mut effective = Vec::new();
7167    for op in MODULE_BASELINE_CONTROL_OPS {
7168        if seen.insert((*op).to_string()) {
7169            effective.push((*op).to_string());
7170        }
7171    }
7172    for op in declared.unwrap_or_default() {
7173        if seen.insert(op.clone()) {
7174            effective.push(op);
7175        }
7176    }
7177    effective
7178}
7179
7180fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7181    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7182}
7183
7184fn target_module_id(target: &RouteTarget) -> &str {
7185    match target {
7186        RouteTarget::ToolProvider { module_id }
7187        | RouteTarget::ManagementSurface { module_id }
7188        | RouteTarget::InternalService { module_id, .. } => module_id,
7189    }
7190}
7191
7192fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7193    roles.iter().any(|role| match (target, role) {
7194        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7195        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7196        (
7197            RouteTarget::InternalService { service_id, .. },
7198            ProviderRole::InternalService {
7199                service_id: provided,
7200                ..
7201            },
7202        ) => service_id == provided,
7203        _ => false,
7204    })
7205}
7206
7207fn is_routable_role(role: &ProviderRole) -> bool {
7208    matches!(
7209        role,
7210        ProviderRole::ToolProvider { .. }
7211            | ProviderRole::ManagementSurface { .. }
7212            | ProviderRole::InternalService { .. }
7213    )
7214}
7215
7216#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7217enum ControlRequestBodyError {
7218    UnknownOp,
7219    InvalidBody,
7220}
7221
7222#[derive(Debug, Deserialize)]
7223struct ControlOpProbe {
7224    op: String,
7225}
7226
7227/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7228/// this set is treated as a forward-compat unknown and ignored rather than errored.
7229const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7230
7231fn is_known_module_push_op(body: &[u8]) -> bool {
7232    serde_json::from_slice::<ControlOpProbe>(body)
7233        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7234        .unwrap_or(false)
7235}
7236
7237fn is_known_module_request_op(body: &[u8]) -> bool {
7238    serde_json::from_slice::<ControlOpProbe>(body)
7239        .map(|probe| is_module_to_subc_op(&probe.op))
7240        .unwrap_or(false)
7241}
7242
7243fn is_module_to_subc_op(op: &str) -> bool {
7244    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7245}
7246
7247fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7248    debug!(
7249        op = %op,
7250        connection_id = connection_id.get(),
7251        corr,
7252        "control dispatch"
7253    );
7254}
7255
7256fn log_slow_control_dispatch(
7257    dispatch_started_at: Option<StdInstant>,
7258    op: &'static str,
7259    connection_id: ConnectionId,
7260    corr: u64,
7261    route_open_timings: Option<&RouteOpenTimings>,
7262) {
7263    let Some(dispatch_started_at) = dispatch_started_at else {
7264        return;
7265    };
7266    // Use the same monotonic clock as the phase timers and relay deadline.
7267    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7268    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7269        if let Some(timings) = route_open_timings {
7270            warn!(
7271                op = %op,
7272                connection_id = connection_id.get(),
7273                corr,
7274                elapsed_ms = elapsed.as_millis() as u64,
7275                module_id = ?timings.module_id,
7276                principal = %timings.principal,
7277                admission_ms = timings.admission.as_millis() as u64,
7278                project_root_ms = timings.project_root.as_millis() as u64,
7279                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7280                bind_send_ms = timings.bind_send.as_millis() as u64,
7281                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7282                "slow control dispatch"
7283            );
7284            return;
7285        }
7286        warn!(
7287            op = %op,
7288            connection_id = connection_id.get(),
7289            corr,
7290            elapsed_ms = elapsed.as_millis() as u64,
7291            "slow control dispatch"
7292        );
7293    }
7294}
7295
7296/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7297#[cfg(test)]
7298type TestCallerLookup =
7299    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7300
7301/// Every new client operation needs an explicit privilege decision here.
7302fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7303    match request {
7304        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7305            Some(PrivilegedOperation::Restart { module_id })
7306        }
7307        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7308            Some(PrivilegedOperation::Swap { module_id })
7309        }
7310        ClientControlRequest::SupervisorReload { module_id } => {
7311            Some(PrivilegedOperation::Reload { module_id })
7312        }
7313        ClientControlRequest::SupervisorRescan { preview } => {
7314            if *preview {
7315                None
7316            } else {
7317                Some(PrivilegedOperation::Rescan)
7318            }
7319        }
7320        // Only the well-formed apply asks for approval. A preview changes
7321        // nothing, and any other combination is refused before it touches
7322        // anything, so prompting the operator for it would ask about a request
7323        // that cannot run.
7324        ClientControlRequest::SupervisorRescanChecked {
7325            preview,
7326            candidate,
7327            expect,
7328        } => (!*preview && candidate.is_none() && expect.is_some())
7329            .then_some(PrivilegedOperation::Rescan),
7330        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7331            Some(PrivilegedOperation::SetEnabled {
7332                module_id,
7333                enabled: *enabled,
7334            })
7335        }
7336        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7337            Some(PrivilegedOperation::ReleaseReserved { module_id })
7338        }
7339        ClientControlRequest::ServerDescribe {}
7340        | ClientControlRequest::CatalogList { .. }
7341        | ClientControlRequest::RouteOpen { .. }
7342        | ClientControlRequest::RoutePoll { .. }
7343        | ClientControlRequest::SupervisorList {}
7344        | ClientControlRequest::SupervisorHealth {}
7345        | ClientControlRequest::SupervisorHealthProbe { .. }
7346        | ClientControlRequest::SupervisorRoutes { .. }
7347        | ClientControlRequest::SupervisorProvenance { .. }
7348        | ClientControlRequest::SupervisorStderrTail { .. }
7349        | ClientControlRequest::SupervisorTerminals { .. }
7350        | ClientControlRequest::SupervisorSpawnSnapshot {}
7351        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7352    }
7353}
7354
7355fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7356    match request {
7357        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7358        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7359        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7360        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7361        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7362        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7363        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7364        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7365        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7366        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7367        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7368        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7369        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7370        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7371        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7372        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7373        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7374        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7375        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7376        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7377    }
7378}
7379
7380/// The caller of one `route.open`, as its `route.open refused` line names it.
7381///
7382/// A connection id alone cannot name a caller after the fact: ids restart at
7383/// every daemon restart, so the same number belongs to different modules
7384/// before and after one. These three fields are what the refused line needs
7385/// to be attributable on its own, and they are rendered with the same helpers
7386/// as the `route.open accepted` line so both directions grep alike.
7387struct RouteOpenCaller {
7388    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
7389    /// when the request presented a consumer identity whose launch nonce did
7390    /// not verify. The claimed module id is never printed as a principal.
7391    principal: String,
7392    /// `none` for an unscoped open; the admitted stamp's label once scope
7393    /// admission issued one; before that, the requested selector's label.
7394    scope: String,
7395    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
7396    /// because it is a string the caller chose.
7397    harness: String,
7398}
7399
7400impl RouteOpenCaller {
7401    fn new(
7402        principal: Option<&Principal>,
7403        scope: Option<&ScopeSelector>,
7404        identity: &BindIdentity,
7405    ) -> Self {
7406        Self {
7407            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7408            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7409            harness: bounded_log_text(&identity.harness),
7410        }
7411    }
7412
7413    /// For a refusal that has no parsed request at hand: names whatever the
7414    /// frame still says, and `unknown` where it says nothing parseable.
7415    fn unknown() -> Self {
7416        Self {
7417            principal: "unknown".to_string(),
7418            scope: "unknown".to_string(),
7419            harness: "unknown".to_string(),
7420        }
7421    }
7422}
7423
7424/// Longest client-chosen text (a scope ref, a harness) a route.open log line
7425/// prints before cutting it.
7426const MAX_LOGGED_TEXT_CHARS: usize = 64;
7427
7428/// `value` Debug-quoted, so control characters land escaped and cannot forge
7429/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
7430/// `…` inside the quotes so one request cannot flood the line.
7431fn bounded_log_text(value: &str) -> String {
7432    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7433    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7434        bounded.push('…');
7435    }
7436    format!("{bounded:?}")
7437}
7438
7439/// The `scope=` field of the `route.open accepted` line, from the stamp that
7440/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7441/// so a ref holding control characters cannot forge a log line, and cut to 64
7442/// characters with a trailing `…` to bound the line. The bind proof and the
7443/// scope's attributes never appear.
7444fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7445    use subc_protocol::scope::ScopeKind;
7446    let kind = match stamp.kind {
7447        ScopeKind::Head => "head",
7448        ScopeKind::Worker => "worker",
7449        ScopeKind::Ephemeral => "ephemeral",
7450    };
7451    format!(
7452        "{kind}/{}/{}@{}",
7453        principal_label(&stamp.owner),
7454        bounded_log_text(&stamp.scope_ref),
7455        stamp.scope_epoch
7456    )
7457}
7458
7459/// The `scope=` field of a `route.open refused` line for an open refused
7460/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
7461/// The kind is not known yet, so the literal `requested` stands in its place.
7462/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
7463/// owner it has not been checked against the configured modules: a quoted
7464/// owner means the value was never validated. The bind proof never appears.
7465fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7466    let epoch = selector
7467        .scope_epoch
7468        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7469    format!(
7470        "requested/{}/{}@{epoch}",
7471        bounded_log_text(&principal_label(&selector.owner)),
7472        bounded_log_text(&selector.scope_ref),
7473    )
7474}
7475
7476fn principal_label(principal: &Principal) -> String {
7477    match principal {
7478        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7479        Principal::Direct => "direct".to_string(),
7480        other => format!("{other:?}"),
7481    }
7482}
7483
7484fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7485    match request {
7486        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7487        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7488        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7489        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7490        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7491    }
7492}
7493
7494fn parse_client_control_request(
7495    body: &[u8],
7496) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7497    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7498        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7499            Ok(probe)
7500                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7501                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7502            {
7503                ControlRequestBodyError::InvalidBody
7504            }
7505            Ok(_) => ControlRequestBodyError::UnknownOp,
7506            Err(_) => ControlRequestBodyError::InvalidBody,
7507        };
7508        (err, classification)
7509    })
7510}
7511
7512fn parse_module_control_request_from_module(
7513    body: &[u8],
7514) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7515    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7516        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7517            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7518            Ok(_) => ControlRequestBodyError::UnknownOp,
7519            Err(_) => ControlRequestBodyError::InvalidBody,
7520        };
7521        (err, classification)
7522    })
7523}
7524
7525#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7526enum ProviderRoleKind {
7527    ToolProvider,
7528    PipelineStage,
7529    ManagementSurface,
7530    InternalService,
7531}
7532
7533fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7534    match role {
7535        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7536        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7537        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7538        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7539    }
7540}
7541
7542fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7543    roles.iter().map(provider_role_kind).collect()
7544}
7545
7546/// Most refused scope records named individually in the log per sync; the
7547/// `refused` count on the accepted line is always complete.
7548const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7549
7550/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7551#[derive(Debug, Default, PartialEq, Eq)]
7552struct ScopeOutcomeCounts {
7553    created: usize,
7554    replaced: usize,
7555    updated: usize,
7556    unchanged: usize,
7557    refused: usize,
7558}
7559
7560impl ScopeOutcomeCounts {
7561    fn of(results: &[ScopeRecordResult]) -> Self {
7562        let mut counts = Self::default();
7563        for result in results {
7564            let slot = match result.outcome {
7565                ScopeRecordOutcome::Created => &mut counts.created,
7566                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7567                ScopeRecordOutcome::Updated => &mut counts.updated,
7568                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7569                ScopeRecordOutcome::Refused => &mut counts.refused,
7570            };
7571            *slot += 1;
7572        }
7573        counts
7574    }
7575}
7576
7577#[cfg(test)]
7578mod scope_outcome_count_tests {
7579    use super::*;
7580
7581    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7582        ScopeRecordResult {
7583            scope_ref: "r".to_string(),
7584            scope_epoch: 1,
7585            outcome,
7586            code: None,
7587            message: None,
7588            version: None,
7589            parent_state: None,
7590        }
7591    }
7592
7593    /// Each outcome lands in its own count, so a refused record can never be
7594    /// hidden inside the total the log already printed.
7595    #[test]
7596    fn every_outcome_is_counted_in_its_own_field() {
7597        let results = [
7598            result(ScopeRecordOutcome::Created),
7599            result(ScopeRecordOutcome::Created),
7600            result(ScopeRecordOutcome::Replaced),
7601            result(ScopeRecordOutcome::Updated),
7602            result(ScopeRecordOutcome::Unchanged),
7603            result(ScopeRecordOutcome::Refused),
7604            result(ScopeRecordOutcome::Refused),
7605            result(ScopeRecordOutcome::Refused),
7606        ];
7607        assert_eq!(
7608            ScopeOutcomeCounts::of(&results),
7609            ScopeOutcomeCounts {
7610                created: 2,
7611                replaced: 1,
7612                updated: 1,
7613                unchanged: 1,
7614                refused: 3,
7615            }
7616        );
7617    }
7618}
7619
7620/// Return whether a catalog change can create a newly violating live route.
7621/// Removing an attested claim is intentionally excluded: it makes fewer routes
7622/// forbidden and therefore must leave the existing route census untouched.
7623fn capability_census_trigger(
7624    old: Option<&CapabilityDeclarations>,
7625    new: Option<&CapabilityDeclarations>,
7626) -> bool {
7627    let old_provides = old
7628        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7629        .unwrap_or_default();
7630    let old_denies = old
7631        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7632        .unwrap_or_default();
7633    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7634        provides: Vec::new(),
7635        requires: Vec::new(),
7636        must_never_reach: Vec::new(),
7637    });
7638
7639    new.provides
7640        .iter()
7641        .any(|capability| !old_provides.contains(capability))
7642        || new
7643            .must_never_reach
7644            .iter()
7645            .any(|capability| !old_denies.contains(capability))
7646}
7647
7648/// Find the first capability an attested opener denies that an attested target
7649/// claims. Both manifests are live registry records, never cached or client data.
7650fn denied_capability<'a>(
7651    opening_manifest: &'a ModuleManifest,
7652    target_manifest: &ModuleManifest,
7653) -> Option<&'a str> {
7654    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7655    let target_capabilities = target_manifest.capabilities.as_ref()?;
7656    opening_capabilities
7657        .must_never_reach
7658        .iter()
7659        .find(|denied| {
7660            target_capabilities
7661                .provides
7662                .iter()
7663                .any(|provided| provided == *denied)
7664        })
7665        .map(String::as_str)
7666}
7667
7668fn catalog_update_frozen_field_message(
7669    registered: &ModuleManifest,
7670    provides: &[ProviderRole],
7671) -> Option<String> {
7672    let old_has_provides = !registered.provides.is_empty();
7673    let new_has_provides = !provides.is_empty();
7674    if old_has_provides != new_has_provides {
7675        return Some(format!(
7676            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7677            registered.module_id
7678        ));
7679    }
7680
7681    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7682        return Some(format!(
7683            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7684            registered.module_id
7685        ));
7686    }
7687
7688    let registered_concurrency = manifest_concurrency(registered);
7689    let mut candidate = registered.clone();
7690    candidate.provides = provides.to_vec();
7691    let candidate_concurrency = manifest_concurrency(&candidate);
7692    if candidate_concurrency != registered_concurrency {
7693        return Some(format!(
7694            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7695            registered.module_id, registered_concurrency, candidate_concurrency
7696        ));
7697    }
7698
7699    // control_ops live beside the manifest in the HELLO body, not inside
7700    // ModuleManifest, so a provides-only catalog.update cannot change them.
7701    None
7702}
7703
7704fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7705    manifest.provides.iter().any(is_routable_role)
7706}
7707
7708/// Returns the routable-provider concurrency subc should enforce for this manifest.
7709///
7710/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7711/// InternalService has no role-specific concurrency field, so it retains the
7712/// existing ModuleManaged default for backward compatibility.
7713fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7714    manifest
7715        .provides
7716        .iter()
7717        .find_map(|provider| match provider {
7718            ProviderRole::ToolProvider { concurrency, .. }
7719            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7720            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7721        })
7722        .unwrap_or(Concurrency::ModuleManaged)
7723}
7724
7725/// True when the manifest carries a ManagementSurface role whose concurrency
7726/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7727/// bytes because the typed manifest deliberately erases that distinction: the
7728/// default exists for wire compatibility, and this probe exists so the default
7729/// stays observable. Any parse irregularity returns false -- the caller only
7730/// logs, and a malformed body already failed registration upstream.
7731fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7732    let has_management_surface = manifest
7733        .provides
7734        .iter()
7735        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7736    if !has_management_surface {
7737        return false;
7738    }
7739    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7740        return false;
7741    };
7742    let Some(provides) = raw
7743        .get("manifest")
7744        .and_then(|manifest| manifest.get("provides"))
7745        .and_then(serde_json::Value::as_array)
7746    else {
7747        return false;
7748    };
7749    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7750    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7751    // against the management_surface_manifest_without_concurrency golden, not
7752    // recalled (the externally-tagged guess was this function's first bug).
7753    provides.iter().any(|role| {
7754        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7755            && role.get("concurrency").is_none()
7756    })
7757}
7758
7759fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7760    if peer_version != PROTOCOL_VERSION {
7761        return Err(format!(
7762            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7763        ));
7764    }
7765    Ok(PROTOCOL_VERSION)
7766}
7767
7768fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7769    Frame::build_with_version(
7770        response_version(frame),
7771        FrameType::Pong,
7772        frame.header.flags,
7773        0,
7774        0,
7775        frame.header.corr,
7776        Vec::new(),
7777    )
7778    .map_err(RouterError::FrameBuild)
7779}
7780
7781pub(crate) fn control_error_frame(
7782    frame: &Frame,
7783    code: &'static str,
7784    message: impl Into<String>,
7785) -> Result<Frame, RouterError> {
7786    control_error_body_frame(
7787        frame,
7788        ErrorBody {
7789            code: code.to_string(),
7790            message: message.into(),
7791            detail: None,
7792        },
7793    )
7794}
7795
7796fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7797    let body = serde_json::to_vec(&error).map_err(|err| {
7798        RouterError::backend(
7799            0,
7800            frame.header.corr,
7801            format!("failed to encode control ERROR: {err}"),
7802        )
7803    })?;
7804
7805    Frame::build_with_version(
7806        response_version(frame),
7807        FrameType::Error,
7808        control_flags(),
7809        0,
7810        0,
7811        frame.header.corr,
7812        body,
7813    )
7814    .map_err(RouterError::FrameBuild)
7815}
7816
7817fn control_response_body_frame<T: Serialize>(
7818    frame: &Frame,
7819    reply: &T,
7820    label: &'static str,
7821) -> Result<Frame, RouterError> {
7822    let body = serde_json::to_vec(reply).map_err(|err| {
7823        RouterError::backend(
7824            0,
7825            frame.header.corr,
7826            format!("failed to encode {label}: {err}"),
7827        )
7828    })?;
7829
7830    Frame::build_with_version(
7831        response_version(frame),
7832        FrameType::Response,
7833        control_flags(),
7834        0,
7835        0,
7836        frame.header.corr,
7837        body,
7838    )
7839    .map_err(RouterError::FrameBuild)
7840}
7841
7842/// Map a forwarding failure to the wire code a client sees.
7843///
7844/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7845/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7846/// chosen here decides whether a caller retries or gives up.
7847///
7848/// That makes attribution the load-bearing property, not merely having a code. A
7849/// permanent fault published as a retryable one produces a fleet-wide retry storm
7850/// against something that can never recover; a transient fault published as
7851/// permanent gives up on work that would have succeeded. Both look correct in a
7852/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7853/// the mapping per variant rather than merely asserting that some code exists.
7854///
7855/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7856/// two codes on the same side of the boundary passes it. Measured rather than
7857/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7858/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7859/// a test named for something else that happens to assert the string.
7860///
7861/// That accidental coverage is deliberately left alone rather than promoted to a
7862/// named test, because it guards a property this function does not promise.
7863/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7864/// specific code within a class, so identity is free to change and only the
7865/// partition is a contract. Splitting it out would assert a guarantee nothing
7866/// depends on — and a suite that promises more than the code does is the harder
7867/// thing to correct later, because the next reader cannot tell which assertions
7868/// are load-bearing.
7869///
7870/// Pin identity here the moment a consumer branches on a specific code.
7871fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7872    match err {
7873        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7874        ForwardingError::NoModuleConnection => "target_unavailable",
7875        ForwardingError::ModuleReloading { .. } => "module_reloading",
7876        ForwardingError::ClientRouteChannelExhausted { .. }
7877        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7878        ForwardingError::StaleModuleEndpoint
7879        | ForwardingError::UnknownReservation { .. }
7880        | ForwardingError::ConnectionClosing { .. }
7881        | ForwardingError::ClientEgressClosed { .. }
7882        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7883        // Only a swap candidate's registration can produce this, and it means
7884        // exactly what a second active HELLO for a live id means.
7885        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7886        ForwardingError::RelayCorrelationExhausted
7887        | ForwardingError::RouteOpenBuild(_)
7888        | ForwardingError::Poisoned => "forwarding_error",
7889    }
7890}
7891
7892fn response_version(frame: &Frame) -> u8 {
7893    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7894        frame.header.ver
7895    } else {
7896        PROTOCOL_VERSION
7897    }
7898}
7899
7900fn control_flags() -> Flags {
7901    Flags::new(false, Priority::Passive, false)
7902}
7903
7904/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7905/// channel. The target is the module (a client never saw the route), so this
7906/// takes the module path: delivered late rather than dropped when the module's
7907/// queue is momentarily full, and never closing its connection.
7908fn send_goodbye_target_best_effort(
7909    counters: &DaemonCounters,
7910    target: &GoodbyeTarget,
7911    context: &'static str,
7912) {
7913    let Ok(frame) = Frame::build_with_version(
7914        target.negotiated_ver,
7915        FrameType::Goodbye,
7916        control_flags(),
7917        target.channel,
7918        target.epoch,
7919        0,
7920        Vec::new(),
7921    ) else {
7922        return;
7923    };
7924    crate::forwarding::send_module_route_goodbye(
7925        counters,
7926        &target.sink,
7927        frame,
7928        target.module_id.as_deref(),
7929        context,
7930    );
7931}
7932
7933pub(crate) fn send_route_control_pushes(
7934    forwarding: &ForwardingTable,
7935    routes: Vec<EndpointRoute>,
7936    push: ClientControlPush,
7937) {
7938    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7939    for route in routes {
7940        let target = route.goodbye_target;
7941        if let Some((existing, channels)) = targets
7942            .iter_mut()
7943            .find(|(existing, _)| existing.connection_id == target.connection_id)
7944        {
7945            debug_assert_eq!(
7946                existing.negotiated_ver, target.negotiated_ver,
7947                "one connection cannot negotiate multiple frame versions"
7948            );
7949            if !channels.contains(&target.channel) {
7950                channels.push(target.channel);
7951            }
7952            continue;
7953        }
7954        let channel = target.channel;
7955        targets.push((target, vec![channel]));
7956    }
7957    for (target, mut channels) in targets {
7958        channels.sort_unstable();
7959        let mut push = push.clone();
7960        match &mut push {
7961            ClientControlPush::RouteClosing {
7962                channels: covered, ..
7963            }
7964            | ClientControlPush::RouteClosed {
7965                channels: covered, ..
7966            } => *covered = channels,
7967        }
7968        let body = match serde_json::to_vec(&push) {
7969            Ok(body) => body,
7970            Err(err) => {
7971                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7972                continue;
7973            }
7974        };
7975        let frame = match Frame::build_with_version(
7976            target.negotiated_ver,
7977            FrameType::Push,
7978            control_flags(),
7979            0,
7980            0,
7981            0,
7982            body.clone(),
7983        ) {
7984            Ok(frame) => frame,
7985            Err(err) => {
7986                warn!(
7987                    route_channel = target.channel,
7988                    error = %err,
7989                    "failed to build route lifecycle control PUSH frame"
7990                );
7991                continue;
7992            }
7993        };
7994        if let Err(err) = target.sink.try_send(frame) {
7995            if target.close_on_delivery_failure() {
7996                warn!(
7997                    target_connection_id = target.connection_id.get(),
7998                    route_channel = target.channel,
7999                    error = %err,
8000                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8001                );
8002                let _ = forwarding.escalate_client_delivery_failure(
8003                    target.connection_id,
8004                    target.channel,
8005                    target.epoch,
8006                    CloseReason::new(
8007                        "route_lifecycle_push_delivery_failed",
8008                        format!(
8009                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8010                            target.channel
8011                        ),
8012                    ),
8013                    crate::forwarding::UndeliveredFrame {
8014                        module_id: target.module_id.as_deref(),
8015                        sink: &target.sink,
8016                    },
8017                );
8018            }
8019        }
8020    }
8021}
8022
8023#[cfg(test)]
8024mod tests {
8025    mod activity_read {
8026        include!("activity_read_tests.rs");
8027    }
8028
8029    mod lifecycle_activity {
8030        include!("control_lifecycle_tests.rs");
8031    }
8032    #[cfg(unix)]
8033    #[tokio::test]
8034    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8035        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8036        let path = dir.join("subc.jsonc");
8037        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8038            "program":"/bin/sleep","args":["60"],"protocol":"none",
8039            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8040        }}}).to_string()).unwrap();
8041        let mut configured = crate::daemon_config::load(&path)
8042            .unwrap()
8043            .unwrap()
8044            .modules
8045            .pop()
8046            .unwrap();
8047        let registry = std::sync::Arc::new(crate::Registry::default());
8048        let handle = crate::SupervisorHandle::new();
8049        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8050            .with_handle(handle.clone());
8051        let module = supervisor
8052            .supervise_configured_with_health(
8053                configured.module_spec(),
8054                true,
8055                configured.health.clone(),
8056                None,
8057                configured.restart,
8058            )
8059            .unwrap();
8060        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8061        let before = module.status().unwrap().pid;
8062        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8063        configured.health.cadence = std::time::Duration::from_secs(3600);
8064        let health_only = handler
8065            .reconcile_supervised_modules(
8066                &supervisor,
8067                vec![configured.clone()],
8068                false,
8069                Principal::Direct,
8070            )
8071            .await
8072            .unwrap();
8073        assert!(
8074            health_only.0.changed_pending_reload.is_empty(),
8075            "health policy is already applied live"
8076        );
8077        assert_eq!(module.status().unwrap().pid, before);
8078        assert_eq!(
8079            module.configuration().unwrap().1.http,
8080            configured.health.http
8081        );
8082        configured.args = vec!["61".into()];
8083        let launch = handler
8084            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8085            .await
8086            .unwrap();
8087        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8088        assert_eq!(
8089            module.status().unwrap().pid,
8090            before,
8091            "a launch edit is stored until reload"
8092        );
8093        module.drain().await.unwrap();
8094    }
8095    use cortexkit_test_support::ScratchDir;
8096    use std::{
8097        collections::BTreeMap,
8098        fmt,
8099        path::PathBuf,
8100        sync::{Arc, Mutex},
8101        time::Duration,
8102    };
8103
8104    use serde_json::{json, Value};
8105    use subc_protocol::{
8106        manifest::{
8107            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8108            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8109        },
8110        session::HealthStatus,
8111        FrameType,
8112    };
8113
8114    use super::*;
8115    use crate::{
8116        forwarding::{DataRoute, DataRouteState},
8117        registry::ChannelState,
8118        router::FrameSink,
8119        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8120        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8121        RouteCtx, Router,
8122    };
8123    use tokio::{
8124        sync::mpsc,
8125        time::{sleep, Instant},
8126    };
8127    use tracing::{
8128        field::{Field, Visit},
8129        Event, Subscriber,
8130    };
8131    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8132
8133    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8134    ///
8135    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8136    /// is only populated for `tests/*.rs` integration test binaries -- this file
8137    /// compiles as part of the library target, which gets neither. This test's
8138    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8139    /// and the sibling binary lives two directories up at
8140    /// `<target-dir>/<profile>/fake-aft-stub`.
8141    ///
8142    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8143    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8144    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8145    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8146    /// `NotFound`, which reads as a broken test rather than an unbuilt
8147    /// dependency -- so state the cause and the remedy instead. Deliberately a
8148    /// panic and not a silent skip: a test that quietly passes when it could not
8149    /// run is worse than one that fails, because it reports health it never
8150    /// verified.
8151    fn fake_aft_stub_path() -> PathBuf {
8152        let mut path = std::env::current_exe().expect("current_exe available in tests");
8153        path.pop(); // .../deps/
8154        path.pop(); // .../<profile>/
8155        path.push(if cfg!(windows) {
8156            "fake-aft-stub.exe"
8157        } else {
8158            "fake-aft-stub"
8159        });
8160        assert!(
8161            path.exists(),
8162            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8163             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8164            path.display()
8165        );
8166        path
8167    }
8168
8169    /// Whether clients retry `code` in place: the predicate itself, never a copy
8170    /// of its set. A copied list breaks silently when a code is added to or
8171    /// removed from the real one, and a stale copy here would let exactly the
8172    /// failure this test exists to catch pass.
8173    fn client_retries(code: &str) -> bool {
8174        subc_protocol::error_codes::is_retryable_route_open(code)
8175    }
8176
8177    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8178    /// of failure is worse than publishing none. A permanent fault dressed as
8179    /// retryable makes every client in the fleet retry forever against something
8180    /// that cannot recover; a transient fault dressed as permanent abandons work
8181    /// that would have succeeded.
8182    ///
8183    /// Asserting "a code exists" cannot catch either, because the string is free
8184    /// to say anything. This enumerates every variant and pins which side of the
8185    /// retry boundary it lands on, so a new variant must be classified here
8186    /// deliberately rather than inheriting whichever arm it was appended to.
8187    #[test]
8188    fn retryability_of_forwarding_codes_matches_the_failure() {
8189        // Transient by nature: the target is booting, reloading, or its endpoint
8190        // was swapped mid-flight. Retrying is how these resolve.
8191        let transient = [
8192            ForwardingError::NoModuleConnection,
8193            ForwardingError::ModuleReloading {
8194                module_id: "m".into(),
8195            },
8196            ForwardingError::StaleModuleEndpoint,
8197            ForwardingError::UnknownReservation {
8198                client_channel: 1,
8199                module_channel: 1,
8200            },
8201            ForwardingError::ConnectionClosing {
8202                connection_id: ConnectionId::new(1),
8203            },
8204            ForwardingError::ClientEgressClosed {
8205                connection_id: ConnectionId::new(1),
8206            },
8207            ForwardingError::ModuleEgressUnavailable {
8208                connection_id: ConnectionId::new(1),
8209            },
8210        ];
8211        for err in transient {
8212            let code = forwarding_error_code(&err);
8213            assert!(
8214                client_retries(code),
8215                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8216            );
8217        }
8218
8219        // Not fixed by retrying. Channel and correlation exhaustion need the
8220        // caller to close routes, and a poisoned lock is a daemon that cannot
8221        // recover at all — the worst thing to advertise as retryable, since every
8222        // client would storm a daemon that will never answer.
8223        let permanent = [
8224            ForwardingError::ConnectionRoleConflict {
8225                connection_id: ConnectionId::new(1),
8226            },
8227            ForwardingError::ClientRouteChannelExhausted {
8228                connection_id: ConnectionId::new(1),
8229            },
8230            ForwardingError::ModuleRouteChannelExhausted {
8231                endpoint: ModuleEndpointId {
8232                    connection_id: ConnectionId::new(1),
8233                    generation: 1,
8234                },
8235            },
8236            ForwardingError::RelayCorrelationExhausted,
8237            ForwardingError::RouteOpenBuild("x".into()),
8238            ForwardingError::Poisoned,
8239        ];
8240        for err in permanent {
8241            let code = forwarding_error_code(&err);
8242            assert!(
8243                !client_retries(code),
8244                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8245            );
8246        }
8247    }
8248
8249    /// The principal is the daemon's answer to "who is calling", and modules
8250    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8251    /// plexus gates connector invocation. So a stamp is an authorization input in
8252    /// another process, not a label — and both possible answers SUCCEED, which is
8253    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8254    /// first-party capability to something that never proved it; a supervised one
8255    /// stamped `Direct` silently strips a module of capability it is entitled to.
8256    ///
8257    /// Neither shows up in a test that only checks the bind succeeded. Before this
8258    /// test the only coverage was accidental —
8259    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8260    /// stamped principal on its way past, so narrowing that wire-shape test to its
8261    /// stated subject would have deleted the last assertion on this value. It
8262    /// still asserts the stamp, which is now redundancy rather than the only
8263    /// guard: both fail under the same mutation, and this one names the reason.
8264    /// SCOPE: this handler's supervisor has spawned nothing, so
8265    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8266    /// is unreachable here. Both assertions below are refusals, and a mutant that
8267    /// refuses everything would satisfy them.
8268    ///
8269    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8270    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8271    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8272    /// at source rather than assumed, since a citation is a claim about another
8273    /// file and ages like one. Recorded because a harness that structurally
8274    /// cannot reach an arm reports "none" for that arm identically to one that
8275    /// covers it and found nothing.
8276    #[tokio::test]
8277    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8278        let handler = ControlHandler::default();
8279        let frame =
8280            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8281
8282        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8283        // any process holding the connection file. Nothing was proved, so nothing
8284        // may be granted beyond the unattested floor.
8285        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8286        assert_eq!(
8287            stamped,
8288            Principal::Direct,
8289            "a caller that proved nothing must not be stamped as a supervised module"
8290        );
8291
8292        // A claimed module_id with a nonce no supervised child was given is a
8293        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8294        // quietly demoted to Direct, or an impersonation attempt looks identical
8295        // to an ordinary unattested connection.
8296        let forged = handler
8297            .route_open_principal(
8298                &frame,
8299                Some(ConsumerIdentity {
8300                    module_id: "aft".to_string(),
8301                    launch_nonce: "not-a-real-nonce".to_string(),
8302                }),
8303            )
8304            .unwrap();
8305        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8306        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8307    }
8308
8309    /// The test above hands `route_open_principal` an identity it built itself,
8310    /// which proves the stamping rule and nothing about where the identity comes
8311    /// from. The real producer is a wire body, and the two are joined by a serde
8312    /// field name that nothing else asserts.
8313    ///
8314    /// That join fails quietly in one specific way: an unrecognised key is simply
8315    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8316    /// yields `None` and every supervised module silently drops to `Direct`.
8317    /// Capability-wise that is the safe direction, but it surfaces far from its
8318    /// cause — as a module mysteriously refused bash — and it would pass every
8319    /// test that builds its own input.
8320    ///
8321    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8322    /// would break every client the moment the daemon gains a field, trading a
8323    /// quiet demotion for a hard refusal on additive change. Asserting the join
8324    /// instead means a rename breaks a test here rather than the fleet.
8325    #[test]
8326    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8327        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8328        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8329        let ClientControlRequest::RouteOpen {
8330            consumer_identity, ..
8331        } = parsed
8332        else {
8333            panic!("route.open body must parse as RouteOpen");
8334        };
8335        assert_eq!(
8336            consumer_identity,
8337            Some(ConsumerIdentity {
8338                module_id: "aft".to_string(),
8339                launch_nonce: "n".to_string(),
8340            }),
8341            "the wire field name must reach the value route_open_principal reads"
8342        );
8343    }
8344
8345    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8346        ModuleManifest::builder(module_id, "0.1.0")
8347            .protocol_ver(protocol_ver)
8348            .provides(vec![ProviderRole::ToolProvider {
8349                tools: vec![Tool {
8350                    name: "read".to_string(),
8351                    description: None,
8352                    execution_mode: ExecutionMode::Pure,
8353                    schema: json!({"type": "object"}),
8354                }],
8355                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8356                concurrency: Concurrency::ModuleManaged,
8357                emits_push: true,
8358                sub_supervises: true,
8359            }])
8360            .build()
8361    }
8362
8363    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8364        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8365    }
8366
8367    fn hello_frame_with_control_ops(
8368        module_id: &str,
8369        protocol_ver: u8,
8370        corr: u64,
8371        control_ops: Option<Vec<String>>,
8372    ) -> Frame {
8373        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8374    }
8375
8376    fn hello_frame_with_nonce(
8377        module_id: &str,
8378        protocol_ver: u8,
8379        corr: u64,
8380        launch_nonce: Option<&str>,
8381    ) -> Frame {
8382        hello_frame_full(
8383            module_id,
8384            protocol_ver,
8385            corr,
8386            None,
8387            launch_nonce.map(ToOwned::to_owned),
8388        )
8389    }
8390
8391    fn hello_frame_full(
8392        module_id: &str,
8393        protocol_ver: u8,
8394        corr: u64,
8395        control_ops: Option<Vec<String>>,
8396        launch_nonce: Option<String>,
8397    ) -> Frame {
8398        let body = serde_json::to_vec(&ModuleHelloBody {
8399            manifest: manifest(module_id, protocol_ver),
8400            protocol_ver,
8401            control_ops,
8402            launch_nonce,
8403        })
8404        .unwrap();
8405        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8406    }
8407
8408    fn hello_frame_with_rich_ui(
8409        module_id: &str,
8410        protocol_ver: u8,
8411        corr: u64,
8412        launch_nonce: Option<&str>,
8413        rich_ui: Value,
8414    ) -> Frame {
8415        let mut body = serde_json::to_value(ModuleHelloBody {
8416            manifest: manifest(module_id, protocol_ver),
8417            protocol_ver,
8418            control_ops: None,
8419            launch_nonce: launch_nonce.map(str::to_string),
8420        })
8421        .unwrap();
8422        body["manifest"]["rich_ui"] = rich_ui;
8423        Frame::build(
8424            FrameType::Hello,
8425            control_flags(),
8426            0,
8427            0,
8428            corr,
8429            serde_json::to_vec(&body).unwrap(),
8430        )
8431        .unwrap()
8432    }
8433
8434    fn non_routable_hello_frame_with_control_ops(
8435        module_id: &str,
8436        corr: u64,
8437        control_ops: Option<Vec<String>>,
8438    ) -> Frame {
8439        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8440        manifest.provides.clear();
8441        let body = serde_json::to_vec(&ModuleHelloBody {
8442            manifest,
8443            protocol_ver: PROTOCOL_VERSION,
8444            control_ops,
8445            launch_nonce: None,
8446        })
8447        .unwrap();
8448        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8449    }
8450
8451    fn capability_grammar_hello_frame(
8452        capabilities: Value,
8453        runtime_computed: Option<Value>,
8454        corr: u64,
8455    ) -> Frame {
8456        let mut body = serde_json::to_value(ModuleHelloBody {
8457            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8458            protocol_ver: PROTOCOL_VERSION,
8459            control_ops: None,
8460            launch_nonce: None,
8461        })
8462        .expect("HELLO body serializes");
8463        body["manifest"]["capabilities"] = capabilities;
8464        if let Some(runtime_computed) = runtime_computed {
8465            body["runtime_computed"] = runtime_computed;
8466        }
8467        Frame::build(
8468            FrameType::Hello,
8469            control_flags(),
8470            0,
8471            0,
8472            corr,
8473            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8474        )
8475        .expect("HELLO frame builds")
8476    }
8477
8478    fn channel_request(channel: u16, corr: u64) -> Frame {
8479        Frame::build(
8480            FrameType::Request,
8481            Flags::new(true, Priority::Interactive, false),
8482            channel,
8483            0,
8484            corr,
8485            b"opaque".to_vec(),
8486        )
8487        .unwrap()
8488    }
8489
8490    fn route_ctx(
8491        connection_id: ConnectionId,
8492    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8493        let (tx, rx) = mpsc::channel(8);
8494        (
8495            RouteCtx {
8496                connection_id,
8497                egress: FrameSink::new(tx),
8498            },
8499            rx,
8500        )
8501    }
8502
8503    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8504        serde_json::from_slice(&frame.body).unwrap()
8505    }
8506
8507    /// Register a module over a connection that has a sink and return the
8508    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8509    /// module's own sink rather than returning it as a reply, so the ack is
8510    /// taken off `rx` here and whatever the test reads next is what followed it.
8511    async fn hello_via_sink(
8512        handler: &ControlHandler,
8513        ctx: &RouteCtx,
8514        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8515        hello: Frame,
8516    ) -> Frame {
8517        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8518        assert!(
8519            replies.is_empty(),
8520            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8521        );
8522        let ack = rx
8523            .try_recv()
8524            .expect("HELLO_ACK is queued on the module sink")
8525            .frame;
8526        assert_eq!(ack.header.ty, FrameType::HelloAck);
8527        ack
8528    }
8529
8530    fn parse_error(frame: &Frame) -> Value {
8531        serde_json::from_slice(&frame.body).unwrap()
8532    }
8533
8534    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8535        serde_json::from_slice(&frame.body).unwrap()
8536    }
8537
8538    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8539        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8540            route_channel,
8541            route_epoch: 0,
8542            kind,
8543        })
8544        .unwrap();
8545        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8546    }
8547
8548    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8549        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8550            module_id: module_id.to_string(),
8551        })
8552        .unwrap();
8553        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8554    }
8555
8556    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8557        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8558    }
8559
8560    fn route_open_frame_with_consumer_capabilities(
8561        corr: u64,
8562        module_id: &str,
8563        project_root: ScratchDir,
8564        consumer_capabilities: Option<Vec<String>>,
8565    ) -> Frame {
8566        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8567            target: RouteTarget::ToolProvider {
8568                module_id: module_id.to_string(),
8569            },
8570            identity: BindIdentity::new(
8571                project_root.path().to_path_buf(),
8572                "unit".to_string(),
8573                "session".to_string(),
8574            ),
8575            consumer_identity: None,
8576            consumer_capabilities,
8577            role_versions: None,
8578            admission_facts: None,
8579            scope: None,
8580        })
8581        .unwrap();
8582        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8583    }
8584
8585    fn route_open_frame_with_role_versions(
8586        corr: u64,
8587        module_id: &str,
8588        project_root: ScratchDir,
8589        role_versions: Option<BTreeMap<String, String>>,
8590    ) -> Frame {
8591        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8592            target: RouteTarget::ToolProvider {
8593                module_id: module_id.to_string(),
8594            },
8595            identity: BindIdentity::new(
8596                project_root.path().to_path_buf(),
8597                "unit".to_string(),
8598                format!("session-{corr}"),
8599            ),
8600            consumer_identity: None,
8601            consumer_capabilities: None,
8602            role_versions,
8603            admission_facts: None,
8604            scope: None,
8605        })
8606        .unwrap();
8607        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8608    }
8609
8610    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8611        entries
8612            .iter()
8613            .map(|(role, version)| (role.to_string(), version.to_string()))
8614            .collect()
8615    }
8616
8617    fn route_open_frame_with_admission_facts(
8618        corr: u64,
8619        module_id: &str,
8620        project_root: ScratchDir,
8621        consumer_identity: Option<subc_control::ConsumerIdentity>,
8622        facts: Option<Value>,
8623    ) -> Frame {
8624        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8625            target: RouteTarget::ToolProvider {
8626                module_id: module_id.to_string(),
8627            },
8628            identity: BindIdentity::new(
8629                project_root.path().to_path_buf(),
8630                "unit".to_string(),
8631                format!("session-{corr}"),
8632            ),
8633            consumer_identity,
8634            consumer_capabilities: None,
8635            role_versions: None,
8636            admission_facts: facts,
8637            scope: None,
8638        })
8639        .unwrap();
8640        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8641    }
8642
8643    #[derive(Clone, Default)]
8644    struct EventCapture {
8645        events: Arc<Mutex<Vec<CapturedEvent>>>,
8646    }
8647
8648    #[derive(Clone, Debug)]
8649    struct CapturedEvent {
8650        target: String,
8651        level: tracing::Level,
8652        fields: BTreeMap<String, String>,
8653    }
8654
8655    impl EventCapture {
8656        fn events(&self) -> Vec<CapturedEvent> {
8657            self.events.lock().unwrap().clone()
8658        }
8659    }
8660
8661    impl<S> Layer<S> for EventCapture
8662    where
8663        S: Subscriber,
8664    {
8665        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8666            let mut visitor = EventFieldVisitor::default();
8667            event.record(&mut visitor);
8668            self.events.lock().unwrap().push(CapturedEvent {
8669                target: event.metadata().target().to_string(),
8670                level: *event.metadata().level(),
8671                fields: visitor.fields,
8672            });
8673        }
8674    }
8675
8676    #[derive(Default)]
8677    struct EventFieldVisitor {
8678        fields: BTreeMap<String, String>,
8679    }
8680
8681    impl Visit for EventFieldVisitor {
8682        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8683            self.fields
8684                .insert(field.name().to_string(), format!("{value:?}"));
8685        }
8686    }
8687
8688    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8689        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8690            status,
8691            detail: Some("warming".to_string()),
8692            metrics: Some(json!({"queue_depth": 3})),
8693        })
8694        .unwrap();
8695        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8696    }
8697
8698    fn route_bind_ack(corr: u64) -> Frame {
8699        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8700        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8701    }
8702
8703    fn unique_project_root(label: &str) -> ScratchDir {
8704        ScratchDir::new(label)
8705    }
8706
8707    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8708        match parse_route_poll(frame) {
8709            ClientControlResponse::RoutePoll {
8710                status: None,
8711                live: Some(live),
8712                ..
8713            } => assert_eq!(live, expected_live),
8714            other => panic!("unexpected route.poll response: {other:?}"),
8715        }
8716    }
8717
8718    fn bind_liveness_route(
8719        registry: &Registry,
8720        forwarding: &ForwardingTable,
8721        module_id: &str,
8722    ) -> (RouteCtx, u16, u32) {
8723        let module_connection = ConnectionId::new(101);
8724        let client_connection = ConnectionId::new(202);
8725        let registration = registry
8726            .register_with_control_ops(
8727                manifest(module_id, PROTOCOL_VERSION),
8728                PROTOCOL_VERSION,
8729                module_connection,
8730                module_baseline_control_ops(),
8731            )
8732            .unwrap();
8733        let (module_tx, _module_rx) = mpsc::channel(8);
8734        let endpoint = forwarding
8735            .register_module_connection(
8736                module_connection,
8737                module_id.to_string(),
8738                PROTOCOL_VERSION,
8739                manifest_concurrency(&registration.manifest),
8740                FrameSink::new(module_tx),
8741            )
8742            .unwrap();
8743        let (client_ctx, _client_rx) = route_ctx(client_connection);
8744        let pending = forwarding
8745            .begin_route_bind_relay_for_test(
8746                client_connection,
8747                client_ctx.egress.clone(),
8748                1,
8749                module_id,
8750            )
8751            .unwrap();
8752        assert_eq!(pending.endpoint, endpoint);
8753        let route_channel = pending.client_channel;
8754        let route_epoch = pending.client_epoch;
8755        forwarding
8756            .complete_pending_relay(
8757                module_connection,
8758                pending.corr,
8759                RouteBindRelayOutcome::Accepted,
8760            )
8761            .unwrap();
8762        (client_ctx, route_channel, route_epoch)
8763    }
8764
8765    struct FakeProcessLiveness {
8766        live: Option<bool>,
8767    }
8768
8769    impl ModuleProcessLiveness for FakeProcessLiveness {
8770        fn process_live(&self, _module_id: &str) -> Option<bool> {
8771            self.live
8772        }
8773    }
8774
8775    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8776    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8777    {
8778        let registry = Arc::new(Registry::default());
8779        let supervisor_handle = SupervisorHandle::new();
8780        let supervisor = Supervisor::new_for_test(
8781            Arc::clone(&registry),
8782            RestartPolicy::new(1, Duration::from_millis(10)),
8783        )
8784        .with_handle(supervisor_handle.clone());
8785        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8786        let module = supervisor
8787            .spawn(ModuleSpec {
8788                module_id: "stderr-tail-wire".to_string(),
8789                program: fake_aft_stub_path(),
8790                args: Vec::new(),
8791                env: vec![
8792                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8793                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8794                ],
8795                reserved: false,
8796                reserved_prefixes: Vec::new(),
8797                protocol: ModuleProtocol::Subc,
8798                overlap: Default::default(),
8799            })
8800            .unwrap();
8801
8802        let deadline = Instant::now() + Duration::from_secs(5);
8803        loop {
8804            let tail = module.stderr_tail(None, None);
8805            if tail
8806                .entries
8807                .iter()
8808                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8809                && tail.entries.iter().any(|entry| {
8810                    matches!(
8811                        entry,
8812                        TailEntry::Line {
8813                            truncated: true,
8814                            ..
8815                        }
8816                    )
8817                })
8818            {
8819                break;
8820            }
8821            assert!(
8822                Instant::now() < deadline,
8823                "module did not produce a truncated line and restart boundary: {tail:?}"
8824            );
8825            sleep(Duration::from_millis(10)).await;
8826        }
8827
8828        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8829        let request = ClientControlRequest::SupervisorStderrTail {
8830            module_id: "stderr-tail-wire".to_string(),
8831            max_lines: None,
8832            max_bytes: None,
8833        };
8834        let frame = Frame::build(
8835            FrameType::Request,
8836            control_flags(),
8837            0,
8838            0,
8839            1,
8840            serde_json::to_vec(&request).unwrap(),
8841        )
8842        .unwrap();
8843        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8844        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8845        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8846            serde_json::from_slice(&responses[0].body).unwrap()
8847        else {
8848            panic!("expected supervisor.stderr_tail response");
8849        };
8850
8851        assert!(
8852            tail.entries
8853                .iter()
8854                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8855            "the control response lost the restart boundary"
8856        );
8857        let Some(StderrTailEntry::Line {
8858            text,
8859            truncated,
8860            at_ms,
8861        }) = tail.entries.iter().find(|entry| {
8862            matches!(
8863                entry,
8864                StderrTailEntry::Line {
8865                    truncated: true,
8866                    ..
8867                }
8868            )
8869        })
8870        else {
8871            panic!("the control response lost the truncated line");
8872        };
8873        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8874        assert!(*truncated);
8875        assert!(
8876            at_ms.is_some(),
8877            "the control response lost the line's capture time"
8878        );
8879    }
8880
8881    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8882    /// read done on the worker thread would stall every other task until it
8883    /// finished; the read must run off the worker so this test's own task keeps
8884    /// running while the read is paused.
8885    #[tokio::test(flavor = "current_thread")]
8886    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8887        let dir = ScratchDir::new("terminals-off-worker");
8888        let journal_path = dir.join("terminals.jsonl");
8889        let registry = Arc::new(Registry::default());
8890        let supervisor_handle = SupervisorHandle::new();
8891        let supervisor =
8892            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8893                .with_handle(supervisor_handle.clone())
8894                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8895        let module = supervisor
8896            .spawn(ModuleSpec {
8897                module_id: "terminal-off-worker".to_string(),
8898                program: fake_aft_stub_path(),
8899                args: Vec::new(),
8900                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8901                reserved: false,
8902                reserved_prefixes: Vec::new(),
8903                protocol: ModuleProtocol::Subc,
8904                overlap: Default::default(),
8905            })
8906            .unwrap();
8907        let deadline = Instant::now() + Duration::from_secs(5);
8908        while module.terminal_history().entries.len() != 2 {
8909            assert!(Instant::now() < deadline, "module did not record two exits");
8910            sleep(Duration::from_millis(10)).await;
8911        }
8912
8913        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8914        let handler =
8915            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8916        let frame = Frame::build(
8917            FrameType::Request,
8918            control_flags(),
8919            0,
8920            0,
8921            1,
8922            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8923                module_id: "terminal-off-worker".to_string(),
8924            })
8925            .unwrap(),
8926        )
8927        .unwrap();
8928        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8929        let spawned_at = std::time::Instant::now();
8930        let read = tokio::spawn({
8931            let handler = Arc::clone(&handler);
8932            async move { handler.handle_control_frame(&ctx, frame).await }
8933        });
8934        // Waiting for the pause from a blocking thread keeps this task pending,
8935        // so the runtime's single worker is free to run the read task.
8936        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8937            .await
8938            .unwrap()
8939            .expect("the history read reached its pause");
8940        let elapsed = spawned_at.elapsed();
8941        assert!(
8942            elapsed < Duration::from_secs(2) && !read.is_finished(),
8943            "this task could not run while the history read was paused \
8944             (resumed after {elapsed:?}, read finished: {})",
8945            read.is_finished()
8946        );
8947
8948        drop(release);
8949        let responses = read.await.unwrap().unwrap();
8950        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8951        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8952            panic!("expected supervisor.terminals response");
8953        };
8954        assert_eq!(terminals.entries.len(), 2);
8955        assert_eq!(terminals.journal_skipped_lines, 0);
8956        assert_eq!(terminals.journal_read_errors, 0);
8957    }
8958
8959    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8960    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8961        let registry = Arc::new(Registry::default());
8962        let supervisor_handle = SupervisorHandle::new();
8963        let supervisor =
8964            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8965                .with_handle(supervisor_handle.clone());
8966        let module = supervisor
8967            .spawn(ModuleSpec {
8968                module_id: "terminal-golden".to_string(),
8969                program: fake_aft_stub_path(),
8970                args: Vec::new(),
8971                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8972                reserved: false,
8973                reserved_prefixes: Vec::new(),
8974                protocol: ModuleProtocol::Subc,
8975                overlap: Default::default(),
8976            })
8977            .unwrap();
8978
8979        let deadline = Instant::now() + Duration::from_secs(5);
8980        while module.terminal_history().entries.len() != 2 {
8981            assert!(
8982                Instant::now() < deadline,
8983                "module did not retain two terminal exits: {:?}",
8984                module.terminal_history()
8985            );
8986            sleep(Duration::from_millis(10)).await;
8987        }
8988
8989        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8990        let request = ClientControlRequest::SupervisorTerminals {
8991            module_id: "terminal-golden".to_string(),
8992        };
8993        let frame = Frame::build(
8994            FrameType::Request,
8995            control_flags(),
8996            0,
8997            0,
8998            1,
8999            serde_json::to_vec(&request).unwrap(),
9000        )
9001        .unwrap();
9002        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9003        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9004        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9005        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9006            panic!("expected supervisor.terminals response");
9007        };
9008        assert_eq!(terminals.entries.len(), 2);
9009        assert_eq!(terminals.dropped, 0);
9010
9011        let mut rendered = serde_json::to_value(response).unwrap();
9012        // Wall-clock fields are the observation contract, but not stable fixture
9013        // bytes; normalize only them after the real handler has shaped the response.
9014        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9015        for (index, entry) in rendered["entries"]
9016            .as_array_mut()
9017            .expect("terminal response entries array")
9018            .iter_mut()
9019            .enumerate()
9020        {
9021            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9022        }
9023
9024        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9025            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9026        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9027        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9028            std::fs::write(&golden_path, &serialized).unwrap();
9029        }
9030        let expected: Value =
9031            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9032        assert_eq!(rendered, expected);
9033    }
9034
9035    #[test]
9036    fn hello_registers_manifest_and_returns_ack() {
9037        let registry = Arc::new(Registry::default());
9038        let handler = ControlHandler::new(Arc::clone(&registry));
9039        let conn = ConnectionId::new(1);
9040
9041        let responses = handler
9042            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9043            .unwrap();
9044
9045        assert_eq!(responses.len(), 1);
9046        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9047        assert_eq!(responses[0].header.channel, 0);
9048        assert_eq!(responses[0].header.corr, 7);
9049        let ack = parse_ack(&responses[0]);
9050        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9051        assert!(ack
9052            .subc_capabilities
9053            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9054        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9055        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9056        assert!(ack
9057            .subc_ops
9058            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9059        assert!(ack
9060            .subc_ops
9061            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9062
9063        let registration = registry.get_module("aft").unwrap().unwrap();
9064        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9065        assert_eq!(registration.state, ChannelState::Active);
9066        assert_eq!(registration.connection_id, conn);
9067        assert_eq!(registration.control_ops, module_baseline_control_ops());
9068    }
9069
9070    #[test]
9071    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9072        let invalid_identifiers = [
9073            ("case_change", "credentials-Provider/v1"),
9074            ("leading_zero", "credentials-provider/v01"),
9075            ("trailing_hyphen", "credentials-provider-/v1"),
9076            ("consecutive_hyphens", "credentials--provider/v1"),
9077            ("uppercase", "Credentials-provider/v1"),
9078            ("missing_v", "credentials-provider/1"),
9079            ("whitespace", "credentials provider/v1"),
9080            ("zero_version", "credentials-provider/v0"),
9081            ("out_of_range_version", "credentials-provider/v4294967296"),
9082            (
9083                "overlength_name",
9084                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9085            ),
9086        ];
9087        let mut cases = invalid_identifiers
9088            .into_iter()
9089            .map(|(name, identifier)| {
9090                (
9091                    format!("identifier_{name}"),
9092                    "capabilities.provides[0]".to_string(),
9093                    identifier.to_string(),
9094                    json!({ "provides": [identifier] }),
9095                    None,
9096                )
9097            })
9098            .collect::<Vec<_>>();
9099        cases.extend([
9100            (
9101                "unknown_need".to_string(),
9102                "capabilities.requires[0].need".to_string(),
9103                "deferred".to_string(),
9104                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9105                None,
9106            ),
9107            (
9108                "duplicate_provides".to_string(),
9109                "capabilities.provides[1]".to_string(),
9110                "credentials-provider/v1".to_string(),
9111                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9112                None,
9113            ),
9114            (
9115                "duplicate_must_never_reach".to_string(),
9116                "capabilities.must_never_reach[1]".to_string(),
9117                "credentials-provider/v1".to_string(),
9118                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9119                None,
9120            ),
9121            (
9122                "duplicate_requires_same_need".to_string(),
9123                "capabilities.requires[1]".to_string(),
9124                "credentials-provider/v1".to_string(),
9125                json!({ "requires": [
9126                    { "capability": "credentials-provider/v1", "need": "required" },
9127                    { "capability": "credentials-provider/v1", "need": "required" }
9128                ] }),
9129                None,
9130            ),
9131            (
9132                "duplicate_requires_conflicting_need".to_string(),
9133                "capabilities.requires[1]".to_string(),
9134                "credentials-provider/v1".to_string(),
9135                json!({ "requires": [
9136                    { "capability": "credentials-provider/v1", "need": "required" },
9137                    { "capability": "credentials-provider/v1", "need": "optional" }
9138                ] }),
9139                None,
9140            ),
9141            (
9142                "capabilities_root_pointer".to_string(),
9143                "runtime_computed[0]".to_string(),
9144                "/capabilities".to_string(),
9145                json!({}),
9146                Some(json!(["/capabilities"])),
9147            ),
9148            (
9149                "capabilities_descendant_pointer".to_string(),
9150                "runtime_computed[0]".to_string(),
9151                "/capabilities/provides".to_string(),
9152                json!({}),
9153                Some(json!(["/capabilities/provides"])),
9154            ),
9155            (
9156                "malformed_pointer_without_leading_slash".to_string(),
9157                "runtime_computed[0]".to_string(),
9158                "capabilities".to_string(),
9159                json!({}),
9160                Some(json!(["capabilities"])),
9161            ),
9162            (
9163                "malformed_pointer_escape".to_string(),
9164                "runtime_computed[0]".to_string(),
9165                "/roles/~2/tools".to_string(),
9166                json!({}),
9167                Some(json!(["/roles/~2/tools"])),
9168            ),
9169            (
9170                "unknown_capabilities_field".to_string(),
9171                "capabilities.future".to_string(),
9172                "<array>".to_string(),
9173                json!({ "future": [] }),
9174                None,
9175            ),
9176        ]);
9177
9178        for (index, (name, field, value, capabilities, runtime_computed)) in
9179            cases.into_iter().enumerate()
9180        {
9181            let registry = Arc::new(Registry::default());
9182            let handler = ControlHandler::new(Arc::clone(&registry));
9183            let response = handler
9184                .handle_control(
9185                    ConnectionId::new((index + 1) as u64),
9186                    capability_grammar_hello_frame(
9187                        capabilities,
9188                        runtime_computed,
9189                        index as u64 + 1,
9190                    ),
9191                )
9192                .expect("invalid HELLO returns a refusal");
9193
9194            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9195            let error = parse_error(&response[0]);
9196            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9197            let message = error["message"]
9198                .as_str()
9199                .expect("error message is a string");
9200            assert!(
9201                message.contains(&field),
9202                "{name}: field missing from {message}"
9203            );
9204            assert!(
9205                message.contains(&value),
9206                "{name}: value missing from {message}"
9207            );
9208            assert_eq!(
9209                registry
9210                    .active_registration_count()
9211                    .expect("registry reads"),
9212                0,
9213                "{name}: refused HELLO must not create a catalog entry"
9214            );
9215        }
9216    }
9217
9218    #[test]
9219    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9220        let registry = Arc::new(Registry::default());
9221        let handler = ControlHandler::new(Arc::clone(&registry));
9222        let capabilities = json!({
9223            "provides": ["credentials-provider/v1"],
9224            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9225            "must_never_reach": ["federation-transport/v1"]
9226        });
9227        let response = handler
9228            .handle_control(
9229                ConnectionId::new(99),
9230                capability_grammar_hello_frame(
9231                    capabilities.clone(),
9232                    Some(json!(["/roles/0/tools"])),
9233                    99,
9234                ),
9235            )
9236            .expect("valid HELLO registers");
9237        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9238
9239        let request = Frame::build(
9240            FrameType::Request,
9241            control_flags(),
9242            0,
9243            0,
9244            100,
9245            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9246                .expect("catalog request serializes"),
9247        )
9248        .expect("catalog request frame builds");
9249        let response = handler
9250            .handle_catalog_list(request, None)
9251            .expect("catalog list succeeds");
9252        let ClientControlResponse::CatalogList { modules, .. } =
9253            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9254        else {
9255            panic!("catalog request must return catalog.list");
9256        };
9257        assert_eq!(modules.len(), 1);
9258        assert_eq!(
9259            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9260            capabilities
9261        );
9262    }
9263
9264    #[test]
9265    fn catalog_list_mirrors_management_operation_description() {
9266        let registry = Arc::new(Registry::default());
9267        let handler = ControlHandler::new(Arc::clone(&registry));
9268        let description = "List managed records and return their identifiers and metadata.";
9269        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9270        manifest.provides = vec![ProviderRole::ManagementSurface {
9271            operations: vec![ManagementOperation {
9272                name: "records.list".to_string(),
9273                kind: ManagementOperationKind::Query,
9274                description: Some(description.to_string()),
9275            }],
9276            config_schema: json!({"type": "object"}),
9277            observability: vec![ObservabilitySurface {
9278                name: "records.stats".to_string(),
9279                kind: ObservabilityKind::Snapshot,
9280            }],
9281            identity_scope: vec![IdentityScope::Project],
9282            concurrency: Concurrency::ModuleManaged,
9283        }];
9284        registry
9285            .register_with_control_ops(
9286                manifest,
9287                PROTOCOL_VERSION,
9288                ConnectionId::new(99),
9289                Vec::new(),
9290            )
9291            .expect("described management manifest registers");
9292
9293        let request = Frame::build(
9294            FrameType::Request,
9295            control_flags(),
9296            0,
9297            0,
9298            100,
9299            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9300                .expect("catalog request serializes"),
9301        )
9302        .expect("catalog request frame builds");
9303        let response = handler
9304            .handle_catalog_list(request, None)
9305            .expect("catalog list succeeds");
9306        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9307        assert_eq!(
9308            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9309            "catalog.list must preserve the declared operation description verbatim"
9310        );
9311    }
9312
9313    #[test]
9314    fn catalog_list_mirrors_registered_operation_delivery() {
9315        let registry = Arc::new(Registry::default());
9316        let handler = ControlHandler::new(Arc::clone(&registry));
9317        let delivery =
9318            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9319        let mut body = serde_json::to_value(ModuleHelloBody {
9320            manifest: manifest("watch-management", PROTOCOL_VERSION),
9321            protocol_ver: PROTOCOL_VERSION,
9322            control_ops: None,
9323            launch_nonce: None,
9324        })
9325        .unwrap();
9326        body["manifest"]["provides"] = json!([{
9327            "role": "management_surface",
9328            "operations": [
9329                {"name": "status", "kind": "query"},
9330                {"name": "future", "kind": "query"},
9331                {"name": "reset", "kind": "mutate"}
9332            ],
9333            "config_schema": {}, "observability": [], "identity_scope": []
9334        }]);
9335        body["manifest"]["operation_delivery"] = delivery.clone();
9336        let hello = Frame::build(
9337            FrameType::Hello,
9338            control_flags(),
9339            0,
9340            0,
9341            99,
9342            serde_json::to_vec(&body).unwrap(),
9343        )
9344        .unwrap();
9345        let response = handler
9346            .handle_control(ConnectionId::new(99), hello)
9347            .unwrap();
9348        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9349
9350        let (_, registrations) = registry.list_modules().unwrap();
9351        let registered = &registrations[0].manifest;
9352        assert!(registered.is_watch_operation("status"));
9353        assert!(!registered.is_watch_operation("future"));
9354        assert!(!registered.is_watch_operation("reset"));
9355        assert!(!registered.is_watch_operation("missing"));
9356
9357        let request = Frame::build(
9358            FrameType::Request,
9359            control_flags(),
9360            0,
9361            0,
9362            100,
9363            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9364        )
9365        .unwrap();
9366        let response = handler.handle_catalog_list(request, None).unwrap();
9367        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9368        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9369        let ClientControlResponse::CatalogList { modules, .. } =
9370            serde_json::from_slice(&response[0].body).unwrap()
9371        else {
9372            panic!("expected catalog.list");
9373        };
9374        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9375    }
9376
9377    #[test]
9378    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9379        let registry = Arc::new(Registry::default());
9380        let supervisor = SupervisorHandle::new();
9381        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9382        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9383        let descriptor = json!({
9384            "schema": "ckios.richui.manifest/1",
9385            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9386            "bytes": 4096,
9387            "chunks": 4
9388        });
9389        let connection = ConnectionId::new(201);
9390        let hello = handler
9391            .handle_control(
9392                connection,
9393                hello_frame_with_rich_ui(
9394                    "vault",
9395                    PROTOCOL_VERSION,
9396                    201,
9397                    Some("the-real-nonce"),
9398                    descriptor.clone(),
9399                ),
9400            )
9401            .unwrap();
9402        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9403
9404        let request =
9405            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9406        let provides = registry
9407            .get_module_by_connection(connection)
9408            .unwrap()
9409            .unwrap()
9410            .manifest
9411            .provides;
9412        handler
9413            .handle_catalog_update(connection, request, provides, None, Some(false))
9414            .unwrap();
9415
9416        let request = Frame::build(
9417            FrameType::Request,
9418            control_flags(),
9419            0,
9420            0,
9421            203,
9422            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9423        )
9424        .unwrap();
9425        let response = handler.handle_catalog_list(request, None).unwrap();
9426        let ClientControlResponse::CatalogList { modules, .. } =
9427            serde_json::from_slice(&response[0].body).unwrap()
9428        else {
9429            panic!("expected catalog.list");
9430        };
9431        assert_eq!(
9432            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9433            descriptor
9434        );
9435        assert!(modules[0].nonce_verified);
9436    }
9437
9438    fn hello_with_role_declarations(declarations: Value) -> Frame {
9439        let mut body = serde_json::to_value(ModuleHelloBody {
9440            manifest: manifest("role-test", PROTOCOL_VERSION),
9441            protocol_ver: PROTOCOL_VERSION,
9442            control_ops: None,
9443            launch_nonce: None,
9444        })
9445        .unwrap();
9446        body["manifest"]["role_declarations"] = declarations;
9447        Frame::build(
9448            FrameType::Hello,
9449            control_flags(),
9450            0,
9451            0,
9452            301,
9453            serde_json::to_vec(&body).unwrap(),
9454        )
9455        .unwrap()
9456    }
9457
9458    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9459        let request = Frame::build(
9460            FrameType::Request,
9461            control_flags(),
9462            0,
9463            0,
9464            302,
9465            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9466        )
9467        .unwrap();
9468        let response = handler.handle_catalog_list(request, None).unwrap();
9469        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9470        let ClientControlResponse::CatalogList { mut modules, .. } =
9471            serde_json::from_slice(&response[0].body).unwrap()
9472        else {
9473            panic!("expected catalog.list");
9474        };
9475        (raw["modules"][0].clone(), modules.remove(0))
9476    }
9477
9478    #[test]
9479    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9480        let registry = Arc::new(Registry::default());
9481        let handler = ControlHandler::new(Arc::clone(&registry));
9482        let declarations = json!({
9483            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9484            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9485        });
9486        let connection = ConnectionId::new(301);
9487        let response = handler
9488            .handle_control(
9489                connection,
9490                hello_with_role_declarations(declarations.clone()),
9491            )
9492            .unwrap();
9493        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9494        let registered = registry
9495            .get_module_by_connection(connection)
9496            .unwrap()
9497            .unwrap()
9498            .manifest;
9499        // Compared as JSON values, not bytes: object key order is not part of
9500        // the relay's promise, and workspace builds unify serde_json's
9501        // preserve_order feature, which changes the order a literal keeps.
9502        assert_eq!(
9503            serde_json::to_value(&registered.role_declarations).unwrap(),
9504            declarations
9505        );
9506        let (raw, entry) = role_declarations_catalog(&handler);
9507        assert_eq!(raw["role_declarations"], declarations);
9508        assert_eq!(entry.role_declarations, registered.role_declarations);
9509
9510        let request =
9511            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9512        handler
9513            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9514            .unwrap();
9515        let (raw, entry) = role_declarations_catalog(&handler);
9516        assert_eq!(raw["role_declarations"], declarations);
9517        assert_eq!(entry.role_declarations, registered.role_declarations);
9518    }
9519
9520    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9521        let registry = Arc::new(Registry::default());
9522        let handler = ControlHandler::new(Arc::clone(&registry));
9523        let capture = EventCapture::default();
9524        let _subscriber =
9525            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9526        let response = handler
9527            .handle_control(
9528                ConnectionId::new(301),
9529                hello_with_role_declarations(declarations),
9530            )
9531            .unwrap();
9532        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9533        assert!(registry
9534            .get_module_by_connection(ConnectionId::new(301))
9535            .unwrap()
9536            .unwrap()
9537            .manifest
9538            .role_declarations
9539            .is_empty());
9540        let (raw, entry) = role_declarations_catalog(&handler);
9541        assert!(raw.get("role_declarations").is_none());
9542        assert!(entry.role_declarations.is_empty());
9543        let warnings: Vec<_> = capture
9544            .events()
9545            .into_iter()
9546            .filter(|event| {
9547                event.fields.get("message").map(String::as_str)
9548                    == Some("dropping whole role_declarations map from HELLO")
9549            })
9550            .collect();
9551        assert_eq!(warnings.len(), 1);
9552        assert_eq!(warnings[0].level, tracing::Level::WARN);
9553        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9554        assert_eq!(warnings[0].fields["reason"], reason);
9555    }
9556
9557    #[test]
9558    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9559        for value in [
9560            json!(null),
9561            json!(false),
9562            json!(0),
9563            json!("opaque"),
9564            json!([]),
9565        ] {
9566            assert_hello_drops_role_declarations_with_warning(
9567                json!({"routing/v1": {}, "broken/v1": value}),
9568                "role_declarations values must be objects",
9569            );
9570        }
9571        assert_hello_drops_role_declarations_with_warning(
9572            json!([]),
9573            "role_declarations must be an object",
9574        );
9575    }
9576
9577    #[test]
9578    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9579        assert_hello_drops_role_declarations_with_warning(
9580            json!({"routing/v1": {}, "bad key": {}}),
9581            "role_declarations key must use capability-name grammar",
9582        );
9583    }
9584
9585    #[test]
9586    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9587        assert_hello_drops_role_declarations_with_warning(
9588            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9589            "role_declarations exceeds 16384 compact JSON bytes",
9590        );
9591    }
9592
9593    #[test]
9594    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9595        let registry = Arc::new(Registry::default());
9596        let handler = ControlHandler::new(registry.clone());
9597        let descriptor = json!({
9598            "schema": "ckios.richui.manifest/1",
9599            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9600            "bytes": 128,
9601            "chunks": 2
9602        });
9603        let malformed = [
9604            ("schema", json!("")),
9605            ("schema", json!("a".repeat(65))),
9606            ("schema", json!("Bad schema")),
9607            (
9608                "manifest_digest",
9609                json!(format!("sha256:{}", "A".repeat(64))),
9610            ),
9611            ("manifest_digest", json!("sha256:abc")),
9612            ("bytes", json!(0)),
9613            ("bytes", json!(16_777_217)),
9614            ("bytes", json!("128")),
9615            ("chunks", json!(0)),
9616            ("chunks", json!(1025)),
9617            ("chunks", json!("2")),
9618        ];
9619        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9620            let mut invalid = descriptor.clone();
9621            invalid[field] = invalid_value;
9622            let response = handler
9623                .handle_control(
9624                    ConnectionId::new(210 + index as u64),
9625                    hello_frame_with_rich_ui(
9626                        &format!("bad-rich-ui-{index}"),
9627                        PROTOCOL_VERSION,
9628                        210 + index as u64,
9629                        None,
9630                        invalid,
9631                    ),
9632                )
9633                .unwrap();
9634            assert_eq!(response[0].header.ty, FrameType::Error);
9635            let error = parse_error(&response[0]);
9636            assert_eq!(error["code"], "invalid_manifest");
9637            assert!(error["message"]
9638                .as_str()
9639                .unwrap()
9640                .contains(&format!("rich_ui.{field}")));
9641            assert!(registry
9642                .get_module(&format!("bad-rich-ui-{index}"))
9643                .unwrap()
9644                .is_none());
9645        }
9646    }
9647
9648    #[test]
9649    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9650        let registry = Arc::new(Registry::default());
9651        let handler = ControlHandler::new(Arc::clone(&registry));
9652        let connection = ConnectionId::new(219);
9653        registry
9654            .register_with_control_ops(
9655                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9656                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9657                        "bad schema",
9658                        format!("sha256:{}", "a".repeat(64)),
9659                        128,
9660                        2,
9661                    )))
9662                    .build(),
9663                PROTOCOL_VERSION,
9664                connection,
9665                Vec::new(),
9666            )
9667            .unwrap();
9668        let request =
9669            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9670        let response = handler
9671            .handle_catalog_update(connection, request, Vec::new(), None, None)
9672            .unwrap();
9673        assert_eq!(response[0].header.ty, FrameType::Error);
9674        let error = parse_error(&response[0]);
9675        assert_eq!(error["code"], "invalid_manifest");
9676        assert!(error["message"]
9677            .as_str()
9678            .unwrap()
9679            .contains("rich_ui.schema"));
9680    }
9681
9682    #[test]
9683    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9684        let registry = Arc::new(Registry::default());
9685        let supervisor = SupervisorHandle::new();
9686        supervisor.open_swap("provider", "candidate-nonce".to_string());
9687        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9688        let connection = ConnectionId::new(218);
9689        let hello = handler
9690            .handle_control(
9691                connection,
9692                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9693            )
9694            .unwrap();
9695        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9696        assert!(registry.list_modules().unwrap().1.is_empty());
9697
9698        registry.promote_candidate("provider").unwrap().unwrap();
9699        let request = Frame::build(
9700            FrameType::Request,
9701            control_flags(),
9702            0,
9703            0,
9704            219,
9705            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9706        )
9707        .unwrap();
9708        let response = handler.handle_catalog_list(request, None).unwrap();
9709        let ClientControlResponse::CatalogList { modules, .. } =
9710            serde_json::from_slice(&response[0].body).unwrap()
9711        else {
9712            panic!("expected catalog.list");
9713        };
9714        assert_eq!(modules.len(), 1);
9715        assert!(modules[0].nonce_verified);
9716    }
9717
9718    #[test]
9719    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9720        let registry = Arc::new(Registry::default());
9721        let handler = ControlHandler::new(Arc::clone(&registry));
9722        let hello = Frame::build(
9723            FrameType::Hello,
9724            control_flags(),
9725            0,
9726            0,
9727            220,
9728            serde_json::to_vec(&ModuleHelloBody {
9729                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9730                protocol_ver: PROTOCOL_VERSION,
9731                control_ops: None,
9732                launch_nonce: Some("unverified".to_string()),
9733            })
9734            .unwrap(),
9735        )
9736        .unwrap();
9737        assert_eq!(
9738            handler
9739                .handle_control(ConnectionId::new(220), hello)
9740                .unwrap()[0]
9741                .header
9742                .ty,
9743            FrameType::HelloAck
9744        );
9745        let request = Frame::build(
9746            FrameType::Request,
9747            control_flags(),
9748            0,
9749            0,
9750            221,
9751            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9752        )
9753        .unwrap();
9754        let response = handler.handle_catalog_list(request, None).unwrap();
9755        let ClientControlResponse::CatalogList { modules, .. } =
9756            serde_json::from_slice(&response[0].body).unwrap()
9757        else {
9758            panic!("expected catalog.list");
9759        };
9760        assert_eq!(
9761            serde_json::to_vec(&modules[0]).unwrap(),
9762            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9763        );
9764        assert!(!modules[0].nonce_verified);
9765    }
9766
9767    #[test]
9768    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9769        let registry = Arc::new(Registry::default());
9770        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9771            [("vault".to_string(), true), ("squatter".to_string(), true)],
9772            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9773        );
9774        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9775        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9776            provides: vec!["credentials-provider/v1".to_string()],
9777            requires: Vec::new(),
9778            must_never_reach: Vec::new(),
9779        });
9780        let frame = Frame::build(
9781            FrameType::Hello,
9782            control_flags(),
9783            0,
9784            0,
9785            77,
9786            serde_json::to_vec(&ModuleHelloBody {
9787                manifest: squatter,
9788                protocol_ver: PROTOCOL_VERSION,
9789                control_ops: None,
9790                launch_nonce: None,
9791            })
9792            .expect("HELLO serializes"),
9793        )
9794        .expect("HELLO frame builds");
9795        let response = handler
9796            .handle_control(ConnectionId::new(77), frame)
9797            .expect("reserved claim receives a typed refusal");
9798        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9799        assert_eq!(
9800            registry
9801                .active_registration_count()
9802                .expect("registry reads"),
9803            0,
9804            "a reserved capability refusal must not leave a catalog entry"
9805        );
9806    }
9807
9808    #[test]
9809    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9810        for settlement in ["timeout", "inconclusive", "drop"] {
9811            let breakers = RouteBindBreakers::default();
9812            let RouteBindAdmission::Admitted {
9813                guard: mut old,
9814                probe: false,
9815            } = breakers.admit("prov")
9816            else {
9817                panic!("ordinary relay admitted")
9818            };
9819            let RouteBindAdmission::Admitted {
9820                guard: mut opener, ..
9821            } = breakers.admit("prov")
9822            else {
9823                panic!("second relay admitted")
9824            };
9825            assert!(
9826                !opener
9827                    .record_timeout(1, Duration::ZERO)
9828                    .unwrap()
9829                    .reopened_after_probe
9830            );
9831            let RouteBindAdmission::Admitted {
9832                guard: mut probe,
9833                probe: true,
9834            } = breakers.admit("prov")
9835            else {
9836                panic!("one cooldown probe admitted")
9837            };
9838            match settlement {
9839                "timeout" => assert!(
9840                    !old.record_timeout(1, Duration::ZERO)
9841                        .unwrap()
9842                        .reopened_after_probe
9843                ),
9844                "inconclusive" => old.record_inconclusive(),
9845                "drop" => drop(old),
9846                _ => unreachable!(),
9847            }
9848            assert!(
9849                matches!(
9850                    breakers.admit("prov"),
9851                    RouteBindAdmission::Refused {
9852                        probe_in_flight: true,
9853                        ..
9854                    }
9855                ),
9856                "{settlement} of a pre-open relay cannot release the real probe"
9857            );
9858            assert!(
9859                probe
9860                    .record_timeout(1, Duration::ZERO)
9861                    .unwrap()
9862                    .reopened_after_probe
9863            );
9864            assert!(matches!(
9865                breakers.admit("prov"),
9866                RouteBindAdmission::Admitted { probe: true, .. }
9867            ));
9868        }
9869        let breakers = RouteBindBreakers::default();
9870        let admit = || match breakers.admit("prov") {
9871            RouteBindAdmission::Admitted { guard, .. } => guard,
9872            _ => panic!("relay admitted"),
9873        };
9874        admit().record_timeout(1, Duration::ZERO);
9875        let mut old_probe = admit();
9876        breakers.reset_for_new_module_connection("prov");
9877        admit().record_timeout(1, Duration::ZERO);
9878        let _new_probe = admit();
9879        old_probe.record_inconclusive();
9880        assert!(matches!(
9881            breakers.admit("prov"),
9882            RouteBindAdmission::Refused {
9883                probe_in_flight: true,
9884                ..
9885            }
9886        ));
9887    }
9888
9889    #[tokio::test]
9890    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9891        for candidate in [false, true] {
9892            let registry = Arc::new(Registry::default());
9893            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9894                [("vault".to_string(), true), ("squatter".to_string(), true)],
9895                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9896            );
9897            let conn = ConnectionId::new(77);
9898            let (ctx, mut rx) = route_ctx(conn);
9899            let initial = capability_manifest("squatter", &[], &[]);
9900            if candidate {
9901                registry
9902                    .register_candidate_with_control_ops(
9903                        initial.clone(),
9904                        PROTOCOL_VERSION,
9905                        conn,
9906                        module_baseline_control_ops(),
9907                    )
9908                    .unwrap();
9909                handler
9910                    .forwarding
9911                    .register_candidate_module_connection(
9912                        conn,
9913                        "squatter".to_string(),
9914                        PROTOCOL_VERSION,
9915                        manifest_concurrency(&initial),
9916                        ctx.egress.clone(),
9917                    )
9918                    .unwrap();
9919            } else {
9920                hello_via_sink(
9921                    &handler,
9922                    &ctx,
9923                    &mut rx,
9924                    hello_frame_with_manifest(initial.clone(), 1),
9925                )
9926                .await;
9927            }
9928            let response = handler
9929                .handle_control_frame(
9930                    &ctx,
9931                    catalog_update_with_capabilities_frame(
9932                        2,
9933                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
9934                            .capabilities
9935                            .unwrap(),
9936                    ),
9937                )
9938                .await
9939                .unwrap();
9940            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9941            assert_eq!(
9942                registry
9943                    .get_module_by_connection(conn)
9944                    .unwrap()
9945                    .unwrap()
9946                    .manifest,
9947                initial
9948            );
9949        }
9950    }
9951
9952    #[test]
9953    fn server_describe_surfaces_required_capability_verdict_fields() {
9954        let registry = Arc::new(Registry::default());
9955        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9956            [
9957                ("consumer".to_string(), true),
9958                ("provider".to_string(), false),
9959            ],
9960            BTreeMap::new(),
9961        );
9962        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9963        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9964            provides: Vec::new(),
9965            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9966                capability: "credentials-provider/v1".to_string(),
9967                need: subc_protocol::manifest::CapabilityNeed::Required,
9968            }],
9969            must_never_reach: Vec::new(),
9970        });
9971        let hello = Frame::build(
9972            FrameType::Hello,
9973            control_flags(),
9974            0,
9975            0,
9976            78,
9977            serde_json::to_vec(&ModuleHelloBody {
9978                manifest: consumer,
9979                protocol_ver: PROTOCOL_VERSION,
9980                control_ops: None,
9981                launch_nonce: None,
9982            })
9983            .expect("HELLO serializes"),
9984        )
9985        .expect("HELLO frame builds");
9986        handler
9987            .handle_control(ConnectionId::new(78), hello)
9988            .expect("consumer registers");
9989        let describe = Frame::build(
9990            FrameType::Request,
9991            control_flags(),
9992            0,
9993            0,
9994            79,
9995            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9996                .expect("request serializes"),
9997        )
9998        .expect("describe frame builds");
9999        let response = handler
10000            .handle_server_describe(describe)
10001            .expect("server.describe succeeds");
10002        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10003        let requirement = &rendered["capability_requirements"][0];
10004        assert_eq!(requirement["consumer"], "consumer");
10005        assert_eq!(requirement["verdict"], "never_provided");
10006        assert_eq!(requirement["episode_seq"], 1);
10007        assert_eq!(requirement["config_satisfiable"], false);
10008        assert_eq!(requirement["runtime_available"], false);
10009        assert!(requirement["detail"]
10010            .as_str()
10011            .expect("detail string")
10012            .contains("credentials-provider/v1"));
10013    }
10014
10015    /// Register a consumer that requires `project-identity/v1` and the provider
10016    /// of it, optionally mark daemon shutdown as begun, then deregister the
10017    /// provider as its connection closes. Returns the `capability.requirement`
10018    /// events that deregistration logged and the verdict the evaluator then
10019    /// holds for the requirement (the one `route.open` refusals read).
10020    fn deregister_required_provider(
10021        daemon_shutdown_begun: bool,
10022    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10023        const CAPABILITY: &str = "project-identity/v1";
10024        let registry = Arc::new(Registry::default());
10025        let daemon_shutdown = DaemonShutdownFlag::default();
10026        let handler = ControlHandler::new(Arc::clone(&registry))
10027            .with_capability_config(
10028                [
10029                    ("consumer".to_string(), true),
10030                    ("provider".to_string(), true),
10031                ],
10032                BTreeMap::new(),
10033            )
10034            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10035        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10036        consumer.capabilities = Some(CapabilityDeclarations {
10037            provides: Vec::new(),
10038            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10039                capability: CAPABILITY.to_string(),
10040                need: subc_protocol::manifest::CapabilityNeed::Required,
10041            }],
10042            must_never_reach: Vec::new(),
10043        });
10044        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10045        let provider_conn = ConnectionId::new(182);
10046        for (conn, manifest, corr) in [
10047            (ConnectionId::new(181), consumer, 181),
10048            (provider_conn, provider, 182),
10049        ] {
10050            let hello = handler
10051                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10052                .expect("module registers");
10053            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10054        }
10055        assert_eq!(
10056            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10057            Some(CapabilityVerdict::Provided),
10058            "the registered provider must satisfy the requirement first"
10059        );
10060        if daemon_shutdown_begun {
10061            daemon_shutdown.set_for_test();
10062        }
10063
10064        let capture = EventCapture::default();
10065        let _subscriber =
10066            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10067        handler
10068            .cleanup_connection(provider_conn)
10069            .expect("provider deregisters");
10070        let events = capture
10071            .events()
10072            .into_iter()
10073            .filter(|event| {
10074                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10075            })
10076            .collect();
10077        (
10078            events,
10079            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10080        )
10081    }
10082
10083    #[test]
10084    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10085        let (events, verdict) = deregister_required_provider(true);
10086        assert_eq!(
10087            verdict,
10088            Some(CapabilityVerdict::NeverProvided),
10089            "shutdown must not change the verdict route.open refusals read"
10090        );
10091        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10092        let event = &events[0];
10093        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10094        assert_eq!(
10095            event.fields.get("cause").map(String::as_str),
10096            Some("\"daemon_shutdown\""),
10097            "{event:?}"
10098        );
10099        assert_eq!(
10100            event.fields.get("verdict").map(String::as_str),
10101            Some("\"never_provided\""),
10102            "{event:?}"
10103        );
10104    }
10105
10106    #[test]
10107    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10108        let (events, verdict) = deregister_required_provider(false);
10109        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10110        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10111        let event = &events[0];
10112        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10113        assert_eq!(event.fields.get("cause"), None, "{event:?}");
10114        assert_eq!(
10115            event.fields.get("verdict").map(String::as_str),
10116            Some("\"never_provided\""),
10117            "{event:?}"
10118        );
10119    }
10120
10121    #[test]
10122    fn catalog_list_omits_capabilities_for_legacy_manifest() {
10123        let registry = Arc::new(Registry::default());
10124        let handler = ControlHandler::new(Arc::clone(&registry));
10125        let hello = handler
10126            .handle_control(
10127                ConnectionId::new(101),
10128                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10129            )
10130            .expect("legacy HELLO registers");
10131        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10132
10133        let request = Frame::build(
10134            FrameType::Request,
10135            control_flags(),
10136            0,
10137            0,
10138            102,
10139            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10140                .expect("catalog request serializes"),
10141        )
10142        .expect("catalog request frame builds");
10143        let response = handler
10144            .handle_catalog_list(request, None)
10145            .expect("catalog list succeeds");
10146        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10147        assert!(
10148            body["modules"][0].get("capabilities").is_none(),
10149            "legacy manifest must retain an absent capabilities field on catalog.list"
10150        );
10151    }
10152
10153    #[test]
10154    fn hello_ack_omits_storage_when_no_storage_config() {
10155        let registry = Arc::new(Registry::default());
10156        let handler = ControlHandler::new(Arc::clone(&registry));
10157        let responses = handler
10158            .handle_control(
10159                ConnectionId::new(1),
10160                hello_frame("aft", PROTOCOL_VERSION, 7),
10161            )
10162            .unwrap();
10163        let ack = parse_ack(&responses[0]);
10164        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10165        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10166    }
10167
10168    #[tokio::test]
10169    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10170        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10171        let registry = Arc::new(Registry::default());
10172        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
10173        let responses = handler
10174            .handle_control(
10175                ConnectionId::new(1),
10176                hello_frame("aft", PROTOCOL_VERSION, 7),
10177            )
10178            .unwrap();
10179        let ack = parse_ack(&responses[0]);
10180        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10181
10182        let described = handler
10183            .handle_control_frame(
10184                &route_ctx(ConnectionId::new(2)).0,
10185                Frame::build(
10186                    FrameType::Request,
10187                    control_flags(),
10188                    0,
10189                    0,
10190                    9,
10191                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10192                )
10193                .unwrap(),
10194            )
10195            .await
10196            .unwrap();
10197        let ClientControlResponse::ServerDescribe { machine_id, .. } =
10198            serde_json::from_slice(&described[0].body).unwrap()
10199        else {
10200            panic!("server.describe answered with another shape");
10201        };
10202        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10203    }
10204
10205    #[test]
10206    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10207        // With a central sqlite storage policy, each registering module gets its
10208        // own resolved descriptor in HELLO_ACK, keyed by its module id.
10209        let registry = Arc::new(Registry::default());
10210        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
10211            crate::daemon_config::StorageConfig::Sqlite {
10212                data_home: std::path::PathBuf::from("/data"),
10213            },
10214        ));
10215
10216        let responses = handler
10217            .handle_control(
10218                ConnectionId::new(1),
10219                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10220            )
10221            .unwrap();
10222        let ack = parse_ack(&responses[0]);
10223        assert_eq!(
10224            ack.storage,
10225            Some(serde_json::json!({
10226                "module_id": "alfonso-routing",
10227                "storage_namespace": "default",
10228                "isolation": { "kind": "module" },
10229                "backend": {
10230                    "backend": "sqlite",
10231                    "path": "/data/cortexkit/alfonso-routing/store.db"
10232                }
10233            })),
10234            "the delivered descriptor is the module's own sqlite store path"
10235        );
10236    }
10237
10238    #[test]
10239    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10240        let registry = Arc::new(Registry::default());
10241        let handler = ControlHandler::new(Arc::clone(&registry));
10242        let conn = ConnectionId::new(1);
10243        let responses = handler
10244            .handle_control(
10245                conn,
10246                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10247            )
10248            .unwrap();
10249        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10250        let registration = registry.get_module("aft").unwrap().unwrap();
10251        assert_eq!(registration.control_ops, module_baseline_control_ops());
10252
10253        let frame =
10254            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10255        assert!(handler
10256            .guard_module_control_op(&frame, "aft", "route.bind")
10257            .unwrap()
10258            .is_none());
10259        let error = handler
10260            .guard_module_control_op(&frame, "aft", "test.synthetic")
10261            .unwrap()
10262            .expect("synthetic ungranted op should be rejected");
10263        assert_eq!(error.header.ty, FrameType::Error);
10264        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10265    }
10266
10267    #[test]
10268    fn hello_control_ops_some_adds_optional_grants() {
10269        let registry = Arc::new(Registry::default());
10270        let handler = ControlHandler::new(Arc::clone(&registry));
10271        handler
10272            .handle_control(
10273                ConnectionId::new(1),
10274                hello_frame_with_control_ops(
10275                    "aft",
10276                    PROTOCOL_VERSION,
10277                    7,
10278                    Some(vec![
10279                        "future.synthetic".to_string(),
10280                        "route.bind".to_string(),
10281                    ]),
10282                ),
10283            )
10284            .unwrap();
10285        let registration = registry.get_module("aft").unwrap().unwrap();
10286        assert_eq!(
10287            registration.control_ops,
10288            vec![
10289                "route.bind".to_string(),
10290                "route.status".to_string(),
10291                "future.synthetic".to_string(),
10292            ]
10293        );
10294        let frame =
10295            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10296        assert!(handler
10297            .guard_module_control_op(&frame, "aft", "future.synthetic")
10298            .unwrap()
10299            .is_none());
10300    }
10301
10302    #[tokio::test]
10303    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10304        let registry = Arc::new(Registry::default());
10305        let forwarding = Arc::new(ForwardingTable::default());
10306        let handler =
10307            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10308        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10309        hello_via_sink(
10310            &handler,
10311            &module_ctx,
10312            &mut module_rx,
10313            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10314        )
10315        .await;
10316
10317        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10318        let responses = handler
10319            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10320            .await
10321            .unwrap();
10322        assert_eq!(responses.len(), 1);
10323        assert_eq!(responses[0].header.ty, FrameType::Error);
10324        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10325        assert!(module_rx.try_recv().is_err());
10326    }
10327
10328    #[tokio::test]
10329    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10330        let registry = Arc::new(Registry::default());
10331        let forwarding = Arc::new(ForwardingTable::default());
10332        let handler =
10333            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10334        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10335        hello_via_sink(
10336            &handler,
10337            &module_ctx,
10338            &mut module_rx,
10339            hello_frame_with_control_ops(
10340                "aft",
10341                PROTOCOL_VERSION,
10342                7,
10343                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10344            ),
10345        )
10346        .await;
10347
10348        let project_root = unique_project_root("demux");
10349        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10350        let route_handler = handler.clone();
10351        let route_task = tokio::spawn(async move {
10352            route_handler
10353                .handle_control_frame(
10354                    &route_client_ctx,
10355                    route_open_frame(100, "aft", project_root),
10356                )
10357                .await
10358                .unwrap()
10359        });
10360        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10361            .await
10362            .unwrap()
10363            .unwrap();
10364        assert!(matches!(
10365            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10366            ModuleControlRequest::RouteBind { .. }
10367        ));
10368
10369        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10370        let health_handler = handler.clone();
10371        let health_task = tokio::spawn(async move {
10372            health_handler
10373                .handle_control_frame(
10374                    &health_client_ctx,
10375                    supervisor_health_probe_frame(101, "aft"),
10376                )
10377                .await
10378                .unwrap()
10379        });
10380        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10381            .await
10382            .unwrap()
10383            .unwrap();
10384        assert_eq!(
10385            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10386            ModuleControlRequest::HealthCheck {}
10387        );
10388
10389        handler
10390            .handle_control_frame(
10391                &module_ctx,
10392                health_response(health_frame.header.corr, HealthStatus::Degraded),
10393            )
10394            .await
10395            .unwrap();
10396        let health_response = health_task.await.unwrap();
10397        assert_eq!(health_response.len(), 1);
10398        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10399            ClientControlResponse::SupervisorHealthProbe {
10400                module_id,
10401                status,
10402                detail,
10403                metrics,
10404            } => {
10405                assert_eq!(module_id, "aft");
10406                assert_eq!(status, HealthStatus::Degraded);
10407                assert_eq!(detail.as_deref(), Some("warming"));
10408                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10409            }
10410            other => panic!("unexpected health response: {other:?}"),
10411        }
10412
10413        handler
10414            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10415            .await
10416            .unwrap();
10417        let route_response = route_task.await.unwrap();
10418        assert!(route_response.is_empty());
10419        let published = route_client_rx.recv().await.unwrap();
10420        assert!(matches!(
10421            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10422            ClientControlResponse::RouteOpen { .. }
10423        ));
10424    }
10425
10426    /// Start one `route.open` on `client_connection` and return its still-running
10427    /// handler task together with the `route.bind` the module received for it.
10428    /// The handler blocks until the module answers, so it has to run as a task
10429    /// while the test drives the module side.
10430    async fn relay_route_open(
10431        handler: &ControlHandler,
10432        client_connection: ConnectionId,
10433        client_egress: &FrameSink,
10434        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10435        corr: u64,
10436        module_id: &str,
10437        project_root_label: &str,
10438    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10439        let ctx = RouteCtx {
10440            connection_id: client_connection,
10441            egress: client_egress.clone(),
10442        };
10443        let handler = handler.clone();
10444        let project_root = unique_project_root(project_root_label);
10445        let module_id = module_id.to_string();
10446        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10447        let task = tokio::spawn(async move {
10448            let _guard = tracing::dispatcher::set_default(&dispatch);
10449            handler
10450                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10451                .await
10452                .unwrap()
10453        });
10454        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10455            .await
10456            .expect("module receives the relayed route.bind")
10457            .expect("module egress is open");
10458        (task, bind.frame)
10459    }
10460
10461    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10462        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10463            ModuleControlRequest::RouteBind {
10464                route_channel,
10465                epoch,
10466                ..
10467            } => (route_channel, epoch),
10468            other => panic!("expected a route.bind request, got {other:?}"),
10469        }
10470    }
10471
10472    fn published_route(frame: &Frame) -> (u16, u32) {
10473        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10474            ClientControlResponse::RouteOpen {
10475                route_channel,
10476                route_epoch,
10477            } => (route_channel, route_epoch),
10478            other => panic!("expected a route.open response, got {other:?}"),
10479        }
10480    }
10481
10482    /// Reproduction of a production outage. A client had `route.open`s in
10483    /// flight to a module and was already marked closing -- its egress had refused a
10484    /// module frame, so the daemon asked its connection to end -- while its sink
10485    /// was still open. When the module acked those binds, the daemon refused to
10486    /// commit a route for a closing client, and that refusal was returned from
10487    /// the MODULE connection's frame handler, where a router error that has no
10488    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
10489    /// the supervisor correctly did not respawn a clean exit, and every seat lost
10490    /// its tools for hours -- one client's teardown took down a connection
10491    /// carrying ~170 other routes.
10492    ///
10493    /// The window is opened here by calling the production path that opens it
10494    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
10495    /// state that matters is "in `closing_connections`, sink still open, relay
10496    /// still pending", and it lasts only from the close request until the
10497    /// connection loop reacts to it; a socket-level test can flood a client into
10498    /// that escalation but cannot pin the module's ack inside the window. Closing
10499    /// the socket instead takes the other path entirely -- connection teardown
10500    /// removes the pending relay under the same lock, so the ack finds nothing.
10501    #[tokio::test]
10502    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10503        let registry = Arc::new(Registry::default());
10504        let forwarding = Arc::new(ForwardingTable::default());
10505        let handler =
10506            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10507
10508        let module_connection = ConnectionId::new(30);
10509        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10510        hello_via_sink(
10511            &handler,
10512            &module_ctx,
10513            &mut module_rx,
10514            hello_frame("aft", PROTOCOL_VERSION, 7),
10515        )
10516        .await;
10517
10518        let dying_client = ConnectionId::new(31);
10519        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10520
10521        // A published route on the dying client. The escalation below only marks
10522        // a connection closing for a route it has already published.
10523        let (first_task, first_bind) = relay_route_open(
10524            &handler,
10525            dying_client,
10526            &dying_ctx.egress,
10527            &mut module_rx,
10528            100,
10529            "aft",
10530            "closing-first",
10531        )
10532        .await;
10533        handler
10534            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10535            .await
10536            .unwrap();
10537        assert!(first_task.await.unwrap().is_empty());
10538        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10539
10540        // A second route.open from the same client, relayed and awaiting its ack.
10541        let (second_task, second_bind) = relay_route_open(
10542            &handler,
10543            dying_client,
10544            &dying_ctx.egress,
10545            &mut module_rx,
10546            101,
10547            "aft",
10548            "closing-second",
10549        )
10550        .await;
10551        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10552
10553        // The window: the client is closing, its sink is still open, and its
10554        // second bind is still pending.
10555        assert!(forwarding
10556            .escalate_client_delivery_failure(
10557                dying_client,
10558                first_channel,
10559                first_epoch,
10560                CloseReason::new(
10561                    "module_to_client_delivery_failed",
10562                    "client egress refused a module frame",
10563                ),
10564                crate::forwarding::UndeliveredFrame {
10565                    module_id: None,
10566                    sink: &dying_ctx.egress,
10567                },
10568            )
10569            .unwrap());
10570        assert!(!dying_ctx.egress.is_closed());
10571
10572        // The frame that used to end the module connection.
10573        let ack = handler
10574            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10575            .await;
10576        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10577        if module_loop_error.is_some() {
10578            // What the server's connection loop does with a router error that has
10579            // no ERROR-frame translation: end the connection, which releases the
10580            // module's registration and every route on it.
10581            handler.cleanup_connection(module_connection).unwrap();
10582        }
10583        // Read the module's next frame before opening the co-tenant's route, so
10584        // the GOODBYE assertion below is about THIS ack and not about later
10585        // traffic. `None` means the module was told nothing.
10586        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10587            .await
10588            .ok()
10589            .flatten();
10590
10591        // 1. The module connection is still registered.
10592        assert!(
10593            registry
10594                .get_module_by_connection(module_connection)
10595                .unwrap()
10596                .is_some(),
10597            "one client's closing connection ended the shared module connection: \
10598             {module_loop_error:?}"
10599        );
10600        // ...and still serving: another client can open and use a route on it.
10601        let cotenant = ConnectionId::new(32);
10602        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10603        let (cotenant_task, cotenant_bind) = relay_route_open(
10604            &handler,
10605            cotenant,
10606            &cotenant_ctx.egress,
10607            &mut module_rx,
10608            102,
10609            "aft",
10610            "closing-cotenant",
10611        )
10612        .await;
10613        handler
10614            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10615            .await
10616            .unwrap();
10617        assert!(cotenant_task.await.unwrap().is_empty());
10618        let (cotenant_channel, cotenant_epoch) =
10619            published_route(&cotenant_rx.recv().await.unwrap());
10620        assert!(matches!(
10621            forwarding
10622                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10623                .unwrap(),
10624            DataRoute::Client(DataRouteState::Bound(_))
10625        ));
10626
10627        // 2. The module was told to drop the binding it created for the route
10628        //    that will never be published.
10629        let goodbye = post_ack_module_frame
10630            .expect("module receives a GOODBYE for the abandoned route channel");
10631        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10632        assert_eq!(goodbye.header.channel, abandoned_channel);
10633        assert_eq!(goodbye.header.epoch, abandoned_epoch);
10634
10635        // 3. The dying client received nothing: no route was ever published to
10636        //    it. Its route.open is answered as unavailable, which the connection
10637        //    loop would write to a socket that is already going away.
10638        assert!(dying_rx.try_recv().is_err());
10639        let second_response = second_task.await.unwrap();
10640        assert_eq!(second_response.len(), 1);
10641        assert_eq!(
10642            parse_error(&second_response[0])["code"],
10643            "target_unavailable"
10644        );
10645    }
10646
10647    /// One `catalog.list` as a client sees it: the generation and the listed
10648    /// module ids with their readiness.
10649    async fn catalog_snapshot(
10650        handler: &ControlHandler,
10651        client_ctx: &RouteCtx,
10652        corr: u64,
10653    ) -> (u64, Vec<(String, bool)>) {
10654        let body =
10655            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10656        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10657        let reply = handler
10658            .handle_control_frame(client_ctx, request)
10659            .await
10660            .unwrap()
10661            .pop()
10662            .unwrap();
10663        let ClientControlResponse::CatalogList {
10664            generation,
10665            modules,
10666            ..
10667        } = serde_json::from_slice(&reply.body).unwrap()
10668        else {
10669            panic!("not a catalog.list reply");
10670        };
10671        (
10672            generation,
10673            modules
10674                .into_iter()
10675                .map(|entry| (entry.module_id, entry.ready))
10676                .collect(),
10677        )
10678    }
10679
10680    /// A handler whose HELLO for module "aft" on connection 40 is parked
10681    /// between reserving its registry entry and installing its forwarding
10682    /// endpoint. Returns the handler, the two-party barrier (wait once to know
10683    /// the HELLO is parked, once more to release it), the HELLO's thread, and
10684    /// the module's context and receiver.
10685    fn hello_held_before_forwarding() -> (
10686        ControlHandler,
10687        Arc<std::sync::Barrier>,
10688        std::thread::JoinHandle<Vec<Frame>>,
10689        RouteCtx,
10690        mpsc::Receiver<crate::router::OutboundFrame>,
10691    ) {
10692        let hold = Arc::new(std::sync::Barrier::new(2));
10693        let handler = ControlHandler::with_forwarding(
10694            Arc::new(Registry::default()),
10695            Arc::new(ForwardingTable::default()),
10696        )
10697        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10698        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10699        let hello = {
10700            let handler = handler.clone();
10701            let module_ctx = module_ctx.clone();
10702            std::thread::spawn(move || {
10703                handler
10704                    .handle_hello(
10705                        module_ctx.connection_id,
10706                        Some(module_ctx.egress.clone()),
10707                        hello_frame("aft", PROTOCOL_VERSION, 7),
10708                    )
10709                    .unwrap()
10710            })
10711        };
10712        (handler, hold, hello, module_ctx, module_rx)
10713    }
10714
10715    /// While a HELLO has reserved its registry entry but its forwarding
10716    /// endpoint is not installed yet, `route.open` to the module can only be
10717    /// refused. `catalog.list` must not list the module in that window: a
10718    /// client that opens a route as soon as it sees a module listed and ready
10719    /// would be refused for a module the daemon had just advertised. The
10720    /// refusal in the window stays the retryable `target_unavailable`, never
10721    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10722    /// Once the HELLO completes, the module is listed and a route to it opens.
10723    #[tokio::test]
10724    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10725        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10726        let client = ConnectionId::new(41);
10727        let (client_ctx, mut client_rx) = route_ctx(client);
10728
10729        hold.wait();
10730        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10731        assert!(handler.registry.get_module("aft").unwrap().is_some());
10732        assert!(!handler
10733            .forwarding
10734            .has_live_module_connection("aft")
10735            .unwrap());
10736        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10737        assert!(
10738            listed.is_empty(),
10739            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10740        );
10741        let refused = handler
10742            .handle_control_frame(
10743                &client_ctx,
10744                route_open_frame(301, "aft", unique_project_root("listed-window")),
10745            )
10746            .await
10747            .unwrap();
10748        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10749        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10750        let duplicate = handler
10751            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10752            .await
10753            .unwrap();
10754        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10755
10756        hold.wait();
10757        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10758        let ack = module_rx.try_recv().unwrap().frame;
10759        assert_eq!(ack.header.ty, FrameType::HelloAck);
10760        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10761        assert_eq!(listed, vec![("aft".to_string(), true)]);
10762
10763        let (open_task, bind) = relay_route_open(
10764            &handler,
10765            client,
10766            &client_ctx.egress,
10767            &mut module_rx,
10768            303,
10769            "aft",
10770            "listed-after",
10771        )
10772        .await;
10773        assert!(handler
10774            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10775            .await
10776            .unwrap()
10777            .is_empty());
10778        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10779        published_route(&client_rx.try_recv().unwrap().frame);
10780    }
10781
10782    /// A catalog watcher keys on the generation: when it changes, it re-reads
10783    /// the listing. So the generation must advance exactly when the module
10784    /// appears. If it advanced while the module was still hidden, a watcher
10785    /// would read the new generation without the module, and the module would
10786    /// later appear under that same generation with nothing telling the
10787    /// watcher to look again.
10788    #[tokio::test]
10789    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10790        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10791        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10792        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10793
10794        hold.wait();
10795        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10796        assert!(listed.is_empty());
10797        assert_eq!(
10798            during, before,
10799            "the generation moved while the module was not listed"
10800        );
10801
10802        hold.wait();
10803        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10804        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10805        assert_eq!(listed, vec![("aft".to_string(), true)]);
10806        assert_eq!(
10807            after,
10808            before + 1,
10809            "the module appeared without exactly one generation bump"
10810        );
10811    }
10812
10813    /// A HELLO whose forwarding endpoint cannot be installed (here the
10814    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
10815    /// refused and leaves nothing behind: no registry entry, no catalog entry,
10816    /// and the catalog generation where it was.
10817    #[tokio::test]
10818    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10819        let registry = Arc::new(Registry::default());
10820        let handler = ControlHandler::with_forwarding(
10821            Arc::clone(&registry),
10822            Arc::new(ForwardingTable::default()),
10823        );
10824        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10825        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10826        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10827        drop(module_rx);
10828
10829        let replies = handler
10830            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10831            .await
10832            .unwrap();
10833        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10834        assert!(registry.get_module("aft").unwrap().is_none());
10835        assert!(registry
10836            .get_module_by_connection(module_ctx.connection_id)
10837            .unwrap()
10838            .is_none());
10839        assert_eq!(
10840            catalog_snapshot(&handler, &client_ctx, 301).await,
10841            (before, Vec::new())
10842        );
10843    }
10844
10845    /// A handler whose `route.open` parks after queueing the route.bind and
10846    /// before marking it enqueued, with module "aft" registered on connection
10847    /// 40. Returns the two-party barrier that parks and releases it.
10848    async fn route_open_held_after_bind_send() -> (
10849        ControlHandler,
10850        Arc<tokio::sync::Barrier>,
10851        RouteCtx,
10852        mpsc::Receiver<crate::router::OutboundFrame>,
10853    ) {
10854        let hold = Arc::new(tokio::sync::Barrier::new(2));
10855        let handler = ControlHandler::with_forwarding(
10856            Arc::new(Registry::default()),
10857            Arc::new(ForwardingTable::default()),
10858        )
10859        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10860        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10861        hello_via_sink(
10862            &handler,
10863            &module_ctx,
10864            &mut module_rx,
10865            hello_frame("aft", PROTOCOL_VERSION, 7),
10866        )
10867        .await;
10868        (handler, hold, module_ctx, module_rx)
10869    }
10870
10871    /// A fast module acks the route.bind, and the reader commits the route,
10872    /// before `route.open` has marked the relay enqueued. The daemon used to
10873    /// read the missing relay as an abandoned bind and send the module a
10874    /// GOODBYE for the route it had just committed: the client was told its
10875    /// route was open, the module unbound the channel, and the client's first
10876    /// request was dropped on the module side as unbound.
10877    #[tokio::test]
10878    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10879        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10880        let client = ConnectionId::new(41);
10881        let (client_ctx, mut client_rx) = route_ctx(client);
10882
10883        let (open_task, bind) = relay_route_open(
10884            &handler,
10885            client,
10886            &client_ctx.egress,
10887            &mut module_rx,
10888            200,
10889            "aft",
10890            "acked-before-mark",
10891        )
10892        .await;
10893        let (module_channel, module_epoch) = route_bind_channel(&bind);
10894        // route.open is now parked between queueing the bind and marking it.
10895        hold.wait().await;
10896
10897        // The module answers inside that window, and the route commits.
10898        assert!(handler
10899            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10900            .await
10901            .unwrap()
10902            .is_empty());
10903        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10904
10905        hold.wait().await;
10906        assert!(
10907            open_task.await.unwrap().is_empty(),
10908            "route.open answers accepted"
10909        );
10910
10911        // Every frame route.open could have sent is already queued, because it
10912        // has returned; the module must have been sent nothing.
10913        match module_rx.try_recv() {
10914            Err(mpsc::error::TryRecvError::Empty) => {}
10915            Ok(outbound) => panic!(
10916                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10917                outbound.frame.header.ty,
10918                outbound.frame.header.channel,
10919                outbound.frame.header.epoch
10920            ),
10921            Err(err) => panic!("module egress closed: {err}"),
10922        }
10923
10924        // The client's first request on the route is the next thing the module
10925        // reads, on the binding it just created.
10926        let router = Router::with_control_handler(Arc::new(handler.clone()));
10927        let request = Frame::build(
10928            FrameType::Request,
10929            Flags::new(true, Priority::Interactive, false),
10930            client_channel,
10931            client_epoch,
10932            201,
10933            b"first request".to_vec(),
10934        )
10935        .unwrap();
10936        router
10937            .route_for_connection(&client_ctx, request)
10938            .await
10939            .unwrap();
10940        let delivered = module_rx.try_recv().unwrap().frame;
10941        assert_eq!(delivered.header.ty, FrameType::Request);
10942        assert_eq!(
10943            (delivered.header.channel, delivered.header.epoch),
10944            (module_channel, module_epoch)
10945        );
10946        assert_eq!(delivered.body, b"first request");
10947    }
10948
10949    /// The client goes away while `route.open` is still queueing the bind, so
10950    /// the relay is dropped with no module reply. The teardown cannot send the
10951    /// GOODBYE itself (it could reach the module ahead of the bind), so
10952    /// `route.open` must send it once the bind is queued; otherwise the module
10953    /// keeps a binding the daemon will never commit.
10954    #[tokio::test]
10955    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10956        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10957        let client = ConnectionId::new(42);
10958        let (client_ctx, _client_rx) = route_ctx(client);
10959
10960        let (open_task, bind) = relay_route_open(
10961            &handler,
10962            client,
10963            &client_ctx.egress,
10964            &mut module_rx,
10965            300,
10966            "aft",
10967            "released-before-mark",
10968        )
10969        .await;
10970        let (module_channel, module_epoch) = route_bind_channel(&bind);
10971        hold.wait().await;
10972
10973        handler.cleanup_connection(client).unwrap();
10974        assert!(
10975            module_rx.try_recv().is_err(),
10976            "client teardown must not GOODBYE a bind route.open is still queueing"
10977        );
10978
10979        hold.wait().await;
10980        let response = open_task.await.unwrap();
10981        assert_eq!(response.len(), 1);
10982        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10983
10984        let goodbye = module_rx
10985            .try_recv()
10986            .expect("module is told to drop the binding the daemon abandoned")
10987            .frame;
10988        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10989        assert_eq!(
10990            (goodbye.header.channel, goodbye.header.epoch),
10991            (module_channel, module_epoch)
10992        );
10993        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10994    }
10995
10996    /// The module acks inside the same window, but the client's egress closed
10997    /// first, so the reader refuses to commit. The ack proves the module holds
10998    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
10999    /// send a second one.
11000    #[tokio::test]
11001    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11002        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11003        let client = ConnectionId::new(43);
11004        let (client_ctx, client_rx) = route_ctx(client);
11005
11006        let (open_task, bind) = relay_route_open(
11007            &handler,
11008            client,
11009            &client_ctx.egress,
11010            &mut module_rx,
11011            400,
11012            "aft",
11013            "gone-client-before-mark",
11014        )
11015        .await;
11016        let (module_channel, module_epoch) = route_bind_channel(&bind);
11017        hold.wait().await;
11018
11019        drop(client_rx);
11020        assert!(client_ctx.egress.is_closed());
11021        handler
11022            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11023            .await
11024            .unwrap();
11025
11026        hold.wait().await;
11027        let response = open_task.await.unwrap();
11028        assert_eq!(response.len(), 1);
11029        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11030
11031        let goodbye = module_rx
11032            .try_recv()
11033            .expect("module is told to drop the binding it acked")
11034            .frame;
11035        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11036        assert_eq!(
11037            (goodbye.header.channel, goodbye.header.epoch),
11038            (module_channel, module_epoch)
11039        );
11040        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11041        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11042    }
11043
11044    /// The fence at the module-loop boundary, stated as its own contract: which
11045    /// forwarding failures are allowed to end the module connection that is being
11046    /// served. A `ConnectionClosing` naming some client is about that client, and
11047    /// a module connection is shared; the same error naming the module's own
11048    /// connection is about this connection and must stay fatal, as must failures
11049    /// that are about the forwarding table itself.
11050    #[test]
11051    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11052        let handler = ControlHandler::default();
11053        let module_connection = ConnectionId::new(30);
11054        let client_connection = ConnectionId::new(31);
11055
11056        handler
11057            .refuse_to_end_module_connection_for_a_client(
11058                module_connection,
11059                77,
11060                ForwardingError::ConnectionClosing {
11061                    connection_id: client_connection,
11062                },
11063            )
11064            .expect("a closing client must never end the module connection");
11065
11066        assert!(matches!(
11067            handler.refuse_to_end_module_connection_for_a_client(
11068                module_connection,
11069                78,
11070                ForwardingError::ConnectionClosing {
11071                    connection_id: module_connection,
11072                },
11073            ),
11074            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11075                connection_id
11076            })) if connection_id == module_connection
11077        ));
11078        assert!(matches!(
11079            handler.refuse_to_end_module_connection_for_a_client(
11080                module_connection,
11081                79,
11082                ForwardingError::Poisoned,
11083            ),
11084            Err(RouterError::Forwarding(ForwardingError::Poisoned))
11085        ));
11086        assert!(matches!(
11087            handler.refuse_to_end_module_connection_for_a_client(
11088                module_connection,
11089                80,
11090                ForwardingError::StaleModuleEndpoint,
11091            ),
11092            Err(RouterError::Forwarding(
11093                ForwardingError::StaleModuleEndpoint
11094            ))
11095        ));
11096    }
11097
11098    /// The spawn-attestation guard is what stops a connected module from claiming
11099    /// another module's identity and being stamped `Reserved` for it. Every other
11100    /// test that supplies a consumer_identity supplies a CORRECT one, because a
11101    /// correct one is what the rest of the flow needs -- so the guard's rejection
11102    /// branch was never the subject of an assertion, only its acceptance branch.
11103    ///
11104    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
11105    /// whole subc-core library suite green; only the forwarding integration tests
11106    /// notice, and they notice for unrelated reasons. This test exists so the
11107    /// refusal itself is asserted where the guard lives: it fails if the identity
11108    /// check stops refusing, which is the direction that matters, since a guard
11109    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
11110    #[tokio::test]
11111    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11112        let registry = Arc::new(Registry::default());
11113        let forwarding = Arc::new(ForwardingTable::default());
11114        let supervisor = SupervisorHandle::new();
11115        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11116        let handler =
11117            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11118                .with_supervisor(supervisor);
11119
11120        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11121        hello_via_sink(
11122            &handler,
11123            &target_ctx,
11124            &mut target_rx,
11125            hello_frame("target", PROTOCOL_VERSION, 1),
11126        )
11127        .await;
11128
11129        // A real supervised module id presenting the wrong nonce. This is the
11130        // impersonation case: the attacker knows a privileged module_id, which is
11131        // public, and guesses at the nonce, which is not.
11132        let wrong_nonce = handler
11133            .handle_control_frame(
11134                &route_ctx(ConnectionId::new(91)).0,
11135                route_open_frame_with_admission_facts(
11136                    20,
11137                    "target",
11138                    unique_project_root("admission-facts"),
11139                    Some(subc_control::ConsumerIdentity {
11140                        module_id: "fed".to_string(),
11141                        launch_nonce: "not-the-real-nonce".to_string(),
11142                    }),
11143                    None,
11144                ),
11145            )
11146            .await
11147            .unwrap();
11148        assert_eq!(
11149            parse_error(&wrong_nonce[0])["code"],
11150            "bad_consumer_identity",
11151            "a mismatched launch nonce must be refused, not stamped Reserved"
11152        );
11153
11154        // A module id the supervisor never spawned at all, so no nonce exists to
11155        // compare against. An implementation that treats "no record" as "nothing
11156        // to check" fails open here while passing the case above.
11157        let never_spawned = handler
11158            .handle_control_frame(
11159                &route_ctx(ConnectionId::new(92)).0,
11160                route_open_frame_with_admission_facts(
11161                    21,
11162                    "target",
11163                    unique_project_root("admission-facts"),
11164                    Some(subc_control::ConsumerIdentity {
11165                        module_id: "never-spawned".to_string(),
11166                        launch_nonce: "any-nonce".to_string(),
11167                    }),
11168                    None,
11169                ),
11170            )
11171            .await
11172            .unwrap();
11173        assert_eq!(
11174            parse_error(&never_spawned[0])["code"],
11175            "bad_consumer_identity",
11176            "an unspawned module_id must be refused rather than accepted for lack of a record"
11177        );
11178    }
11179
11180    /// The refusal test above proves the guard says NO. Nothing proved it can say
11181    /// YES, and the difference is not academic: replacing the whole authorization
11182    /// with `false` -- admitting no consumer identity at all, revoking Reserved
11183    /// standing for every supervised module in the fleet -- leaves 110 of the 111
11184    /// library tests GREEN. The one that notices does so by HANGING, because it
11185    /// waits for a bind that can no longer happen.
11186    ///
11187    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
11188    /// or flaky test, invites a RETRY rather than an investigation, and the retry
11189    /// hangs too and gets blamed on the runner. So a total revocation of the
11190    /// daemon's trust grant would have shipped behind a symptom nobody attributes
11191    /// to code.
11192    ///
11193    /// The bias is structural rather than accidental. A REFUSAL looks like a
11194    /// failure someone writes a test for; a GRANT looks like the happy path. Every
11195    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
11196    /// suite for that reason, and this one is the purest case in the daemon.
11197    ///
11198    /// This test asserts the EFFECT rather than the absence of an error: the module
11199    /// receives a RouteBind and it carries `Reserved` naming the attested module.
11200    /// A guard that admitted nobody would produce no bind at all; one that admitted
11201    /// everybody would stamp the wrong principal, which the refusal test catches.
11202    #[tokio::test]
11203    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11204        let registry = Arc::new(Registry::default());
11205        let forwarding = Arc::new(ForwardingTable::default());
11206        let supervisor = SupervisorHandle::new();
11207        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11208        let handler =
11209            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11210                .with_supervisor(supervisor);
11211
11212        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11213        hello_via_sink(
11214            &handler,
11215            &target_ctx,
11216            &mut target_rx,
11217            hello_frame("target", PROTOCOL_VERSION, 1),
11218        )
11219        .await;
11220
11221        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11222        let route_handler = handler.clone();
11223        let route_task = tokio::spawn(async move {
11224            route_handler
11225                .handle_control_frame(
11226                    &client_ctx,
11227                    route_open_frame_with_admission_facts(
11228                        30,
11229                        "target",
11230                        unique_project_root("admission-facts"),
11231                        Some(subc_control::ConsumerIdentity {
11232                            module_id: "fed".to_string(),
11233                            launch_nonce: "fed-nonce".to_string(),
11234                        }),
11235                        None,
11236                    ),
11237                )
11238                .await
11239                .unwrap()
11240        });
11241
11242        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
11243        // the very mutation it exists to catch -- a guard that admits nobody -- no
11244        // bind is ever sent, so it HUNG rather than failing. That reproduces the
11245        // exact defect being fixed: a total revocation detected only as a stalled
11246        // suite, which reads as flakiness and invites a retry. An acceptance test
11247        // that waits for an effect must bound the wait, or a red becomes a hang.
11248        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11249            .await
11250            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11251            .expect("module control channel closed before route.bind");
11252        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11253        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11254            panic!("expected route.bind")
11255        };
11256        assert_eq!(
11257            principal,
11258            Some(Principal::Reserved {
11259                module_id: "fed".to_string()
11260            }),
11261            "a correctly attested consumer must be stamped Reserved for its own id"
11262        );
11263
11264        handler
11265            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11266            .await
11267            .unwrap();
11268        assert!(route_task.await.unwrap().is_empty());
11269        assert!(
11270            matches!(
11271                serde_json::from_slice::<ClientControlResponse>(
11272                    &client_rx.recv().await.unwrap().body
11273                )
11274                .unwrap(),
11275                ClientControlResponse::RouteOpen { .. }
11276            ),
11277            "the route must actually open, not merely avoid an error"
11278        );
11279    }
11280
11281    #[tokio::test(start_paused = true)]
11282    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11283        let registry = Arc::new(Registry::default());
11284        let forwarding = Arc::new(ForwardingTable::default());
11285        let supervisor = SupervisorHandle::new();
11286        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11287        let handler =
11288            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11289                .with_supervisor(supervisor);
11290
11291        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11292        hello_via_sink(
11293            &handler,
11294            &target_ctx,
11295            &mut target_rx,
11296            hello_frame("target", PROTOCOL_VERSION, 1),
11297        )
11298        .await;
11299
11300        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11301        let direct_handler = handler.clone();
11302        let direct_open = tokio::spawn(async move {
11303            direct_handler
11304                .handle_control_frame(
11305                    &direct_ctx,
11306                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
11307                )
11308                .await
11309                .unwrap()
11310        });
11311        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11312            .await
11313            .expect("no direct route.bind within 5s")
11314            .expect("target control channel closed before direct route.bind");
11315        handler
11316            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11317            .await
11318            .unwrap();
11319        assert!(direct_open.await.unwrap().is_empty());
11320        let _ = direct_rx.recv().await.unwrap();
11321
11322        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11323        let reserved_handler = handler.clone();
11324        let reserved_open = tokio::spawn(async move {
11325            reserved_handler
11326                .handle_control_frame(
11327                    &reserved_ctx,
11328                    route_open_frame_with_admission_facts(
11329                        3,
11330                        "target",
11331                        unique_project_root("admission-facts"),
11332                        Some(ConsumerIdentity {
11333                            module_id: "fed".to_string(),
11334                            launch_nonce: "fed-nonce".to_string(),
11335                        }),
11336                        None,
11337                    ),
11338                )
11339                .await
11340                .unwrap()
11341        });
11342        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11343            .await
11344            .expect("no reserved route.bind within 5s")
11345            .expect("target control channel closed before reserved route.bind");
11346        handler
11347            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11348            .await
11349            .unwrap();
11350        assert!(reserved_open.await.unwrap().is_empty());
11351        let _ = reserved_rx.recv().await.unwrap();
11352
11353        forwarding
11354            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11355            .unwrap();
11356        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11357        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11358            module_id: Some("target".to_string()),
11359        })
11360        .unwrap();
11361        let census_frame =
11362            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11363        let response = handler
11364            .handle_control_frame(&census_ctx, census_frame)
11365            .await
11366            .unwrap()
11367            .pop()
11368            .unwrap();
11369        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11370        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11371        assert!(matches!(
11372            decoded,
11373            ClientControlResponse::SupervisorRoutes { .. }
11374        ));
11375        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11376        assert_eq!(routes.len(), 2);
11377        assert!(routes.iter().all(|route| route["draining"] == true));
11378        // The census carries WHY: the reason the drain was begun with, in the
11379        // route.closing vocabulary, on every draining route this drain marked.
11380        assert!(
11381            routes.iter().all(|route| route["drain_reason"] == "reload"),
11382            "draining routes must name the drain's reason: {routes:?}"
11383        );
11384        assert!(routes.iter().any(|route| {
11385            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11386        }));
11387        assert!(routes.iter().any(|route| {
11388            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11389        }));
11390
11391        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11392            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11393        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11394            std::fs::write(
11395                &golden_path,
11396                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11397            )
11398            .unwrap();
11399        }
11400        let expected: Value =
11401            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11402        assert_eq!(actual, expected);
11403    }
11404
11405    async fn query_live_roots(
11406        handler: &ControlHandler,
11407        module_ctx: &RouteCtx,
11408    ) -> ModuleControlResponseToModule {
11409        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11410        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11411        let response = handler
11412            .handle_control_frame(module_ctx, frame)
11413            .await
11414            .unwrap()
11415            .pop()
11416            .unwrap();
11417        serde_json::from_slice(&response.body).unwrap()
11418    }
11419
11420    #[tokio::test(start_paused = true)]
11421    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11422        let registry = Arc::new(Registry::default());
11423        let forwarding = Arc::new(ForwardingTable::default());
11424        let handler = ControlHandler::with_forwarding(registry, forwarding);
11425        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11426        hello_via_sink(
11427            &handler,
11428            &target_ctx,
11429            &mut target_rx,
11430            hello_frame("target", PROTOCOL_VERSION, 1),
11431        )
11432        .await;
11433        let root = unique_project_root("live-roots-known");
11434        let path = ProjectRootId::from_path_allowing_missing(root.path())
11435            .unwrap()
11436            .as_path()
11437            .to_path_buf();
11438        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11439        let open_handler = handler.clone();
11440        let opened = tokio::spawn(async move {
11441            open_handler
11442                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11443                .await
11444                .unwrap()
11445        });
11446        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11447            .await
11448            .unwrap()
11449            .unwrap();
11450        handler
11451            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11452            .await
11453            .unwrap();
11454        assert!(opened.await.unwrap().is_empty());
11455        let _ = client_rx.recv().await.unwrap();
11456
11457        let root = unique_project_root("live-roots-pending");
11458        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11459            .unwrap()
11460            .as_path()
11461            .to_path_buf();
11462        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11463        let open_handler = handler.clone();
11464        let pending = tokio::spawn(async move {
11465            open_handler
11466                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11467                .await
11468                .unwrap()
11469        });
11470        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11471            .await
11472            .unwrap()
11473            .unwrap();
11474        let actual = query_live_roots(&handler, &target_ctx).await;
11475        let ModuleControlResponseToModule::LiveRoots {
11476            roots,
11477            unknown_root_bindings,
11478            total_bindings,
11479        } = actual
11480        else {
11481            panic!("expected live roots")
11482        };
11483        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11484        assert_eq!(unknown_root_bindings, 0);
11485        assert_eq!(
11486            roots.len(),
11487            2,
11488            "root-known arm must retain each canonical root"
11489        );
11490        assert_eq!(
11491            total_bindings,
11492            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11493        );
11494        let counts = roots
11495            .iter()
11496            .map(|root| (root.project_root.clone(), root.bound, root.pending))
11497            .collect::<Vec<_>>();
11498        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11499        expected.sort_by(|a, b| a.0.cmp(&b.0));
11500        assert_eq!(
11501            counts, expected,
11502            "roots must sort by path and count pending separately"
11503        );
11504        handler
11505            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11506            .await
11507            .unwrap();
11508        assert!(pending.await.unwrap().is_empty());
11509    }
11510
11511    #[tokio::test(start_paused = true)]
11512    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11513        let forwarding = Arc::new(ForwardingTable::default());
11514        let handler =
11515            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11516        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11517        hello_via_sink(
11518            &handler,
11519            &target_ctx,
11520            &mut target_rx,
11521            hello_frame("target", PROTOCOL_VERSION, 1),
11522        )
11523        .await;
11524        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11525        let pending = forwarding
11526            .begin_route_bind_relay_for_test(
11527                client_ctx.connection_id,
11528                client_ctx.egress.clone(),
11529                2,
11530                "target",
11531            )
11532            .unwrap();
11533        forwarding
11534            .complete_pending_relay(
11535                target_ctx.connection_id,
11536                pending.corr,
11537                RouteBindRelayOutcome::Accepted,
11538            )
11539            .unwrap();
11540        let actual = query_live_roots(&handler, &target_ctx).await;
11541        let ModuleControlResponseToModule::LiveRoots {
11542            roots,
11543            unknown_root_bindings,
11544            total_bindings,
11545        } = actual
11546        else {
11547            panic!("expected live roots")
11548        };
11549        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11550        assert_eq!(
11551            unknown_root_bindings, 1,
11552            "unknown-root arm must not read as no bindings"
11553        );
11554        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11555        assert_eq!(
11556            total_bindings,
11557            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11558        );
11559    }
11560
11561    /// A module reads HELLO_ACK as its first frame and exits on anything else,
11562    /// so the ack has to be on its outbound queue before the module is
11563    /// routable. The connection loop writes a handler's replies only after the
11564    /// handler returns; this test stops in exactly that gap, runs a real
11565    /// route.open from another connection, and only then writes whatever the
11566    /// HELLO handler returned, the way the loop would. If the ack were still a
11567    /// reply, the route.bind request would reach the module first.
11568    #[tokio::test(start_paused = true)]
11569    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11570        let forwarding = Arc::new(ForwardingTable::default());
11571        let handler =
11572            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11573        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11574        let replies = handler
11575            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11576            .await
11577            .unwrap();
11578        let queued_by_hello = module_rx.len();
11579
11580        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11581        let open_handler = handler.clone();
11582        let open = tokio::spawn(async move {
11583            open_handler
11584                .handle_control_frame(
11585                    &client_ctx,
11586                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11587                )
11588                .await
11589                .unwrap()
11590        });
11591        // Let the route.open run until its route.bind is on the module's queue.
11592        let mut spins = 0;
11593        while module_rx.len() == queued_by_hello {
11594            spins += 1;
11595            assert!(spins < 10_000, "route.open never queued a route.bind");
11596            tokio::task::yield_now().await;
11597        }
11598
11599        // Now the connection loop's half: write the HELLO handler's replies.
11600        for reply in replies {
11601            module_ctx.egress.send(reply).await.unwrap();
11602        }
11603
11604        let first = module_rx.recv().await.unwrap().frame;
11605        assert_eq!(
11606            first.header.ty,
11607            FrameType::HelloAck,
11608            "the first frame a registering module reads must be its HELLO_ACK"
11609        );
11610        assert_eq!(first.header.corr, 7);
11611        let second = module_rx.recv().await.unwrap().frame;
11612        assert_eq!(second.header.ty, FrameType::Request);
11613        assert!(
11614            matches!(
11615                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11616                ModuleControlRequest::RouteBind { .. }
11617            ),
11618            "the route.bind follows the ack"
11619        );
11620        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11621
11622        handler
11623            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11624            .await
11625            .unwrap();
11626        assert!(open.await.unwrap().is_empty());
11627        let _ = client_rx.recv().await.unwrap();
11628    }
11629
11630    #[tokio::test(start_paused = true)]
11631    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11632        let handler = ControlHandler::with_forwarding(
11633            Arc::new(Registry::default()),
11634            Arc::new(ForwardingTable::default()),
11635        );
11636        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11637        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11638        hello_via_sink(
11639            &handler,
11640            &first_ctx,
11641            &mut first_rx,
11642            hello_frame("first", PROTOCOL_VERSION, 1),
11643        )
11644        .await;
11645        hello_via_sink(
11646            &handler,
11647            &second_ctx,
11648            &mut second_rx,
11649            hello_frame("second", PROTOCOL_VERSION, 2),
11650        )
11651        .await;
11652        let root = unique_project_root("second-only");
11653        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11654        let cloned = handler.clone();
11655        let open = tokio::spawn(async move {
11656            cloned
11657                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11658                .await
11659                .unwrap()
11660        });
11661        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11662            .await
11663            .unwrap()
11664            .unwrap();
11665        let first = query_live_roots(&handler, &first_ctx).await;
11666        let second = query_live_roots(&handler, &second_ctx).await;
11667        assert!(
11668            matches!(
11669                first,
11670                ModuleControlResponseToModule::LiveRoots {
11671                    total_bindings: 0,
11672                    ..
11673                }
11674            ),
11675            "cross-module scope must not expose another module's roots"
11676        );
11677        assert!(
11678            matches!(
11679                second,
11680                ModuleControlResponseToModule::LiveRoots {
11681                    total_bindings: 1,
11682                    ..
11683                }
11684            ),
11685            "second module must see its pending route"
11686        );
11687        handler
11688            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11689            .await
11690            .unwrap();
11691        assert!(open.await.unwrap().is_empty());
11692    }
11693
11694    #[tokio::test(start_paused = true)]
11695    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11696        let handler = ControlHandler::with_forwarding(
11697            Arc::new(Registry::default()),
11698            Arc::new(ForwardingTable::default()),
11699        );
11700        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11701        hello_via_sink(
11702            &handler,
11703            &target_ctx,
11704            &mut target_rx,
11705            hello_frame("target", PROTOCOL_VERSION, 1),
11706        )
11707        .await;
11708        let actual = query_live_roots(&handler, &target_ctx).await;
11709        let ModuleControlResponseToModule::LiveRoots {
11710            roots,
11711            unknown_root_bindings,
11712            total_bindings,
11713        } = actual
11714        else {
11715            panic!("expected live roots")
11716        };
11717        assert!(roots.is_empty());
11718        assert_eq!(unknown_root_bindings, 0);
11719        assert_eq!(total_bindings, 0);
11720        assert_eq!(
11721            total_bindings,
11722            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11723        );
11724    }
11725
11726    /// Read the vendored fed corpus rather than hand-building a package.
11727    ///
11728    /// A hand-built object encodes what the test author believed the carrier
11729    /// emits. These vectors are what it actually emits, and one of them exists
11730    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11731    /// ignores additive unknown fields at the traversal emit terminus."
11732    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11733        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11734            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11735        let text = std::fs::read_to_string(&path)
11736            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11737        let vectors: Vec<(String, Value)> = text
11738            .lines()
11739            .filter(|line| !line.trim().is_empty())
11740            .map(|line| {
11741                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11742                let id = entry["corpus_id"]
11743                    .as_str()
11744                    .expect("every vector carries a corpus_id")
11745                    .to_string();
11746                (id, entry["package"].clone())
11747            })
11748            .collect();
11749        // Pin the count: a corpus that silently shrinks would take its coverage
11750        // with it, and a suite reading N-1 vectors reports the same clean pass
11751        // as one reading N.
11752        assert_eq!(
11753            vectors.len(),
11754            3,
11755            "vendored fed corpus changed size; re-sync from subc-federation"
11756        );
11757
11758        // Pin what makes the corpus DISCRIMINATING, not just present.
11759        //
11760        // The relay test below takes its expected value from the corpus, so the
11761        // corpus supplies the test's power to detect a lossy relay rather than
11762        // its correctness. A relay that dropped unrecognised fields would still
11763        // be caught -- but only by a package carrying fields it does not know.
11764        // Shrink every package to the handful of keys any implementation would
11765        // recognise and the test keeps passing over an input that can no longer
11766        // fail, which is the same clean green as a corpus that shrank away.
11767        //
11768        // So assert the precondition rather than duplicating the packages here:
11769        // at least one vector must carry a field beyond the small common set.
11770        // That is one claim to maintain instead of nine, and it fails loudly if
11771        // a re-sync ever flattens the corpus.
11772        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11773        let richest = vectors
11774            .iter()
11775            .filter_map(|(_, package)| package.as_object())
11776            .map(|object| {
11777                object
11778                    .keys()
11779                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11780                    .count()
11781            })
11782            .max()
11783            .unwrap_or(0);
11784        assert!(
11785            richest >= 2,
11786            "vendored corpus no longer carries a package with unmodelled fields, \
11787             so the relay test can no longer distinguish a verbatim relay from a lossy one"
11788        );
11789
11790        vectors
11791    }
11792
11793    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
11794    ///
11795    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
11796    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
11797    /// three-key object, so a relay that quietly dropped fields it did not
11798    /// recognise would satisfy it. These vectors carry nine keys including ones
11799    /// this crate has no type for, so a typed relay fails here and only here.
11800    #[tokio::test]
11801    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11802        for (corpus_id, package) in fed_admission_facts_vectors() {
11803            let registry = Arc::new(Registry::default());
11804            let forwarding = Arc::new(ForwardingTable::default());
11805            let supervisor = SupervisorHandle::new();
11806            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11807            let handler =
11808                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11809                    .with_supervisor(supervisor)
11810                    .with_admission_facts_config(
11811                        Some("fed".to_string()),
11812                        Some(vec!["target".to_string()]),
11813                    );
11814
11815            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11816            hello_via_sink(
11817                &handler,
11818                &target_ctx,
11819                &mut target_rx,
11820                hello_frame("target", PROTOCOL_VERSION, 1),
11821            )
11822            .await;
11823
11824            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11825            let route_handler = handler.clone();
11826            let expected = package.clone();
11827            let route_task = tokio::spawn(async move {
11828                route_handler
11829                    .handle_control_frame(
11830                        &client_ctx,
11831                        route_open_frame_with_admission_facts(
11832                            20,
11833                            "target",
11834                            unique_project_root("admission-facts"),
11835                            Some(subc_control::ConsumerIdentity {
11836                                module_id: "fed".to_string(),
11837                                launch_nonce: "fed-nonce".to_string(),
11838                            }),
11839                            Some(package),
11840                        ),
11841                    )
11842                    .await
11843                    .unwrap()
11844            });
11845
11846            let bind_frame = target_rx.recv().await.unwrap();
11847            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11848            let ModuleControlRequest::RouteBind {
11849                admission_facts, ..
11850            } = bind
11851            else {
11852                panic!("{corpus_id}: expected route.bind")
11853            };
11854            assert_eq!(
11855                admission_facts,
11856                Some(expected),
11857                "{corpus_id}: relay must not add, drop or reshape any field"
11858            );
11859
11860            handler
11861                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11862                .await
11863                .unwrap();
11864            route_task.await.unwrap();
11865        }
11866    }
11867
11868    #[tokio::test]
11869    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11870        let registry = Arc::new(Registry::default());
11871        let forwarding = Arc::new(ForwardingTable::default());
11872        let supervisor = SupervisorHandle::new();
11873        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11874        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11875        let handler =
11876            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11877                .with_supervisor(supervisor)
11878                .with_admission_facts_config(
11879                    Some("fed".to_string()),
11880                    Some(vec!["target".to_string()]),
11881                );
11882
11883        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11884        hello_via_sink(
11885            &handler,
11886            &target_ctx,
11887            &mut target_rx,
11888            hello_frame("target", PROTOCOL_VERSION, 1),
11889        )
11890        .await;
11891        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11892        hello_via_sink(
11893            &handler,
11894            &other_ctx,
11895            &mut other_rx,
11896            hello_frame("other", PROTOCOL_VERSION, 2),
11897        )
11898        .await;
11899
11900        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11901        let expected_facts = facts.clone();
11902        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11903        let route_handler = handler.clone();
11904        let route_task = tokio::spawn(async move {
11905            route_handler
11906                .handle_control_frame(
11907                    &client_ctx,
11908                    route_open_frame_with_admission_facts(
11909                        10,
11910                        "target",
11911                        unique_project_root("admission-facts"),
11912                        Some(subc_control::ConsumerIdentity {
11913                            module_id: "fed".to_string(),
11914                            launch_nonce: "fed-nonce".to_string(),
11915                        }),
11916                        Some(facts.clone()),
11917                    ),
11918                )
11919                .await
11920                .unwrap()
11921        });
11922        let bind_frame = target_rx.recv().await.unwrap();
11923        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11924        let ModuleControlRequest::RouteBind {
11925            admission_facts, ..
11926        } = bind
11927        else {
11928            panic!("expected route.bind")
11929        };
11930        assert_eq!(admission_facts, Some(expected_facts));
11931        handler
11932            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11933            .await
11934            .unwrap();
11935        assert!(route_task.await.unwrap().is_empty());
11936        assert!(matches!(
11937            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11938                .unwrap(),
11939            ClientControlResponse::RouteOpen { .. }
11940        ));
11941
11942        let direct = handler
11943            .handle_control_frame(
11944                &route_ctx(ConnectionId::new(73)).0,
11945                route_open_frame_with_admission_facts(
11946                    11,
11947                    "target",
11948                    unique_project_root("admission-facts"),
11949                    None,
11950                    Some(json!({"x": 1})),
11951                ),
11952            )
11953            .await
11954            .unwrap();
11955        assert_eq!(
11956            parse_error(&direct[0])["code"],
11957            "admission_facts_not_permitted"
11958        );
11959
11960        let different_reserved = handler
11961            .handle_control_frame(
11962                &route_ctx(ConnectionId::new(77)).0,
11963                route_open_frame_with_admission_facts(
11964                    15,
11965                    "target",
11966                    unique_project_root("admission-facts"),
11967                    Some(subc_control::ConsumerIdentity {
11968                        module_id: "other".to_string(),
11969                        launch_nonce: "other-nonce".to_string(),
11970                    }),
11971                    Some(json!({"x": 1})),
11972                ),
11973            )
11974            .await
11975            .unwrap();
11976        assert_eq!(
11977            parse_error(&different_reserved[0])["code"],
11978            "admission_facts_not_permitted"
11979        );
11980
11981        let other_target = handler
11982            .handle_control_frame(
11983                &route_ctx(ConnectionId::new(74)).0,
11984                route_open_frame_with_admission_facts(
11985                    12,
11986                    "other",
11987                    unique_project_root("admission-facts"),
11988                    Some(subc_control::ConsumerIdentity {
11989                        module_id: "fed".to_string(),
11990                        launch_nonce: "fed-nonce".to_string(),
11991                    }),
11992                    Some(json!({"x": 1})),
11993                ),
11994            )
11995            .await
11996            .unwrap();
11997        assert_eq!(
11998            parse_error(&other_target[0])["code"],
11999            "admission_facts_target_not_allowed"
12000        );
12001
12002        let nonexistent = handler
12003            .handle_control_frame(
12004                &route_ctx(ConnectionId::new(75)).0,
12005                route_open_frame_with_admission_facts(
12006                    13,
12007                    "missing",
12008                    unique_project_root("admission-facts"),
12009                    None,
12010                    Some(json!({"x": 1})),
12011                ),
12012            )
12013            .await
12014            .unwrap();
12015        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12016
12017        let described = handler
12018            .handle_control_frame(
12019                &route_ctx(ConnectionId::new(76)).0,
12020                Frame::build(
12021                    FrameType::Request,
12022                    control_flags(),
12023                    0,
12024                    0,
12025                    14,
12026                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12027                )
12028                .unwrap(),
12029            )
12030            .await
12031            .unwrap();
12032        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12033            serde_json::from_slice(&described[0].body).unwrap()
12034        else {
12035            panic!("expected server.describe response")
12036        };
12037        assert!(capabilities
12038            .iter()
12039            .any(|cap| cap == "admission_facts_relay_v1"));
12040    }
12041
12042    #[tokio::test]
12043    async fn admission_facts_without_configured_carrier_are_rejected() {
12044        let registry = Arc::new(Registry::default());
12045        let forwarding = Arc::new(ForwardingTable::default());
12046        let handler = ControlHandler::with_forwarding(registry, forwarding);
12047        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12048        hello_via_sink(
12049            &handler,
12050            &target_ctx,
12051            &mut target_rx,
12052            hello_frame("target", PROTOCOL_VERSION, 1),
12053        )
12054        .await;
12055
12056        let responses = handler
12057            .handle_control_frame(
12058                &route_ctx(ConnectionId::new(79)).0,
12059                route_open_frame_with_admission_facts(
12060                    16,
12061                    "target",
12062                    unique_project_root("admission-facts"),
12063                    None,
12064                    Some(json!({"x": 1})),
12065                ),
12066            )
12067            .await
12068            .unwrap();
12069        assert_eq!(
12070            parse_error(&responses[0])["code"],
12071            "admission_facts_not_permitted"
12072        );
12073    }
12074
12075    #[tokio::test]
12076    async fn route_open_relays_consumer_capabilities_verbatim() {
12077        let registry = Arc::new(Registry::default());
12078        let forwarding = Arc::new(ForwardingTable::default());
12079        let handler =
12080            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12081        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12082        hello_via_sink(
12083            &handler,
12084            &module_ctx,
12085            &mut module_rx,
12086            hello_frame("aft", PROTOCOL_VERSION, 7),
12087        )
12088        .await;
12089
12090        let expected = vec!["elicitation".to_string(), "roots".to_string()];
12091        let expected_for_request = expected.clone();
12092        let project_root = unique_project_root("consumer-capabilities-present");
12093        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12094        let route_handler = handler.clone();
12095        let route_task = tokio::spawn(async move {
12096            route_handler
12097                .handle_control_frame(
12098                    &client_ctx,
12099                    route_open_frame_with_consumer_capabilities(
12100                        401,
12101                        "aft",
12102                        project_root,
12103                        Some(expected_for_request),
12104                    ),
12105                )
12106                .await
12107                .unwrap()
12108        });
12109        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12110            .await
12111            .unwrap()
12112            .unwrap();
12113        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12114        let ModuleControlRequest::RouteBind {
12115            consumer_capabilities,
12116            ..
12117        } = bind
12118        else {
12119            panic!("expected route.bind request, got {bind:?}");
12120        };
12121        assert_eq!(consumer_capabilities, Some(expected.clone()));
12122
12123        handler
12124            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12125            .await
12126            .unwrap();
12127        let route_response = route_task.await.unwrap();
12128        assert!(route_response.is_empty());
12129        let published = client_rx.recv().await.unwrap();
12130        assert!(matches!(
12131            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12132            ClientControlResponse::RouteOpen { .. }
12133        ));
12134    }
12135
12136    #[tokio::test]
12137    async fn route_open_without_consumer_capabilities_relays_none() {
12138        let registry = Arc::new(Registry::default());
12139        let forwarding = Arc::new(ForwardingTable::default());
12140        let handler =
12141            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12142        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12143        hello_via_sink(
12144            &handler,
12145            &module_ctx,
12146            &mut module_rx,
12147            hello_frame("aft", PROTOCOL_VERSION, 7),
12148        )
12149        .await;
12150
12151        let project_root = unique_project_root("consumer-capabilities-absent");
12152        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12153        let route_handler = handler.clone();
12154        let route_task = tokio::spawn(async move {
12155            route_handler
12156                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12157                .await
12158                .unwrap()
12159        });
12160        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12161            .await
12162            .unwrap()
12163            .unwrap();
12164        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12165        let ModuleControlRequest::RouteBind {
12166            consumer_capabilities,
12167            ..
12168        } = bind
12169        else {
12170            panic!("expected route.bind request, got {bind:?}");
12171        };
12172        assert_eq!(consumer_capabilities, None);
12173
12174        handler
12175            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12176            .await
12177            .unwrap();
12178        let route_response = route_task.await.unwrap();
12179        assert!(route_response.is_empty());
12180        let published = client_rx.recv().await.unwrap();
12181        assert!(matches!(
12182            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12183            ClientControlResponse::RouteOpen { .. }
12184        ));
12185    }
12186
12187    /// Opens a route to a freshly registered `aft` with `sent` as the
12188    /// route.open's role_versions, acks the bind, and returns the role_versions
12189    /// the module's bind carried.
12190    async fn bind_role_versions_for(
12191        sent: Option<BTreeMap<String, String>>,
12192        connection: u64,
12193    ) -> Option<BTreeMap<String, String>> {
12194        let registry = Arc::new(Registry::default());
12195        let forwarding = Arc::new(ForwardingTable::default());
12196        let handler =
12197            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12198        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12199        hello_via_sink(
12200            &handler,
12201            &module_ctx,
12202            &mut module_rx,
12203            hello_frame("aft", PROTOCOL_VERSION, 7),
12204        )
12205        .await;
12206        let project_root = unique_project_root("role-versions");
12207        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12208        let route_handler = handler.clone();
12209        let route_task = tokio::spawn(async move {
12210            route_handler
12211                .handle_control_frame(
12212                    &client_ctx,
12213                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
12214                )
12215                .await
12216                .unwrap()
12217        });
12218        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12219            .await
12220            .expect("a well-formed route.open reaches the module as a bind")
12221            .unwrap();
12222        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12223        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12224            panic!("expected route.bind request, got {bind:?}");
12225        };
12226        handler
12227            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12228            .await
12229            .unwrap();
12230        assert!(route_task.await.unwrap().is_empty());
12231        let published = client_rx.recv().await.unwrap();
12232        assert!(matches!(
12233            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12234            ClientControlResponse::RouteOpen { .. }
12235        ));
12236        role_versions
12237    }
12238
12239    #[tokio::test]
12240    async fn route_open_relays_role_versions_verbatim() {
12241        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12242        assert_eq!(
12243            bind_role_versions_for(Some(sent.clone()), 141).await,
12244            Some(sent)
12245        );
12246    }
12247
12248    /// An empty map declares nothing, so the provider sees no field rather
12249    /// than an empty object it would have to treat as a second "none".
12250    #[tokio::test]
12251    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12252        assert_eq!(bind_role_versions_for(None, 143).await, None);
12253        assert_eq!(
12254            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12255            None
12256        );
12257    }
12258
12259    #[tokio::test]
12260    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12261        let registry = Arc::new(Registry::default());
12262        let forwarding = Arc::new(ForwardingTable::default());
12263        let handler =
12264            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12265        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12266        hello_via_sink(
12267            &handler,
12268            &module_ctx,
12269            &mut module_rx,
12270            hello_frame("aft", PROTOCOL_VERSION, 7),
12271        )
12272        .await;
12273
12274        let nine: BTreeMap<String, String> = (0..9)
12275            .map(|index| (format!("role-{index}"), "v1".to_string()))
12276            .collect();
12277        for (label, malformed) in [
12278            (
12279                "invalid role name",
12280                role_versions(&[("Tool_Provider", "v1")]),
12281            ),
12282            ("invalid version", role_versions(&[("tool-provider", "v0")])),
12283            ("nine entries", nine),
12284        ] {
12285            // A refused open answers at once; one that reached the module
12286            // would wait for its bind ack and trip this timeout.
12287            let responses = tokio::time::timeout(
12288                Duration::from_secs(1),
12289                handler.handle_control_frame(
12290                    &route_ctx(ConnectionId::new(148)).0,
12291                    route_open_frame_with_role_versions(
12292                        404,
12293                        "aft",
12294                        unique_project_root("role-versions-malformed"),
12295                        Some(malformed),
12296                    ),
12297                ),
12298            )
12299            .await
12300            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12301            .unwrap();
12302            assert_eq!(responses.len(), 1, "{label}");
12303            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12304            let error = parse_error(&responses[0]);
12305            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12306            assert_eq!(
12307                error["detail"]["field"], "role_versions",
12308                "{label}: {error}"
12309            );
12310            assert!(
12311                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12312                "{label}: a malformed declaration is terminal"
12313            );
12314            assert!(
12315                module_rx.try_recv().is_err(),
12316                "{label}: the module must never see a bind"
12317            );
12318        }
12319    }
12320
12321    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
12322    /// consumer can tell this daemon forwards the field from one that would
12323    /// drop it.
12324    #[tokio::test]
12325    async fn route_role_versions_capability_is_advertised() {
12326        let handler = ControlHandler::new(Arc::new(Registry::default()));
12327        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12328        let ack = hello_via_sink(
12329            &handler,
12330            &ctx,
12331            &mut rx,
12332            hello_frame("m", PROTOCOL_VERSION, 1),
12333        )
12334        .await;
12335        let ack = parse_ack(&ack);
12336        assert!(
12337            ack.subc_capabilities
12338                .iter()
12339                .any(|c| c == "route-role-versions/v1"),
12340            "{:?}",
12341            ack.subc_capabilities
12342        );
12343
12344        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12345        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12346        let reply = handler
12347            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12348            .await
12349            .unwrap()
12350            .pop()
12351            .unwrap();
12352        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12353            serde_json::from_slice(&reply.body).unwrap()
12354        else {
12355            panic!("not a server.describe reply");
12356        };
12357        assert!(
12358            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12359            "{capabilities:?}"
12360        );
12361    }
12362
12363    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12364    /// catalog entry without `events` (the module declares none) from a daemon
12365    /// that never reports declared events.
12366    #[tokio::test]
12367    async fn catalog_events_capability_is_advertised() {
12368        let handler = ControlHandler::new(Arc::new(Registry::default()));
12369        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12370        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12371        let reply = handler
12372            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12373            .await
12374            .unwrap()
12375            .pop()
12376            .unwrap();
12377        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12378            serde_json::from_slice(&reply.body).unwrap()
12379        else {
12380            panic!("not a server.describe reply");
12381        };
12382        assert!(
12383            capabilities
12384                .iter()
12385                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12386            "{capabilities:?}"
12387        );
12388    }
12389
12390    #[tokio::test]
12391    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12392        let handler = ControlHandler::new(Arc::new(Registry::default()));
12393        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12394        let ack = hello_via_sink(
12395            &handler,
12396            &module,
12397            &mut module_rx,
12398            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12399        )
12400        .await;
12401        let ack = parse_ack(&ack);
12402        assert!(
12403            ack.subc_capabilities
12404                .iter()
12405                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12406            "{:?}",
12407            ack.subc_capabilities
12408        );
12409        assert!(ack
12410            .subc_capabilities
12411            .iter()
12412            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12413
12414        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12415        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12416        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12417        let reply = handler
12418            .handle_control_frame(&client, frame)
12419            .await
12420            .unwrap()
12421            .pop()
12422            .unwrap();
12423        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12424            serde_json::from_slice(&reply.body).unwrap()
12425        else {
12426            panic!("not a server.describe reply");
12427        };
12428        assert!(
12429            capabilities
12430                .iter()
12431                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12432            "{capabilities:?}"
12433        );
12434        assert!(capabilities
12435            .iter()
12436            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12437    }
12438
12439    #[tokio::test]
12440    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12441        let handler = ControlHandler::new(Arc::new(Registry::default()));
12442        let (module, mut rx) = route_ctx(ConnectionId::new(156));
12443        let ack = hello_via_sink(
12444            &handler,
12445            &module,
12446            &mut rx,
12447            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12448        )
12449        .await;
12450        assert!(parse_ack(&ack)
12451            .subc_capabilities
12452            .iter()
12453            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12454        assert!(parse_ack(&ack)
12455            .subc_capabilities
12456            .iter()
12457            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12458        let (client, _rx) = route_ctx(ConnectionId::new(157));
12459        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12460        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12461        let reply = handler
12462            .handle_control_frame(&client, frame)
12463            .await
12464            .unwrap()
12465            .pop()
12466            .unwrap();
12467        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12468            serde_json::from_slice(&reply.body).unwrap()
12469        else {
12470            panic!("server.describe reply")
12471        };
12472        assert!(capabilities
12473            .iter()
12474            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12475        assert!(capabilities
12476            .iter()
12477            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12478    }
12479
12480    #[tokio::test]
12481    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12482        let handler = ControlHandler::new(Arc::new(Registry::default()));
12483        let (module, mut rx) = route_ctx(ConnectionId::new(158));
12484        let ack = hello_via_sink(
12485            &handler,
12486            &module,
12487            &mut rx,
12488            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12489        )
12490        .await;
12491        assert!(parse_ack(&ack)
12492            .subc_capabilities
12493            .iter()
12494            .any(|cap| cap == "scope-workdir/v1"));
12495        let (client, _rx) = route_ctx(ConnectionId::new(159));
12496        let frame = Frame::build(
12497            FrameType::Request,
12498            control_flags(),
12499            0,
12500            0,
12501            5,
12502            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12503        )
12504        .unwrap();
12505        let reply = handler
12506            .handle_control_frame(&client, frame)
12507            .await
12508            .unwrap()
12509            .remove(0);
12510        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12511            serde_json::from_slice(&reply.body).unwrap()
12512        else {
12513            panic!("server.describe reply")
12514        };
12515        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12516    }
12517
12518    #[tokio::test]
12519    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12520        let handler = ControlHandler::new(Arc::new(Registry::default()));
12521        let (module, mut rx) = route_ctx(ConnectionId::new(154));
12522        let ack = hello_via_sink(
12523            &handler,
12524            &module,
12525            &mut rx,
12526            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12527        )
12528        .await;
12529        assert!(parse_ack(&ack)
12530            .subc_capabilities
12531            .iter()
12532            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12533        let (client, _rx) = route_ctx(ConnectionId::new(155));
12534        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12535        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12536        let reply = handler
12537            .handle_control_frame(&client, frame)
12538            .await
12539            .unwrap()
12540            .pop()
12541            .unwrap();
12542        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12543            serde_json::from_slice(&reply.body).unwrap()
12544        else {
12545            panic!("not a server.describe reply")
12546        };
12547        assert!(capabilities
12548            .iter()
12549            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12550    }
12551
12552    #[tokio::test]
12553    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12554        let registry = Arc::new(Registry::default());
12555        let forwarding = Arc::new(ForwardingTable::default());
12556        let handler =
12557            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12558                .with_health_probe_timeout(Duration::from_secs(5));
12559        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12560        hello_via_sink(
12561            &handler,
12562            &module_ctx,
12563            &mut module_rx,
12564            non_routable_hello_frame_with_control_ops(
12565                "mcp",
12566                300,
12567                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12568            ),
12569        )
12570        .await;
12571        assert!(registry
12572            .get_module("mcp")
12573            .unwrap()
12574            .unwrap()
12575            .manifest
12576            .provides
12577            .is_empty());
12578
12579        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12580        let route_response = handler
12581            .handle_control_frame(
12582                &route_client_ctx,
12583                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12584            )
12585            .await
12586            .unwrap();
12587        assert_eq!(route_response[0].header.ty, FrameType::Error);
12588        assert_eq!(
12589            parse_error(&route_response[0])["code"],
12590            "target_unavailable"
12591        );
12592        assert!(parse_error(&route_response[0])["message"]
12593            .as_str()
12594            .unwrap()
12595            .contains("does not provide the requested target"));
12596        assert!(module_rx.try_recv().is_err());
12597
12598        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12599        let health_handler = handler.clone();
12600        let health_task = tokio::spawn(async move {
12601            health_handler
12602                .handle_control_frame(
12603                    &health_client_ctx,
12604                    supervisor_health_probe_frame(302, "mcp"),
12605                )
12606                .await
12607                .unwrap()
12608        });
12609        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12610            .await
12611            .unwrap()
12612            .unwrap();
12613        assert_eq!(
12614            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12615            ModuleControlRequest::HealthCheck {}
12616        );
12617        handler
12618            .handle_control_frame(
12619                &module_ctx,
12620                health_response(health_frame.header.corr, HealthStatus::Ok),
12621            )
12622            .await
12623            .unwrap();
12624        let health_response = health_task.await.unwrap();
12625        assert_eq!(health_response[0].header.ty, FrameType::Response);
12626        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12627            ClientControlResponse::SupervisorHealthProbe {
12628                module_id, status, ..
12629            } => {
12630                assert_eq!(module_id, "mcp");
12631                assert_eq!(status, HealthStatus::Ok);
12632            }
12633            other => panic!("unexpected health response: {other:?}"),
12634        }
12635
12636        // Exercise the forwarding cleanup path directly while leaving the registry
12637        // advertisement in place. If cleanup leaves a stale control sink behind,
12638        // the next probe will enqueue onto it and wait for the long probe timeout
12639        // instead of returning an immediate no-connection error.
12640        forwarding
12641            .cleanup_connection(module_ctx.connection_id)
12642            .unwrap();
12643        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12644        let cleanup_response = tokio::time::timeout(
12645            Duration::from_millis(200),
12646            handler.handle_control_frame(
12647                &cleanup_probe_ctx,
12648                supervisor_health_probe_frame(303, "mcp"),
12649            ),
12650        )
12651        .await
12652        .expect("probe should fail immediately when the control lane is gone")
12653        .unwrap();
12654        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12655        assert_eq!(
12656            parse_error(&cleanup_response[0])["code"],
12657            "target_unavailable"
12658        );
12659        assert!(parse_error(&cleanup_response[0])["message"]
12660            .as_str()
12661            .unwrap()
12662            .contains("no module connection"));
12663
12664        handler
12665            .cleanup_connection(module_ctx.connection_id)
12666            .unwrap();
12667    }
12668
12669    #[tokio::test]
12670    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12671        let registry = Arc::new(Registry::default());
12672        let supervisor_handle = SupervisorHandle::new();
12673        let supervisor =
12674            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12675                .with_handle(supervisor_handle.clone())
12676                .with_connection_file_path(
12677                    std::env::temp_dir()
12678                        .join(format!("subc-route-open-warming-{}", std::process::id())),
12679                );
12680        let module = supervisor
12681            .supervise_configured(
12682                ModuleSpec {
12683                    module_id: "warming".to_string(),
12684                    program: fake_aft_stub_path(),
12685                    args: Vec::new(),
12686                    env: Vec::new(),
12687                    reserved: false,
12688                    reserved_prefixes: Vec::new(),
12689                    protocol: ModuleProtocol::Subc,
12690                    overlap: Default::default(),
12691                },
12692                true,
12693            )
12694            .unwrap();
12695        assert_eq!(module.state().unwrap(), ModuleState::Running);
12696
12697        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12698        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12699        let response = handler
12700            .handle_control_frame(
12701                &ctx,
12702                route_open_frame(304, "warming", unique_project_root("warming")),
12703            )
12704            .await
12705            .unwrap();
12706        module.stop().await.unwrap();
12707
12708        assert_eq!(response[0].header.ty, FrameType::Error);
12709        let error = parse_error(&response[0]);
12710        assert_eq!(error["code"], "module_warming");
12711        assert!(error["message"]
12712            .as_str()
12713            .unwrap()
12714            .contains("state=running, enabled=true, live=false"));
12715    }
12716
12717    #[test]
12718    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12719        let handler = ControlHandler::new(Arc::new(Registry::default()));
12720        let capture = EventCapture::default();
12721        let _subscriber =
12722            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12723        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12724        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12725        let pending = (0..limit).collect::<Vec<_>>();
12726        let response = handler
12727            .route_open_capacity_refusal(
12728                &ctx,
12729                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12730                "busy",
12731                pending.len(),
12732                limit,
12733            )
12734            .unwrap();
12735        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12736        let event = capture
12737            .events()
12738            .into_iter()
12739            .find(|event| {
12740                event.target == "control"
12741                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12742            })
12743            .expect("connection admission refusal event");
12744        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12745        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12746        // Refused before `handle_route_open` parses the request, so the caller
12747        // fields are read from the frame itself.
12748        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
12749        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
12750    }
12751
12752    #[test]
12753    fn route_open_target_cap_logs_admission_reason_and_capacity() {
12754        let handler = ControlHandler::new(Arc::new(Registry::default()));
12755        let capture = EventCapture::default();
12756        let _subscriber =
12757            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12758        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12759        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12760        let guards = (0..limit)
12761            .map(|_| {
12762                handler
12763                    .route_bind_concurrency
12764                    .try_admit("busy", limit)
12765                    .unwrap()
12766            })
12767            .collect::<Vec<_>>();
12768        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12769            Err(in_flight) => in_flight,
12770            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12771        };
12772        let response = handler
12773            .route_open_target_capacity_refusal(
12774                &ctx,
12775                &RouteOpenCaller::unknown(),
12776                &route_open_frame(397, "busy", unique_project_root("target-cap")),
12777                "busy",
12778                in_flight,
12779            )
12780            .unwrap();
12781        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12782        let event = capture
12783            .events()
12784            .into_iter()
12785            .find(|event| {
12786                event.target == "control"
12787                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12788            })
12789            .expect("target admission refusal event");
12790        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12791        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12792        drop(guards);
12793    }
12794
12795    #[cfg(unix)]
12796    #[tokio::test(start_paused = true)]
12797    async fn supervisor_list_shows_a_failed_modules_next_retry() {
12798        let registry = Arc::new(Registry::default());
12799        let handle = SupervisorHandle::new();
12800        let supervisor =
12801            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
12802                .with_handle(handle.clone());
12803        let module = supervisor
12804            .spawn(crate::ModuleSpec {
12805                module_id: "retry-list".into(),
12806                program: PathBuf::from("/bin/sh"),
12807                args: vec!["-c".into(), "exit 17".into()],
12808                env: vec![],
12809                reserved: false,
12810                reserved_prefixes: vec![],
12811                protocol: ModuleProtocol::None,
12812                overlap: Default::default(),
12813            })
12814            .unwrap();
12815        let deadline = std::time::Instant::now() + Duration::from_secs(10);
12816        while module.state().unwrap() != crate::ModuleState::Failed {
12817            assert!(
12818                std::time::Instant::now() < deadline,
12819                "waiting for budget failure"
12820            );
12821            tokio::task::yield_now().await;
12822        }
12823        let handler = ControlHandler::new(registry).with_supervisor(handle);
12824        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12825        let replies = handler.handle_supervisor_list(frame).await.unwrap();
12826        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12827        assert_eq!(body["modules"][0]["state"], "failed");
12828        let retry = body["modules"][0]["next_retry_at_ms"]
12829            .as_u64()
12830            .expect("scheduled retry on the wire");
12831        let now = std::time::SystemTime::now()
12832            .duration_since(std::time::UNIX_EPOCH)
12833            .unwrap()
12834            .as_millis() as u64;
12835        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
12836        module.set_enabled(false).await.unwrap();
12837        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12838        let replies = handler.handle_supervisor_list(frame).await.unwrap();
12839        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12840        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
12841    }
12842
12843    #[tokio::test]
12844    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12845        let registry = Arc::new(Registry::default());
12846        let handle = SupervisorHandle::new();
12847        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12848            .with_handle(handle.clone());
12849        supervisor
12850            .supervise_configured(
12851                crate::ModuleSpec {
12852                    module_id: "drift".into(),
12853                    program: PathBuf::from("disabled-module"),
12854                    args: Vec::new(),
12855                    env: Vec::new(),
12856                    reserved: false,
12857                    reserved_prefixes: Vec::new(),
12858                    protocol: ModuleProtocol::Subc,
12859                    overlap: Default::default(),
12860                },
12861                false,
12862            )
12863            .unwrap();
12864        let handler = ControlHandler::new(registry).with_supervisor(handle);
12865        for configured in [None, Some(false), Some(true)] {
12866            handler.capability_evaluator.configure(
12867                configured.map(|enabled| ("drift".to_string(), enabled)),
12868                BTreeMap::new(),
12869            );
12870            let frame =
12871                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12872            let replies = handler
12873                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12874                .await
12875                .unwrap();
12876            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12877            assert!(body.get("configured_enabled").is_none());
12878            assert_eq!(body["op"], "supervisor.ack");
12879            let frame =
12880                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12881            let replies = handler.handle_supervisor_list(frame).await.unwrap();
12882            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12883            assert_eq!(
12884                body["modules"][0].get("configured_enabled"),
12885                configured.map(Value::Bool).as_ref()
12886            );
12887        }
12888    }
12889
12890    #[tokio::test(flavor = "current_thread")]
12891    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12892        let handler = ControlHandler::new(Arc::new(Registry::default()));
12893        let capture = EventCapture::default();
12894        let _subscriber =
12895            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12896        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12897        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12898
12899        let _ = handler
12900            .handle_client_control_request(
12901                &ctx,
12902                frame,
12903                ClientControlRequest::SupervisorSetEnabled {
12904                    module_id: "broca".to_string(),
12905                    enabled: false,
12906                },
12907                &mut None,
12908            )
12909            .await
12910            .unwrap();
12911
12912        let events = capture
12913            .events()
12914            .into_iter()
12915            .filter(|event| {
12916                event.target == "control"
12917                    && event.fields.get("message").map(String::as_str)
12918                        == Some("supervisor request received")
12919            })
12920            .collect::<Vec<_>>();
12921        assert_eq!(events.len(), 1, "one supervisor request log line");
12922        let event = &events[0];
12923        assert_eq!(event.level, tracing::Level::INFO);
12924        assert_eq!(
12925            event.fields.get("op"),
12926            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12927        );
12928        assert_eq!(
12929            event.fields.get("module_id"),
12930            Some(&"\"broca\"".to_string())
12931        );
12932        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12933        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12934        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12935    }
12936
12937    #[tokio::test(flavor = "current_thread")]
12938    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12939        let registry = Arc::new(Registry::default());
12940        let module_connection = ConnectionId::new(303);
12941        registry
12942            .register_with_control_ops(
12943                manifest("aft", PROTOCOL_VERSION),
12944                PROTOCOL_VERSION,
12945                module_connection,
12946                module_baseline_control_ops(),
12947            )
12948            .unwrap();
12949        let handler = ControlHandler::new(registry);
12950        let capture = EventCapture::default();
12951        let _subscriber =
12952            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12953        let (ctx, _rx) = route_ctx(module_connection);
12954        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12955
12956        let _ = handler
12957            .handle_client_control_request(
12958                &ctx,
12959                frame,
12960                ClientControlRequest::SupervisorSetEnabled {
12961                    module_id: "broca".to_string(),
12962                    enabled: false,
12963                },
12964                &mut None,
12965            )
12966            .await
12967            .unwrap();
12968
12969        let callers = capture
12970            .events()
12971            .into_iter()
12972            .filter(|event| {
12973                event.fields.get("message").map(String::as_str)
12974                    == Some("supervisor request received")
12975            })
12976            .map(|event| event.fields.get("caller").cloned())
12977            .collect::<Vec<_>>();
12978        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12979    }
12980
12981    #[tokio::test(flavor = "current_thread")]
12982    async fn supervisor_list_does_not_log_request_received() {
12983        let handler = ControlHandler::new(Arc::new(Registry::default()));
12984        let capture = EventCapture::default();
12985        let _subscriber =
12986            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12987        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12988        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12989
12990        handler
12991            .handle_client_control_request(
12992                &ctx,
12993                frame,
12994                ClientControlRequest::SupervisorList {},
12995                &mut None,
12996            )
12997            .await
12998            .unwrap();
12999
13000        assert!(capture.events().into_iter().all(|event| {
13001            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13002        }));
13003    }
13004
13005    #[tokio::test(flavor = "current_thread")]
13006    async fn supervisor_rescan_preview_does_not_log_request_received() {
13007        let handler = ControlHandler::new(Arc::new(Registry::default()));
13008        let capture = EventCapture::default();
13009        let _subscriber =
13010            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13011        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13012        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13013
13014        handler
13015            .handle_client_control_request(
13016                &ctx,
13017                frame,
13018                ClientControlRequest::SupervisorRescan { preview: true },
13019                &mut None,
13020            )
13021            .await
13022            .unwrap();
13023
13024        assert!(capture.events().into_iter().all(|event| {
13025            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13026        }));
13027    }
13028
13029    struct ApproveOperator;
13030    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13031        fn prompt(
13032            &self,
13033            _: &str,
13034            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13035        ) -> crate::operator_confirm::ProviderResult {
13036            crate::operator_confirm::ProviderResult::Approved
13037        }
13038    }
13039
13040    #[tokio::test]
13041    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13042        assert_eq!(
13043            ControlHandler::default().privileged_operator_prompt,
13044            PrivilegedOperatorPrompt::Log
13045        );
13046        for (initial, changed, running) in [
13047            ("log", "enforce", PrivilegedOperatorPrompt::Log),
13048            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13049        ] {
13050            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13051            let path = dir.join("subc.jsonc");
13052            std::fs::write(
13053                &path,
13054                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13055            )
13056            .unwrap();
13057            let config = crate::daemon_config::load(&path).unwrap().unwrap();
13058            let registry = Arc::new(Registry::default());
13059            let handle = SupervisorHandle::new();
13060            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13061                .with_handle(handle.clone());
13062            let handler = ControlHandler::new(registry)
13063                .with_supervisor(handle)
13064                .with_privileged_operator_prompt(config.privileged_operator_prompt)
13065                .with_supervisor_rescan(supervisor, &path, config.port);
13066            handler.forwarding.operator_confirms().configure(
13067                Arc::new(ApproveOperator),
13068                crate::operator_confirm::OperatorLimits::default(),
13069            );
13070            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13071
13072            for (disk_mode, needs_restart) in [
13073                (Some(initial), false),
13074                (Some(changed), true),
13075                (Some(changed), true),
13076                (None, running == PrivilegedOperatorPrompt::Enforce),
13077                (Some(initial), false),
13078            ] {
13079                let mut doc = json!({"version":1});
13080                if let Some(mode) = disk_mode {
13081                    doc["privileged_operator_prompt"] = json!(mode);
13082                }
13083                std::fs::write(&path, doc.to_string()).unwrap();
13084                for preview in [true, false] {
13085                    let request = ClientControlRequest::SupervisorRescan { preview };
13086                    let frame = Frame::build(
13087                        FrameType::Request,
13088                        control_flags(),
13089                        0,
13090                        0,
13091                        1,
13092                        serde_json::to_vec(&request).unwrap(),
13093                    )
13094                    .unwrap();
13095                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13096                    assert_eq!(frames.len(), 1);
13097                    let ClientControlResponse::SupervisorRescan { result } =
13098                        serde_json::from_slice(&frames[0].body).unwrap()
13099                    else {
13100                        panic!("expected a rescan response");
13101                    };
13102                    let expected = if needs_restart {
13103                        vec!["privileged_operator_prompt"]
13104                    } else {
13105                        Vec::new()
13106                    };
13107                    assert_eq!(result.restart_required, expected);
13108                    assert_eq!(handler.privileged_operator_prompt, running);
13109                    assert_eq!(
13110                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13111                        running
13112                    );
13113                }
13114            }
13115        }
13116    }
13117
13118    #[tokio::test]
13119    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13120        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13121        let path = dir.join("subc.jsonc");
13122        let registry = Arc::new(Registry::default());
13123        let handle = SupervisorHandle::new();
13124        let supervisor =
13125            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13126        let handler = ControlHandler::new(registry)
13127            .with_supervisor(handle)
13128            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13129            .with_supervisor_rescan(supervisor, &path, None);
13130        handler.forwarding.operator_confirms().configure(
13131            Arc::new(ApproveOperator),
13132            crate::operator_confirm::OperatorLimits::default(),
13133        );
13134        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13135        for preview in [true, false] {
13136            std::fs::write(
13137                &path,
13138                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13139            )
13140            .unwrap();
13141            let request = ClientControlRequest::SupervisorRescan { preview };
13142            let frame = Frame::build(
13143                FrameType::Request,
13144                control_flags(),
13145                0,
13146                0,
13147                1,
13148                serde_json::to_vec(&request).unwrap(),
13149            )
13150            .unwrap();
13151            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13152            assert_eq!(frames.len(), 1);
13153            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13154            assert_eq!(
13155                handler.privileged_operator_prompt,
13156                PrivilegedOperatorPrompt::Enforce
13157            );
13158        }
13159    }
13160
13161    /// One wire code has several senders, so the refusal line names the check
13162    /// that refused. This drives the shared refusal path for ordinary refusals
13163    /// with an unregistered
13164    /// target and requires the branch label on the event.
13165    #[tokio::test]
13166    async fn route_open_refusal_names_the_check_that_refused() {
13167        let handler = ControlHandler::new(Arc::new(Registry::default()));
13168        let capture = EventCapture::default();
13169        let _subscriber =
13170            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13171        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13172        let response = handler
13173            .handle_control_frame(
13174                &ctx,
13175                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13176            )
13177            .await
13178            .unwrap();
13179
13180        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13181        let event = capture
13182            .events()
13183            .into_iter()
13184            .find(|event| {
13185                event.target == "control"
13186                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13187            })
13188            .expect("route.open refusal event");
13189        assert_eq!(
13190            event.fields.get("reason"),
13191            Some(&"\"not_registered\"".to_string())
13192        );
13193    }
13194
13195    #[tokio::test]
13196    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13197        let registry = Arc::new(Registry::default());
13198        let supervisor_handle = SupervisorHandle::new();
13199        let supervisor =
13200            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13201                .with_handle(supervisor_handle.clone())
13202                .with_connection_file_path(std::env::temp_dir().join(format!(
13203                    "subc-route-open-refusal-info-{}",
13204                    std::process::id()
13205                )));
13206        let module = supervisor
13207            .supervise_configured(
13208                ModuleSpec {
13209                    module_id: "warming".to_string(),
13210                    program: fake_aft_stub_path(),
13211                    args: Vec::new(),
13212                    env: Vec::new(),
13213                    reserved: false,
13214                    reserved_prefixes: Vec::new(),
13215                    protocol: ModuleProtocol::Subc,
13216                    overlap: Default::default(),
13217                },
13218                true,
13219            )
13220            .unwrap();
13221        assert_eq!(module.state().unwrap(), ModuleState::Running);
13222
13223        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13224        assert!(handler
13225            .counters()
13226            .snapshot()
13227            .get("route_open_refused_by_code")
13228            .is_none());
13229        let capture = EventCapture::default();
13230        let _subscriber =
13231            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13232        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13233        let response = handler
13234            .handle_control_frame(
13235                &ctx,
13236                route_open_frame(394, "warming", unique_project_root("refusal-info")),
13237            )
13238            .await
13239            .unwrap();
13240        module.stop().await.unwrap();
13241
13242        assert_eq!(parse_error(&response[0])["code"], "module_warming");
13243        let event = capture
13244            .events()
13245            .into_iter()
13246            .find(|event| {
13247                event.target == "control"
13248                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13249            })
13250            .expect("route.open refusal event");
13251        assert_eq!(
13252            event.fields.get("module_id"),
13253            Some(&"\"warming\"".to_string())
13254        );
13255        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13256        assert_eq!(
13257            event.fields.get("reason"),
13258            Some(&"\"supervised_not_registered\"".to_string())
13259        );
13260        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13261        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13262        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13263        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13264        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13265        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13266        assert_eq!(
13267            handler.counters().snapshot()["route_open_refused_by_code"],
13268            json!({ "module_warming": 1 })
13269        );
13270    }
13271
13272    const OUTAGE_START: &str = "route.open refusing module: not serving";
13273    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13274
13275    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13276        capture
13277            .events()
13278            .into_iter()
13279            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13280            .collect()
13281    }
13282
13283    fn supervise_stub(
13284        registry: &Arc<Registry>,
13285        module_id: &str,
13286        enabled: bool,
13287    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13288        let supervisor_handle = SupervisorHandle::new();
13289        let supervisor =
13290            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13291                .with_handle(supervisor_handle.clone())
13292                .with_connection_file_path(std::env::temp_dir().join(format!(
13293                    "subc-route-outage-{module_id}-{}",
13294                    std::process::id()
13295                )));
13296        let module = supervisor
13297            .supervise_configured(
13298                ModuleSpec {
13299                    module_id: module_id.to_string(),
13300                    program: fake_aft_stub_path(),
13301                    args: Vec::new(),
13302                    env: Vec::new(),
13303                    reserved: false,
13304                    reserved_prefixes: Vec::new(),
13305                    protocol: ModuleProtocol::Subc,
13306                    overlap: Default::default(),
13307                },
13308                enabled,
13309            )
13310            .unwrap();
13311        (supervisor_handle, module)
13312    }
13313
13314    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13315        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13316            module_id: module_id.to_string(),
13317            drain_timeout_ms: Some(50),
13318        })
13319        .unwrap();
13320        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13321    }
13322
13323    /// Two handlers built over one forwarding table must share one outage
13324    /// tracker; separate trackers would each log their own opening line for
13325    /// the same outage.
13326    #[test]
13327    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13328        let registry = Arc::new(Registry::default());
13329        let forwarding = Arc::new(ForwardingTable::default());
13330        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13331        let second = ControlHandler::with_forwarding(registry, forwarding);
13332        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13333    }
13334
13335    /// A client can name any module id it likes. Refusing an unknown one,
13336    /// however often, must not create outage state or outage lines, or the
13337    /// tracker would be a memory sink any client could fill.
13338    #[tokio::test(flavor = "current_thread")]
13339    async fn route_open_unknown_module_refusals_add_no_outage_state() {
13340        let handler = ControlHandler::new(Arc::new(Registry::default()));
13341        let capture = EventCapture::default();
13342        let _subscriber =
13343            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13344        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13345        for corr in 0..8 {
13346            let response = handler
13347                .handle_control_frame(
13348                    &ctx,
13349                    route_open_frame(
13350                        380 + corr,
13351                        &format!("nobody-{corr}"),
13352                        unique_project_root("outage-unknown"),
13353                    ),
13354                )
13355                .await
13356                .unwrap();
13357            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13358        }
13359
13360        assert_eq!(handler.route_outages.tracked_module_count(), 0);
13361        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13362        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13363    }
13364
13365    /// Drives the refusal path end to end: a supervised module that served
13366    /// before and stopped being registered with no instruction to stop is a
13367    /// WARN, and the same module refused after an operator `supervisor.restart`
13368    /// is an INFO.
13369    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13370    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13371        let registry = Arc::new(Registry::default());
13372        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
13373        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13374        let capture = EventCapture::default();
13375        let _subscriber =
13376            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13377        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13378        // The stub never registers, so pretend it served once: otherwise every
13379        // refusal would fall in its startup window.
13380        handler.route_outages.record_accepted("outage-restart");
13381
13382        let response = handler
13383            .handle_control_frame(
13384                &ctx,
13385                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13386            )
13387            .await
13388            .unwrap();
13389        assert_eq!(response[0].header.ty, FrameType::Error);
13390        let starts = outage_lines(&capture, OUTAGE_START);
13391        assert_eq!(starts.len(), 1, "{starts:?}");
13392        assert_eq!(starts[0].level, tracing::Level::WARN);
13393        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13394        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13395        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13396        handler.route_outages.record_accepted("outage-restart");
13397        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13398
13399        let restart = handler
13400            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13401            .await
13402            .unwrap();
13403        assert_eq!(
13404            restart[0].header.ty,
13405            FrameType::Response,
13406            "{:?}",
13407            parse_error(&restart[0])
13408        );
13409        handler
13410            .handle_control_frame(
13411                &ctx,
13412                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13413            )
13414            .await
13415            .unwrap();
13416        module.stop().await.unwrap();
13417
13418        let starts = outage_lines(&capture, OUTAGE_START);
13419        assert_eq!(starts.len(), 2, "{starts:?}");
13420        assert_eq!(starts[1].level, tracing::Level::INFO);
13421        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13422    }
13423
13424    /// A restart refused before it touched the module (here: the module is
13425    /// disabled) must clear its operator mark, so the next real outage is
13426    /// still reported as a warning.
13427    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13428    async fn failed_operator_restart_leaves_no_operator_mark() {
13429        let registry = Arc::new(Registry::default());
13430        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
13431        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13432        let capture = EventCapture::default();
13433        let _subscriber =
13434            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13435        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13436        handler.route_outages.record_accepted("outage-disabled");
13437
13438        let restart = handler
13439            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13440            .await
13441            .unwrap();
13442        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13443        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13444
13445        handler
13446            .handle_control_frame(
13447                &ctx,
13448                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13449            )
13450            .await
13451            .unwrap();
13452        let starts = outage_lines(&capture, OUTAGE_START);
13453        assert_eq!(starts.len(), 1, "{starts:?}");
13454        assert_eq!(starts[0].level, tracing::Level::WARN);
13455    }
13456
13457    #[tokio::test(flavor = "current_thread")]
13458    async fn route_open_unknown_module_escapes_target_module_id() {
13459        let handler = ControlHandler::new(Arc::new(Registry::default()));
13460        let capture = EventCapture::default();
13461        let _subscriber =
13462            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13463        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13464        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13465        let response = handler
13466            .handle_control_frame(
13467                &ctx,
13468                route_open_frame(
13469                    395,
13470                    hostile_module_id,
13471                    unique_project_root("hostile-target-module-id"),
13472                ),
13473            )
13474            .await
13475            .unwrap();
13476
13477        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13478        let event = capture
13479            .events()
13480            .into_iter()
13481            .find(|event| {
13482                event.target == "control"
13483                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13484            })
13485            .expect("route.open unknown-module refusal event");
13486        let logged = event.fields.get("module_id").expect("module_id field");
13487        assert!(!logged.bytes().any(|byte| byte < 0x20));
13488        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13489    }
13490
13491    #[tokio::test(flavor = "current_thread")]
13492    async fn route_open_module_rejection_uses_daemon_counter_key() {
13493        let registry = Arc::new(Registry::default());
13494        let forwarding = Arc::new(ForwardingTable::default());
13495        let handler =
13496            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13497        let module_connection = ConnectionId::new(95);
13498        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13499        hello_via_sink(
13500            &handler,
13501            &module_ctx,
13502            &mut module_rx,
13503            hello_frame("aft", PROTOCOL_VERSION, 395),
13504        )
13505        .await;
13506
13507        let client_connection = ConnectionId::new(96);
13508        let (client_ctx, _client_rx) = route_ctx(client_connection);
13509        let capture = EventCapture::default();
13510        let _subscriber =
13511            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13512        let (route_task, bind) = relay_route_open(
13513            &handler,
13514            client_connection,
13515            &client_ctx.egress,
13516            &mut module_rx,
13517            396,
13518            "aft",
13519            "hostile-module-code",
13520        )
13521        .await;
13522        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13523        let rejection = Frame::build(
13524            FrameType::Error,
13525            control_flags(),
13526            0,
13527            0,
13528            bind.header.corr,
13529            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13530        )
13531        .unwrap();
13532        handler
13533            .handle_control_frame(&module_ctx, rejection)
13534            .await
13535            .unwrap();
13536
13537        let response = route_task.await.unwrap();
13538        assert_eq!(parse_error(&response[0])["code"], hostile_code);
13539        let counters = handler.counters().snapshot();
13540        assert_eq!(
13541            counters["route_open_refused_by_code"],
13542            json!({ "module_rejected": 1 })
13543        );
13544        assert!(counters["route_open_refused_by_code"]
13545            .get(hostile_code)
13546            .is_none());
13547
13548        let event = capture
13549            .events()
13550            .into_iter()
13551            .find(|event| {
13552                event.target == "control"
13553                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13554            })
13555            .expect("route.open module-rejection refusal event");
13556        let logged = event.fields.get("module_code").expect("module_code field");
13557        assert!(!logged.bytes().any(|byte| byte < 0x20));
13558        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13559    }
13560
13561    #[tokio::test]
13562    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13563        let registry = Arc::new(Registry::default());
13564        let supervisor_handle = SupervisorHandle::new();
13565        let missing_program = std::env::temp_dir().join(format!(
13566            "subc-route-open-missing-program-{}",
13567            std::process::id()
13568        ));
13569        let supervisor =
13570            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13571                .with_handle(supervisor_handle.clone());
13572        let module = supervisor
13573            .supervise_configured(
13574                ModuleSpec {
13575                    module_id: "failed".to_string(),
13576                    program: missing_program,
13577                    args: Vec::new(),
13578                    env: Vec::new(),
13579                    reserved: false,
13580                    reserved_prefixes: Vec::new(),
13581                    protocol: ModuleProtocol::Subc,
13582                    overlap: Default::default(),
13583                },
13584                true,
13585            )
13586            .unwrap();
13587        assert_eq!(module.state().unwrap(), ModuleState::Failed);
13588
13589        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13590        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13591        let response = handler
13592            .handle_control_frame(
13593                &ctx,
13594                route_open_frame(305, "failed", unique_project_root("failed")),
13595            )
13596            .await
13597            .unwrap();
13598
13599        assert_eq!(response[0].header.ty, FrameType::Error);
13600        let error = parse_error(&response[0]);
13601        assert_eq!(error["code"], "target_unavailable");
13602        assert!(error["message"]
13603            .as_str()
13604            .unwrap()
13605            .contains("state=failed, enabled=true, live=false"));
13606    }
13607
13608    #[tokio::test]
13609    async fn route_open_role_mismatch_remains_target_unavailable() {
13610        let registry = Arc::new(Registry::default());
13611        let handler = ControlHandler::new(Arc::clone(&registry));
13612        handler
13613            .handle_control(
13614                ConnectionId::new(41),
13615                non_routable_hello_frame_with_control_ops("health-only", 306, None),
13616            )
13617            .unwrap();
13618
13619        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13620        let response = handler
13621            .handle_control_frame(
13622                &ctx,
13623                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13624            )
13625            .await
13626            .unwrap();
13627
13628        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13629        assert!(parse_error(&response[0])["message"]
13630            .as_str()
13631            .unwrap()
13632            .contains("does not provide the requested target"));
13633    }
13634
13635    #[tokio::test]
13636    async fn route_open_inactive_registration_remains_target_unavailable() {
13637        let registry = Arc::new(Registry::default());
13638        let handler = ControlHandler::new(Arc::clone(&registry));
13639        handler
13640            .handle_control(
13641                ConnectionId::new(43),
13642                hello_frame("inactive", PROTOCOL_VERSION, 308),
13643            )
13644            .unwrap();
13645        assert!(registry
13646            .set_module_state_for_test("inactive", ChannelState::Closed)
13647            .unwrap());
13648
13649        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13650        let response = handler
13651            .handle_control_frame(
13652                &ctx,
13653                route_open_frame(309, "inactive", unique_project_root("inactive")),
13654            )
13655            .await
13656            .unwrap();
13657
13658        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13659        assert!(parse_error(&response[0])["message"]
13660            .as_str()
13661            .unwrap()
13662            .contains("is not active"));
13663    }
13664
13665    #[tokio::test]
13666    async fn late_health_reply_is_recorded_through_the_module_response_path() {
13667        let registry = Arc::new(Registry::default());
13668        let forwarding = Arc::new(ForwardingTable::default());
13669        let supervisor_handle = SupervisorHandle::new();
13670        let supervisor =
13671            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
13672                .with_forwarding(Arc::clone(&forwarding))
13673                .with_handle(supervisor_handle.clone());
13674        let module = supervisor
13675            .supervise_configured(
13676                crate::ModuleSpec {
13677                    module_id: "late-health-response".to_string(),
13678                    program: PathBuf::from("disabled-module"),
13679                    args: Vec::new(),
13680                    env: Vec::new(),
13681                    reserved: false,
13682                    reserved_prefixes: Vec::new(),
13683                    protocol: ModuleProtocol::Subc,
13684                    overlap: Default::default(),
13685                },
13686                false,
13687            )
13688            .unwrap();
13689        let handler =
13690            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13691                .with_supervisor(supervisor_handle);
13692        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13693        handler
13694            .handle_control_frame(
13695                &module_ctx,
13696                hello_frame_with_control_ops(
13697                    "late-health-response",
13698                    PROTOCOL_VERSION,
13699                    7,
13700                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13701                ),
13702            )
13703            .await
13704            .unwrap();
13705        let probe_started_at = Instant::now() - Duration::from_millis(80);
13706        let pending = forwarding
13707            .begin_health_probe_rpc_for(
13708                "late-health-response",
13709                MODULE_CONTROL_OP_HEALTH_CHECK,
13710                probe_started_at,
13711                Instant::now() - Duration::from_millis(1),
13712            )
13713            .unwrap();
13714        assert!(forwarding
13715            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13716            .unwrap());
13717
13718        let responses = handler
13719            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13720            .await
13721            .unwrap();
13722
13723        assert!(responses.is_empty());
13724        let health = module.status().unwrap().health;
13725        assert_eq!(health.late_answer_count, 1);
13726        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13727    }
13728
13729    #[tokio::test]
13730    async fn health_probe_timeout_and_module_death_are_typed() {
13731        let registry = Arc::new(Registry::default());
13732        let forwarding = Arc::new(ForwardingTable::default());
13733        let handler =
13734            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13735                .with_health_probe_timeout(Duration::from_millis(50));
13736        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13737        hello_via_sink(
13738            &handler,
13739            &module_ctx,
13740            &mut module_rx,
13741            hello_frame_with_control_ops(
13742                "aft",
13743                PROTOCOL_VERSION,
13744                7,
13745                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13746            ),
13747        )
13748        .await;
13749
13750        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13751        let responses = handler
13752            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13753            .await
13754            .unwrap();
13755        assert_eq!(responses[0].header.ty, FrameType::Error);
13756        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13757        let _ = module_rx.try_recv();
13758
13759        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13760        let health_handler = handler.clone();
13761        let death_task = tokio::spawn(async move {
13762            health_handler
13763                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13764                .await
13765                .unwrap()
13766        });
13767        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13768            .await
13769            .unwrap()
13770            .unwrap();
13771        handler
13772            .cleanup_connection(module_ctx.connection_id)
13773            .unwrap();
13774        let responses = death_task.await.unwrap();
13775        assert_eq!(responses[0].header.ty, FrameType::Error);
13776        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13777    }
13778
13779    #[test]
13780    fn hello_requires_exact_protocol_version() {
13781        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13782            let registry = Arc::new(Registry::default());
13783            let handler = ControlHandler::new(Arc::clone(&registry));
13784            let responses = handler
13785                .handle_control(
13786                    ConnectionId::new(connection),
13787                    hello_frame("aft", offered, 9),
13788                )
13789                .unwrap();
13790
13791            assert_eq!(responses.len(), 1);
13792            assert_eq!(responses[0].header.ty, FrameType::Error);
13793            let error = parse_error(&responses[0]);
13794            assert_eq!(error["code"], "version_unsupported");
13795            assert!(registry.get_module("aft").unwrap().is_none());
13796            assert_eq!(registry.active_registration_count().unwrap(), 0);
13797        }
13798    }
13799
13800    #[test]
13801    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13802        let registry = Arc::new(Registry::default());
13803        let forwarding = Arc::new(ForwardingTable::default());
13804        let handler =
13805            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13806        let module_connection = ConnectionId::new(301);
13807        let registration = registry
13808            .register_with_control_ops(
13809                manifest("aft-push", PROTOCOL_VERSION),
13810                PROTOCOL_VERSION,
13811                module_connection,
13812                module_baseline_control_ops(),
13813            )
13814            .unwrap();
13815        let (module_tx, _module_rx) = mpsc::channel(8);
13816        let endpoint = forwarding
13817            .register_module_connection(
13818                module_connection,
13819                "aft-push".to_string(),
13820                PROTOCOL_VERSION,
13821                manifest_concurrency(&registration.manifest),
13822                FrameSink::new(module_tx),
13823            )
13824            .unwrap();
13825
13826        // A push op this version does not know is ignored (forward-compat), not errored.
13827        let unknown = Frame::build(
13828            FrameType::Push,
13829            control_flags(),
13830            0,
13831            0,
13832            5,
13833            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13834        )
13835        .unwrap();
13836        let out = handler.handle_status_update(endpoint, unknown).unwrap();
13837        assert!(
13838            out.is_empty(),
13839            "unknown push op must be ignored, got {out:?}"
13840        );
13841
13842        // A malformed body for a KNOWN op is a real error worth surfacing.
13843        let malformed = Frame::build(
13844            FrameType::Push,
13845            control_flags(),
13846            0,
13847            0,
13848            6,
13849            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13850        )
13851        .unwrap();
13852        let out = handler.handle_status_update(endpoint, malformed).unwrap();
13853        assert_eq!(out.len(), 1);
13854        assert_eq!(out[0].header.ty, FrameType::Error);
13855        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13856    }
13857
13858    #[test]
13859    fn hello_rejected_when_connection_already_owns_client_routes() {
13860        let registry = Arc::new(Registry::default());
13861        let forwarding = Arc::new(ForwardingTable::default());
13862        let handler =
13863            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13864        // Commits a client route on connection 202 (bound to a module on conn 101).
13865        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
13866        let client_connection = ConnectionId::new(202);
13867
13868        // That same connection now tries to register as a module: rejected, so one
13869        // connection never holds both client-route and module-endpoint state.
13870        let responses = handler
13871            .handle_control(
13872                client_connection,
13873                hello_frame("aft-second", PROTOCOL_VERSION, 9),
13874            )
13875            .unwrap();
13876        assert_eq!(responses[0].header.ty, FrameType::Error);
13877        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13878        assert!(registry.get_module("aft-second").unwrap().is_none());
13879    }
13880
13881    #[tokio::test]
13882    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13883        let registry = Arc::new(Registry::default());
13884        let forwarding = Arc::new(ForwardingTable::default());
13885        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13886        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13887        hello_via_sink(
13888            &handler,
13889            &module_ctx,
13890            &mut module_rx,
13891            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13892        )
13893        .await;
13894        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13895        let pending = forwarding
13896            .begin_route_bind_relay_for_test(
13897                client_ctx.connection_id,
13898                client_ctx.egress.clone(),
13899                2,
13900                "alpha",
13901            )
13902            .unwrap();
13903        forwarding
13904            .complete_pending_relay(
13905                module_ctx.connection_id,
13906                pending.corr,
13907                RouteBindRelayOutcome::Accepted,
13908            )
13909            .unwrap();
13910        client_rx.try_recv().unwrap();
13911        for module_id in ["beta", "alpha"] {
13912            let replies = handler
13913                .handle_control_frame(
13914                    &module_ctx,
13915                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13916                )
13917                .await
13918                .unwrap();
13919            assert_eq!(replies.len(), 1, "second HELLO must be refused");
13920            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13921        }
13922        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13923        assert!(registry.get_module("beta").unwrap().is_none());
13924        assert!(matches!(
13925            forwarding
13926                .lookup_data_route(
13927                    client_ctx.connection_id,
13928                    pending.client_channel,
13929                    pending.client_epoch,
13930                )
13931                .unwrap(),
13932            DataRoute::Client(DataRouteState::Bound(_))
13933        ));
13934        assert!(handler
13935            .hello_launch_nonces
13936            .lock()
13937            .unwrap()
13938            .presented(module_ctx.connection_id, Some("alpha-nonce")));
13939        assert!(module_rx.try_recv().is_err());
13940    }
13941
13942    #[test]
13943    fn reserved_module_hello_requires_matching_launch_nonce() {
13944        let registry = Arc::new(Registry::default());
13945        let supervisor = SupervisorHandle::new();
13946        // The supervisor recorded the nonce it injected when it spawned the reserved
13947        // module; the HELLO verifier checks against the same shared handle.
13948        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13949        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13950
13951        // A HELLO with NO nonce is rejected.
13952        let no_nonce = handler
13953            .handle_control(
13954                ConnectionId::new(1),
13955                hello_frame("vault", PROTOCOL_VERSION, 1),
13956            )
13957            .unwrap();
13958        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13959        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13960        assert!(registry.get_module("vault").unwrap().is_none());
13961
13962        // A HELLO with the WRONG nonce is rejected.
13963        let wrong = handler
13964            .handle_control(
13965                ConnectionId::new(2),
13966                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13967            )
13968            .unwrap();
13969        assert_eq!(wrong[0].header.ty, FrameType::Error);
13970        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13971        assert!(registry.get_module("vault").unwrap().is_none());
13972
13973        // A HELLO with the CORRECT nonce registers.
13974        let ok = handler
13975            .handle_control(
13976                ConnectionId::new(3),
13977                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13978            )
13979            .unwrap();
13980        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13981        assert!(registry.get_module("vault").unwrap().is_some());
13982    }
13983
13984    #[test]
13985    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13986        let registry = Arc::new(Registry::default());
13987        let supervisor = SupervisorHandle::new();
13988        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13989        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13990        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13991
13992        let squat = handler
13993            .handle_control(
13994                ConnectionId::new(1),
13995                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13996            )
13997            .unwrap();
13998        assert_eq!(squat[0].header.ty, FrameType::Error);
13999        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14000        assert!(parse_error(&squat[0])["message"]
14001            .as_str()
14002            .unwrap()
14003            .contains("fed:"));
14004
14005        let accepted_peer = handler
14006            .handle_control(
14007                ConnectionId::new(2),
14008                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14009            )
14010            .unwrap();
14011        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14012
14013        let accepted_short = handler
14014            .handle_control(
14015                ConnectionId::new(3),
14016                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14017            )
14018            .unwrap();
14019        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14020
14021        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14022            let response = handler
14023                .handle_control(
14024                    ConnectionId::new(conn),
14025                    hello_frame(module_id, PROTOCOL_VERSION, conn),
14026                )
14027                .unwrap();
14028            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14029        }
14030    }
14031
14032    #[test]
14033    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14034        let registry = Arc::new(Registry::default());
14035        let supervisor = SupervisorHandle::new();
14036        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14037        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14038        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14039        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14040
14041        let owner_nonce = handler
14042            .handle_control(
14043                ConnectionId::new(1),
14044                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14045            )
14046            .unwrap();
14047        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14048        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14049        assert!(registry.get_module("fed:special").unwrap().is_none());
14050
14051        let exact_nonce = handler
14052            .handle_control(
14053                ConnectionId::new(2),
14054                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14055            )
14056            .unwrap();
14057        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14058        assert!(registry.get_module("fed:special").unwrap().is_some());
14059    }
14060
14061    #[test]
14062    fn non_reserved_module_ignores_launch_nonce() {
14063        let registry = Arc::new(Registry::default());
14064        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
14065        // registration succeeds whether a spawned process echoes a nonce or not.
14066        let handler = ControlHandler::new(Arc::clone(&registry));
14067        let no_nonce = handler
14068            .handle_control(
14069                ConnectionId::new(1),
14070                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14071            )
14072            .unwrap();
14073        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14074        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14075
14076        let echoed_nonce = handler
14077            .handle_control(
14078                ConnectionId::new(2),
14079                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14080            )
14081            .unwrap();
14082        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14083        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14084    }
14085
14086    #[test]
14087    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14088        let handler = ControlHandler::default();
14089        let conn = ConnectionId::new(1);
14090        let malformed = Frame::build(
14091            FrameType::Hello,
14092            control_flags(),
14093            0,
14094            0,
14095            3,
14096            b"{not json".to_vec(),
14097        )
14098        .unwrap();
14099
14100        let error = handler.handle_control(conn, malformed).unwrap();
14101        assert_eq!(error[0].header.ty, FrameType::Error);
14102        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14103
14104        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14105        let pong = handler.handle_control(conn, ping).unwrap();
14106        assert_eq!(pong[0].header.ty, FrameType::Pong);
14107        assert_eq!(pong[0].header.corr, 4);
14108    }
14109
14110    #[test]
14111    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14112        let registry = Arc::new(Registry::default());
14113        let handler = ControlHandler::new(Arc::clone(&registry));
14114
14115        handler
14116            .handle_control(
14117                ConnectionId::new(1),
14118                hello_frame("aft", PROTOCOL_VERSION, 1),
14119            )
14120            .unwrap();
14121        let duplicate = handler
14122            .handle_control(
14123                ConnectionId::new(2),
14124                hello_frame("aft", PROTOCOL_VERSION, 2),
14125            )
14126            .unwrap();
14127
14128        assert_eq!(duplicate[0].header.ty, FrameType::Error);
14129        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14130        let registration = registry.get_module("aft").unwrap().unwrap();
14131        assert_eq!(registration.connection_id, ConnectionId::new(1));
14132    }
14133
14134    #[test]
14135    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14136        let registry = Arc::new(Registry::default());
14137        let forwarding = Arc::new(ForwardingTable::default());
14138        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14139        let handler =
14140            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14141                .with_process_liveness(process_liveness);
14142        let (ctx, route_channel, route_epoch) =
14143            bind_liveness_route(&registry, &forwarding, "aft-dead");
14144        let responses = handler
14145            .handle_route_poll(
14146                &ctx,
14147                route_poll_frame(41, PollKind::Liveness, route_channel),
14148                route_channel,
14149                route_epoch,
14150                PollKind::Liveness,
14151            )
14152            .unwrap();
14153
14154        assert_eq!(responses.len(), 1);
14155        assert_eq!(responses[0].header.ty, FrameType::Response);
14156        assert_route_poll_liveness(&responses[0], false);
14157    }
14158
14159    #[test]
14160    fn liveness_poll_without_process_source_uses_bound_route() {
14161        let registry = Arc::new(Registry::default());
14162        let forwarding = Arc::new(ForwardingTable::default());
14163        let handler =
14164            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14165        let (ctx, route_channel, route_epoch) =
14166            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
14167        let responses = handler
14168            .handle_route_poll(
14169                &ctx,
14170                route_poll_frame(42, PollKind::Liveness, route_channel),
14171                route_channel,
14172                route_epoch,
14173                PollKind::Liveness,
14174            )
14175            .unwrap();
14176
14177        assert_route_poll_liveness(&responses[0], true);
14178    }
14179
14180    #[test]
14181    fn liveness_poll_untracked_process_source_uses_bound_route() {
14182        let registry = Arc::new(Registry::default());
14183        let forwarding = Arc::new(ForwardingTable::default());
14184        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14185        let handler =
14186            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14187                .with_process_liveness(process_liveness);
14188        let (ctx, route_channel, route_epoch) =
14189            bind_liveness_route(&registry, &forwarding, "aft-untracked");
14190        let responses = handler
14191            .handle_route_poll(
14192                &ctx,
14193                route_poll_frame(43, PollKind::Liveness, route_channel),
14194                route_channel,
14195                route_epoch,
14196                PollKind::Liveness,
14197            )
14198            .unwrap();
14199
14200        assert_route_poll_liveness(&responses[0], true);
14201    }
14202
14203    #[tokio::test]
14204    async fn unknown_op_returns_unknown_control_op() {
14205        let handler = ControlHandler::default();
14206        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14207        let request = Frame::build(
14208            FrameType::Request,
14209            control_flags(),
14210            0,
14211            0,
14212            55,
14213            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14214        )
14215        .unwrap();
14216
14217        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14218
14219        assert_eq!(response.len(), 1);
14220        assert_eq!(response[0].header.ty, FrameType::Error);
14221        assert_eq!(response[0].header.corr, 55);
14222        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14223    }
14224
14225    #[tokio::test]
14226    async fn supervisor_provenance_rejects_unknown_exact_module() {
14227        let handler = ControlHandler::default();
14228        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14229        let request = Frame::build(
14230            FrameType::Request,
14231            control_flags(),
14232            0,
14233            0,
14234            57,
14235            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14236        )
14237        .unwrap();
14238
14239        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14240
14241        assert_eq!(response.len(), 1);
14242        assert_eq!(response[0].header.ty, FrameType::Error);
14243        assert_eq!(response[0].header.corr, 57);
14244        let error = parse_error(&response[0]);
14245        assert_eq!(error["code"], "unknown_module");
14246        assert_eq!(error["message"], "module_id 'missing' is not supervised");
14247    }
14248
14249    #[test]
14250    fn provenance_probe_override_keeps_handler_tests_deterministic() {
14251        let expected = subc_control::RunningImageAgreement::Unavailable {
14252            reason: subc_control::RunningImageUnavailableReason::HashFailed,
14253        };
14254        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14255        assert_eq!(handler.provenance_probe_override, Some(expected));
14256    }
14257
14258    #[test]
14259    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14260        let verdict = reload_verdict(
14261            std::path::Path::new("/bin/new"),
14262            Some(std::path::Path::new("/bin/old")),
14263            subc_control::RunningImageAgreement::Unavailable {
14264                reason: subc_control::RunningImageUnavailableReason::HashFailed,
14265            },
14266        );
14267        assert!(matches!(
14268            verdict.path,
14269            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14270                if configured == std::path::Path::new("/bin/new")
14271                    && spawned_from == std::path::Path::new("/bin/old")
14272        ));
14273    }
14274
14275    #[test]
14276    fn launch_spec_without_running_digest_is_unavailable_not_match() {
14277        use subc_control::{
14278            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14279        };
14280        let configured = crate::supervise::ModuleSpec {
14281            module_id: "legacy".into(),
14282            program: "/module".into(),
14283            args: Vec::new(),
14284            env: Vec::new(),
14285            reserved: false,
14286            reserved_prefixes: Vec::new(),
14287            protocol: subc_control::ModuleProtocol::Subc,
14288            overlap: Default::default(),
14289        };
14290        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14291        assert_eq!(
14292            launch_spec_agreement(&configured, false, Some(&digest)),
14293            Agreement::Unavailable {
14294                reason: Reason::NotRunning
14295            }
14296        );
14297        assert_eq!(
14298            launch_spec_agreement(&configured, true, None),
14299            Agreement::Unavailable {
14300                reason: Reason::SpawnedSpecUnavailable
14301            }
14302        );
14303        assert_eq!(
14304            launch_spec_agreement(&configured, true, Some(&digest)),
14305            Agreement::Match
14306        );
14307    }
14308
14309    #[cfg(windows)]
14310    #[test]
14311    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14312        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14313        let configured = dir.join("module.exe");
14314        std::fs::write(&configured, b"executable path fixture").unwrap();
14315        let spawned = std::fs::canonicalize(&configured).unwrap();
14316        assert_ne!(
14317            configured, spawned,
14318            "fixture must exercise different path spellings"
14319        );
14320        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14321        let lower_case = configured.to_string_lossy().to_lowercase();
14322        for spelling in [
14323            &configured,
14324            Path::new(&forward_slashes),
14325            Path::new(&lower_case),
14326        ] {
14327            let verdict = reload_verdict(
14328                spelling,
14329                Some(&spawned),
14330                subc_control::RunningImageAgreement::Unavailable {
14331                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14332                },
14333            );
14334            assert_eq!(
14335                verdict.path,
14336                ReloadPathAgreement::Match,
14337                "configured {spelling:?}, spawned {spawned:?}"
14338            );
14339        }
14340    }
14341
14342    #[cfg(windows)]
14343    #[test]
14344    fn windows_reload_path_keeps_different_existing_program_pending() {
14345        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14346        let old = dir.join("old.exe");
14347        let new = dir.join("new.exe");
14348        std::fs::write(&old, b"same executable content").unwrap();
14349        std::fs::copy(&old, &new).unwrap();
14350        let spawned = std::fs::canonicalize(&old).unwrap();
14351        let image = subc_control::RunningImageAgreement::Unavailable {
14352            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14353        };
14354        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14355        assert!(
14356            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14357            if configured == new && spawned_from == spawned)
14358        );
14359        let missing = dir.join("missing.exe");
14360        assert!(matches!(
14361            reload_verdict(&missing, Some(&spawned), image).path,
14362            ReloadPathAgreement::Mismatch { .. }
14363        ));
14364    }
14365
14366    #[test]
14367    fn reload_verdict_detects_replaced_image_at_same_path() {
14368        let image = subc_control::RunningImageAgreement::Mismatch {
14369            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14370                digest: "old".into(),
14371            },
14372            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14373                digest: "new".into(),
14374            },
14375        };
14376        let verdict = reload_verdict(
14377            std::path::Path::new("/bin/same"),
14378            Some(std::path::Path::new("/bin/same")),
14379            image.clone(),
14380        );
14381        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14382        assert_eq!(verdict.image, image);
14383    }
14384
14385    #[test]
14386    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14387        let image = subc_control::RunningImageAgreement::Unavailable {
14388            reason: subc_control::RunningImageUnavailableReason::NotRunning,
14389        };
14390        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14391        assert_eq!(
14392            verdict.path,
14393            subc_control::ReloadPathAgreement::Unavailable {
14394                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14395            }
14396        );
14397        assert_eq!(verdict.image, image);
14398
14399        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14400            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14401        };
14402        let verdict = reload_verdict(
14403            std::path::Path::new("/bin/same"),
14404            Some(std::path::Path::new("/bin/same")),
14405            unconfirmed.clone(),
14406        );
14407        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14408        assert_eq!(verdict.image, unconfirmed);
14409    }
14410
14411    #[test]
14412    fn reload_verdict_preserves_each_image_unavailability_reason() {
14413        use subc_control::RunningImageUnavailableReason as Reason;
14414
14415        for reason in [
14416            Reason::NotRunning,
14417            Reason::UnsupportedPlatform,
14418            Reason::RunningExecutableUnreadable,
14419            Reason::SpawnedPathUnreadable,
14420            Reason::HashFailed,
14421            Reason::ProcessIdentityUnconfirmed,
14422            Reason::Unknown("future_probe_reason".to_string()),
14423        ] {
14424            let image = subc_control::RunningImageAgreement::Unavailable {
14425                reason: reason.clone(),
14426            };
14427            let verdict = reload_verdict(
14428                std::path::Path::new("/bin/same"),
14429                Some(std::path::Path::new("/bin/same")),
14430                image.clone(),
14431            );
14432            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14433            assert_eq!(verdict.image, image, "{reason:?}");
14434        }
14435    }
14436
14437    #[tokio::test]
14438    async fn malformed_control_bodies_return_invalid_control_body() {
14439        let handler = ControlHandler::default();
14440        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14441
14442        for (corr, body) in [
14443            (56, br#"{"route_channel":1}"#.as_slice()),
14444            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14445            (
14446                58,
14447                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14448            ),
14449        ] {
14450            let request = Frame::build(
14451                FrameType::Request,
14452                control_flags(),
14453                0,
14454                0,
14455                corr,
14456                body.to_vec(),
14457            )
14458            .unwrap();
14459            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14460
14461            assert_eq!(response.len(), 1);
14462            assert_eq!(response[0].header.ty, FrameType::Error);
14463            assert_eq!(response[0].header.corr, corr);
14464            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14465        }
14466    }
14467
14468    #[tokio::test]
14469    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14470        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14471        let registry = Arc::new(Registry::default());
14472        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14473        let router = Router::with_control_handler(Arc::clone(&control));
14474        let connection = router.begin_connection();
14475        let (ctx, mut rx) = route_ctx(connection.id());
14476
14477        router
14478            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14479            .await
14480            .unwrap();
14481        let response = rx.recv().await.unwrap();
14482        let ack = parse_ack(&response);
14483        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14484        let channel = 1;
14485
14486        let goodbye =
14487            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14488        router.route_for_connection(&ctx, goodbye).await.unwrap();
14489        assert!(rx.try_recv().is_err());
14490        assert!(registry.get_module("aft").unwrap().is_none());
14491
14492        router
14493            .route_for_connection(&ctx, channel_request(channel, 13))
14494            .await
14495            .unwrap();
14496        let error_frame = rx.recv().await.unwrap();
14497        assert_eq!(error_frame.header.ty, FrameType::Error);
14498        assert_eq!(error_frame.header.channel, channel);
14499        let captured = crate::router::test_log::captured_logs(&logs);
14500        assert_eq!(
14501            captured
14502                .lines()
14503                .filter(|line| {
14504                    line.contains("module_id=aft")
14505                        && line.contains("reason=explicit_goodbye")
14506                        && line.contains("module registration ended")
14507                })
14508                .count(),
14509            1,
14510            "unexpected GOODBYE registry log: {captured}"
14511        );
14512    }
14513
14514    #[tokio::test]
14515    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14516        let registry = Arc::new(Registry::default());
14517        let handler = ControlHandler::new(registry).with_capability_config(
14518            [("prov".to_string(), true), ("cons".to_string(), true)],
14519            BTreeMap::new(),
14520        );
14521        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14522        register_capability_manifest(
14523            &handler,
14524            &provider_ctx,
14525            &mut provider_rx,
14526            capability_manifest("prov", &["thing/v1"], &[]),
14527            1,
14528        )
14529        .await;
14530        let mut consumer = capability_manifest("cons", &[], &[]);
14531        consumer.capabilities.as_mut().unwrap().requires.push(
14532            subc_protocol::manifest::CapabilityRequirement {
14533                capability: "thing/v1".to_string(),
14534                need: subc_protocol::manifest::CapabilityNeed::Required,
14535            },
14536        );
14537        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14538        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14539        assert_eq!(
14540            handler.capability_evaluator.verdict("cons", "thing/v1"),
14541            Some(CapabilityVerdict::Provided)
14542        );
14543        let (mut client_rx, _) = open_route_for_capability_test(
14544            &handler,
14545            &provider_ctx,
14546            &mut provider_rx,
14547            703,
14548            3,
14549            "prov",
14550            None,
14551        )
14552        .await;
14553        let goodbye =
14554            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14555        handler
14556            .handle_control_frame(&provider_ctx, goodbye)
14557            .await
14558            .unwrap();
14559        assert_eq!(
14560            handler.capability_evaluator.verdict("cons", "thing/v1"),
14561            Some(CapabilityVerdict::NeverProvided)
14562        );
14563        let closed = client_rx
14564            .try_recv()
14565            .expect("GOODBYE pushes route.closed before route GOODBYE");
14566        assert!(
14567            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14568            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14569        );
14570        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14571        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14572    }
14573
14574    struct RouteClosedFixture {
14575        handler: ControlHandler,
14576        #[cfg(unix)]
14577        supervisor: Supervisor,
14578        module_ctx: RouteCtx,
14579        #[cfg(unix)]
14580        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14581        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14582        route: Arc<crate::forwarding::RouteBinding>,
14583    }
14584
14585    async fn route_closed_fixture() -> RouteClosedFixture {
14586        let registry = Arc::new(Registry::default());
14587        let handler = ControlHandler::new(Arc::clone(&registry));
14588        #[cfg(unix)]
14589        let supervisor = Supervisor::new(registry, RestartPolicy::default())
14590            .with_forwarding(Arc::clone(&handler.forwarding));
14591        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14592        register_capability_manifest(
14593            &handler,
14594            &module_ctx,
14595            &mut module_rx,
14596            capability_manifest("prov", &[], &[]),
14597            1,
14598        )
14599        .await;
14600        let (client_rx, bind) = open_route_for_capability_test(
14601            &handler,
14602            &module_ctx,
14603            &mut module_rx,
14604            702,
14605            2,
14606            "prov",
14607            None,
14608        )
14609        .await;
14610        let ModuleControlRequest::RouteBind {
14611            route_channel: channel,
14612            epoch: route_epoch,
14613            ..
14614        } = bind
14615        else {
14616            panic!("expected route.bind");
14617        };
14618        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14619            handler
14620                .forwarding
14621                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14622                .unwrap()
14623        else {
14624            panic!("the acknowledged route must be bound");
14625        };
14626        RouteClosedFixture {
14627            handler,
14628            #[cfg(unix)]
14629            supervisor,
14630            module_ctx,
14631            #[cfg(unix)]
14632            module_rx,
14633            client_rx,
14634            route,
14635        }
14636    }
14637
14638    impl RouteClosedFixture {
14639        #[cfg(unix)]
14640        async fn cut(&mut self) {
14641            let notices = async {
14642                let mut draining = self.module_rx.recv().await.unwrap();
14643                assert!(matches!(
14644                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14645                        &draining.body
14646                    )
14647                    .unwrap(),
14648                    subc_protocol::session::ModuleControlCommand::Draining {
14649                        reason: RouteCloseReason::Restart,
14650                        ..
14651                    }
14652                ));
14653                draining.flushed.take().unwrap().send(()).unwrap();
14654                let mut closing = self.client_rx.recv().await.unwrap();
14655                assert!(matches!(
14656                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14657                    ClientControlPush::RouteClosing {
14658                        reason: RouteCloseReason::Restart,
14659                        ..
14660                    }
14661                ));
14662                closing.flushed.take().unwrap().send(()).unwrap();
14663            };
14664            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14665            result.unwrap();
14666        }
14667
14668        fn disconnect(&mut self) -> ClientControlPush {
14669            self.handler
14670                .cleanup_connection(self.module_ctx.connection_id)
14671                .unwrap();
14672            let closed = self.client_rx.try_recv().unwrap();
14673            assert_eq!(closed.header.ty, FrameType::Push);
14674            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14675            assert!(matches!(
14676                &push,
14677                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14678                    if channels == &[self.route.client_channel]
14679            ));
14680            assert_eq!(
14681                self.client_rx.try_recv().unwrap().header.ty,
14682                FrameType::Goodbye
14683            );
14684            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14685            push
14686        }
14687    }
14688
14689    #[cfg(unix)]
14690    #[tokio::test(start_paused = true)]
14691    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14692        let mut fixture = route_closed_fixture().await;
14693        fixture.cut().await;
14694        assert!(matches!(
14695            fixture.disconnect(),
14696            ClientControlPush::RouteClosed {
14697                reason: RouteCloseReason::Restart,
14698                drained: true,
14699                ..
14700            }
14701        ));
14702    }
14703
14704    #[cfg(unix)]
14705    #[tokio::test(start_paused = true)]
14706    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14707        let mut fixture = route_closed_fixture().await;
14708        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14709        let started = tokio::time::Instant::now();
14710        fixture.cut().await;
14711        assert!(started.elapsed() >= Duration::from_secs(2));
14712        assert_eq!(fixture.route.flow.in_flight(), 1);
14713        assert!(matches!(
14714            fixture.disconnect(),
14715            ClientControlPush::RouteClosed {
14716                reason: RouteCloseReason::Restart,
14717                drained: false,
14718                ..
14719            }
14720        ));
14721    }
14722
14723    #[tokio::test]
14724    async fn module_connection_crash_reports_crash_and_not_drained() {
14725        let mut fixture = route_closed_fixture().await;
14726        // The server calls cleanup_connection on module EOF. Without a planned
14727        // daemon shutdown, that disconnect must remain a crash with no prior
14728        // route.closing notice, even when the route has no in-flight calls.
14729        assert!(matches!(
14730            fixture.disconnect(),
14731            ClientControlPush::RouteClosed {
14732                reason: RouteCloseReason::Crash,
14733                drained: false,
14734                ..
14735            }
14736        ));
14737    }
14738
14739    #[tokio::test]
14740    async fn dropping_router_connection_releases_registration() {
14741        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14742        let registry = Arc::new(Registry::default());
14743        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14744        let router = Router::with_control_handler(Arc::clone(&control));
14745        let connection = router.begin_connection();
14746        let connection_id = connection.id();
14747        let (ctx, mut rx) = route_ctx(connection_id);
14748
14749        router
14750            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14751            .await
14752            .unwrap();
14753        let response = rx.recv().await.unwrap();
14754        let ack = parse_ack(&response);
14755        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14756        assert!(registry.get_module("aft").unwrap().is_some());
14757
14758        drop(connection);
14759
14760        assert!(registry.get_module("aft").unwrap().is_none());
14761        assert_eq!(registry.active_registration_count().unwrap(), 0);
14762
14763        control
14764            .cleanup_connection(ConnectionId::new(u64::MAX))
14765            .unwrap();
14766        let captured = crate::router::test_log::captured_logs(&logs);
14767        println!("captured registry lifecycle logs:\n{captured}");
14768        let events: Vec<_> = captured
14769            .lines()
14770            .filter(|line| {
14771                line.contains("module registered module_id=aft ")
14772                    || line.contains("module registration ended")
14773            })
14774            .collect();
14775        assert_eq!(
14776            events.len(),
14777            2,
14778            "unexpected registry lifecycle logs: {captured}"
14779        );
14780        assert!(events[0].contains("module registered module_id=aft "));
14781        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14782        assert!(events[1].contains(&format!(
14783            "module_id=aft connection_id={}",
14784            connection_id.get()
14785        )));
14786        assert!(events[1].contains("reason=connection_closed"));
14787        assert!(events[1].contains("module registration ended"));
14788        assert_eq!(
14789            captured
14790                .lines()
14791                .filter(|line| line.contains("module registered module_id=aft "))
14792                .count(),
14793            1,
14794            "legacy registration admission line must appear once: {captured}"
14795        );
14796    }
14797
14798    #[tokio::test]
14799    async fn hello_registration_keeps_legacy_module_registered_line_once() {
14800        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14801        let registry = Arc::new(Registry::default());
14802        let control = ControlHandler::new(Arc::clone(&registry));
14803        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14804        hello_via_sink(
14805            &control,
14806            &ctx,
14807            &mut rx,
14808            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14809        )
14810        .await;
14811
14812        let captured = crate::router::test_log::captured_logs(&logs);
14813        let admissions: Vec<_> = captured
14814            .lines()
14815            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14816            .collect();
14817        assert_eq!(
14818            admissions.len(),
14819            1,
14820            "legacy admission line must remain exactly once: {captured}"
14821        );
14822        assert!(admissions[0].contains("routable_provider=true"));
14823        assert!(admissions[0].contains("connection_id=777"));
14824    }
14825
14826    fn capability_manifest(
14827        module_id: &str,
14828        provides: &[&str],
14829        must_never_reach: &[&str],
14830    ) -> ModuleManifest {
14831        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14832        manifest.capabilities = Some(CapabilityDeclarations {
14833            provides: provides
14834                .iter()
14835                .map(|capability| (*capability).to_string())
14836                .collect(),
14837            requires: Vec::new(),
14838            must_never_reach: must_never_reach
14839                .iter()
14840                .map(|capability| (*capability).to_string())
14841                .collect(),
14842        });
14843        manifest
14844    }
14845
14846    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14847        Frame::build(
14848            FrameType::Hello,
14849            control_flags(),
14850            0,
14851            0,
14852            corr,
14853            serde_json::to_vec(&ModuleHelloBody {
14854                protocol_ver: manifest.protocol_ver,
14855                manifest,
14856                control_ops: None,
14857                launch_nonce: None,
14858            })
14859            .expect("capability test HELLO serializes"),
14860        )
14861        .expect("capability test HELLO frame builds")
14862    }
14863
14864    fn catalog_update_with_capabilities_frame(
14865        corr: u64,
14866        capabilities: CapabilityDeclarations,
14867    ) -> Frame {
14868        Frame::build(
14869            FrameType::Request,
14870            control_flags(),
14871            0,
14872            0,
14873            corr,
14874            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14875                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14876                capabilities: Some(capabilities),
14877                ready: None,
14878            })
14879            .expect("capability catalog.update serializes"),
14880        )
14881        .expect("capability catalog.update frame builds")
14882    }
14883
14884    async fn register_capability_manifest(
14885        handler: &ControlHandler,
14886        ctx: &RouteCtx,
14887        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14888        manifest: ModuleManifest,
14889        corr: u64,
14890    ) {
14891        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14892    }
14893
14894    async fn open_route_for_capability_test(
14895        handler: &ControlHandler,
14896        target_ctx: &RouteCtx,
14897        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14898        client_connection_id: u64,
14899        corr: u64,
14900        target_module_id: &str,
14901        consumer_identity: Option<ConsumerIdentity>,
14902    ) -> (
14903        mpsc::Receiver<crate::router::OutboundFrame>,
14904        ModuleControlRequest,
14905    ) {
14906        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14907        let route_handler = handler.clone();
14908        let target_module_id = target_module_id.to_string();
14909        let route_task = tokio::spawn(async move {
14910            route_handler
14911                .handle_control_frame(
14912                    &client_ctx,
14913                    route_open_frame_with_admission_facts(
14914                        corr,
14915                        &target_module_id,
14916                        unique_project_root("admission-facts"),
14917                        consumer_identity,
14918                        None,
14919                    ),
14920                )
14921                .await
14922                .expect("capability test route.open succeeds")
14923        });
14924        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14925            .await
14926            .expect("capability test route.open must reach route.bind")
14927            .expect("target control receiver stays open");
14928        let bind_request: ModuleControlRequest =
14929            serde_json::from_slice(&bind.body).expect("route.bind decodes");
14930        handler
14931            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14932            .await
14933            .expect("capability test route.bind ACK succeeds");
14934        assert!(route_task.await.expect("route.open task joins").is_empty());
14935        let opened = client_rx
14936            .recv()
14937            .await
14938            .expect("successful route.open publishes a response");
14939        assert!(matches!(
14940            serde_json::from_slice::<ClientControlResponse>(&opened.body),
14941            Ok(ClientControlResponse::RouteOpen { .. })
14942        ));
14943        (client_rx, bind_request)
14944    }
14945
14946    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14947        assert_eq!(frame.header.ty, FrameType::Push);
14948        assert_eq!(frame.header.channel, 0);
14949        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14950            .expect("route.closed control push decodes");
14951        let ClientControlPush::RouteClosed { channels, .. } = &push else {
14952            panic!("expected route.closed");
14953        };
14954        assert_eq!(channels.len(), 1, "exactly one violating route closed");
14955        let channels = channels.clone();
14956        assert_eq!(
14957            push,
14958            ClientControlPush::RouteClosed {
14959                module_id: target_module_id.to_string(),
14960                channels,
14961                reason: RouteCloseReason::CapabilityDenied,
14962                drained: false,
14963                abandoned: 0,
14964                excluded_subscriptions: 0,
14965                terminal: Some(false),
14966            }
14967        );
14968    }
14969
14970    #[tokio::test]
14971    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14972        let registry = Arc::new(Registry::default());
14973        let forwarding = Arc::new(ForwardingTable::default());
14974        let supervisor = SupervisorHandle::new();
14975        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14976        let handler =
14977            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14978                .with_supervisor(supervisor);
14979        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14980        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14981        register_capability_manifest(
14982            &handler,
14983            &target_ctx,
14984            &mut target_rx,
14985            capability_manifest("target", &["credentials-provider/v1"], &[]),
14986            1,
14987        )
14988        .await;
14989        register_capability_manifest(
14990            &handler,
14991            &opener_ctx,
14992            &mut opener_rx,
14993            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14994            2,
14995        )
14996        .await;
14997
14998        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14999        let replies = handler
15000            .handle_control_frame(
15001                &client_ctx,
15002                route_open_frame_with_admission_facts(
15003                    3,
15004                    "target",
15005                    unique_project_root("admission-facts"),
15006                    Some(ConsumerIdentity {
15007                        module_id: "opener".to_string(),
15008                        launch_nonce: "opener-nonce".to_string(),
15009                    }),
15010                    None,
15011                ),
15012            )
15013            .await
15014            .expect("denied route.open returns a typed frame");
15015        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15016        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15017        assert!(
15018            target_rx.try_recv().is_err(),
15019            "forbidden route.open must not relay route.bind"
15020        );
15021    }
15022
15023    #[tokio::test]
15024    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15025        let registry = Arc::new(Registry::default());
15026        let forwarding = Arc::new(ForwardingTable::default());
15027        let supervisor = SupervisorHandle::new();
15028        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15029        let handler =
15030            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15031                .with_supervisor(supervisor);
15032        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15033        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15034        register_capability_manifest(
15035            &handler,
15036            &target_ctx,
15037            &mut target_rx,
15038            capability_manifest("target", &["credentials-provider/v1"], &[]),
15039            1,
15040        )
15041        .await;
15042        register_capability_manifest(
15043            &handler,
15044            &old_opener_ctx,
15045            &mut old_opener_rx,
15046            capability_manifest("opener", &[], &[]),
15047            2,
15048        )
15049        .await;
15050        let (mut client_rx, _) = open_route_for_capability_test(
15051            &handler,
15052            &target_ctx,
15053            &mut target_rx,
15054            712,
15055            3,
15056            "target",
15057            Some(ConsumerIdentity {
15058                module_id: "opener".to_string(),
15059                launch_nonce: "opener-nonce".to_string(),
15060            }),
15061        )
15062        .await;
15063        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15064
15065        handler
15066            .cleanup_connection(old_opener_ctx.connection_id)
15067            .expect("old opener registration cleans up");
15068        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15069        register_capability_manifest(
15070            &handler,
15071            &new_opener_ctx,
15072            &mut new_opener_rx,
15073            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15074            4,
15075        )
15076        .await;
15077
15078        assert_capability_denied_push(
15079            client_rx
15080                .try_recv()
15081                .expect("HELLO deny addition must emit route.closed")
15082                .frame,
15083            "target",
15084        );
15085        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15086        assert!(matches!(
15087            target_rx.try_recv(),
15088            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15089        ));
15090    }
15091
15092    #[tokio::test]
15093    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15094        let registry = Arc::new(Registry::default());
15095        let forwarding = Arc::new(ForwardingTable::default());
15096        let supervisor = SupervisorHandle::new();
15097        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15098        let handler =
15099            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15100                .with_supervisor(supervisor);
15101        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15102        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15103        register_capability_manifest(
15104            &handler,
15105            &target_ctx,
15106            &mut target_rx,
15107            capability_manifest("target", &[], &[]),
15108            1,
15109        )
15110        .await;
15111        register_capability_manifest(
15112            &handler,
15113            &opener_ctx,
15114            &mut opener_rx,
15115            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15116            2,
15117        )
15118        .await;
15119        let (mut client_rx, _) = open_route_for_capability_test(
15120            &handler,
15121            &target_ctx,
15122            &mut target_rx,
15123            722,
15124            3,
15125            "target",
15126            Some(ConsumerIdentity {
15127                module_id: "opener".to_string(),
15128                launch_nonce: "opener-nonce".to_string(),
15129            }),
15130        )
15131        .await;
15132        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15133
15134        let replies = handler
15135            .handle_control_frame(
15136                &target_ctx,
15137                catalog_update_with_capabilities_frame(
15138                    4,
15139                    CapabilityDeclarations {
15140                        provides: vec!["credentials-provider/v1".to_string()],
15141                        requires: Vec::new(),
15142                        must_never_reach: Vec::new(),
15143                    },
15144                ),
15145            )
15146            .await
15147            .expect("claim catalog.update succeeds");
15148        assert!(matches!(
15149            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15150            Ok(ModuleControlResponseToModule::CatalogUpdate {})
15151        ));
15152        assert_capability_denied_push(
15153            client_rx
15154                .try_recv()
15155                .expect("claim addition must emit route.closed")
15156                .frame,
15157            "target",
15158        );
15159        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15160        assert!(matches!(
15161            target_rx.try_recv(),
15162            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15163        ));
15164    }
15165
15166    #[tokio::test]
15167    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15168        let registry = Arc::new(Registry::default());
15169        let forwarding = Arc::new(ForwardingTable::default());
15170        let supervisor = SupervisorHandle::new();
15171        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15172        let handler =
15173            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15174                .with_supervisor(supervisor);
15175        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15176        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15177        register_capability_manifest(
15178            &handler,
15179            &target_ctx,
15180            &mut target_rx,
15181            capability_manifest("target", &["credentials-provider/v1"], &[]),
15182            1,
15183        )
15184        .await;
15185        register_capability_manifest(
15186            &handler,
15187            &opener_ctx,
15188            &mut opener_rx,
15189            capability_manifest("opener", &[], &[]),
15190            2,
15191        )
15192        .await;
15193        let (mut client_rx, _) = open_route_for_capability_test(
15194            &handler,
15195            &target_ctx,
15196            &mut target_rx,
15197            732,
15198            3,
15199            "target",
15200            Some(ConsumerIdentity {
15201                module_id: "opener".to_string(),
15202                launch_nonce: "opener-nonce".to_string(),
15203            }),
15204        )
15205        .await;
15206        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15207
15208        handler
15209            .handle_control_frame(
15210                &target_ctx,
15211                catalog_update_with_capabilities_frame(
15212                    4,
15213                    CapabilityDeclarations {
15214                        provides: Vec::new(),
15215                        requires: Vec::new(),
15216                        must_never_reach: Vec::new(),
15217                    },
15218                ),
15219            )
15220            .await
15221            .expect("claim removal catalog.update succeeds");
15222        assert_eq!(
15223            forwarding.active_binding_count().unwrap(),
15224            1,
15225            "removing an attested target claim must leave the route census unchanged"
15226        );
15227        assert!(
15228            client_rx.try_recv().is_err(),
15229            "claim removal must not emit route.closed capability_denied"
15230        );
15231        assert!(
15232            target_rx.try_recv().is_err(),
15233            "claim removal must not send the target a route GOODBYE"
15234        );
15235    }
15236
15237    /// A direct client may open a route to a denied capability provider; this
15238    /// policy applies only to attested supervised module origins, not to direct clients.
15239    #[tokio::test]
15240    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15241        let registry = Arc::new(Registry::default());
15242        let forwarding = Arc::new(ForwardingTable::default());
15243        let supervisor = SupervisorHandle::new();
15244        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15245        let handler =
15246            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15247                .with_supervisor(supervisor);
15248        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15249        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15250        register_capability_manifest(
15251            &handler,
15252            &target_ctx,
15253            &mut target_rx,
15254            capability_manifest("target", &["credentials-provider/v1"], &[]),
15255            1,
15256        )
15257        .await;
15258        register_capability_manifest(
15259            &handler,
15260            &opener_ctx,
15261            &mut opener_rx,
15262            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15263            2,
15264        )
15265        .await;
15266
15267        let (_client_rx, bind) = open_route_for_capability_test(
15268            &handler,
15269            &target_ctx,
15270            &mut target_rx,
15271            742,
15272            3,
15273            "target",
15274            None,
15275        )
15276        .await;
15277        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15278            panic!("direct scope-honesty route must bind");
15279        };
15280        assert_eq!(principal, Some(Principal::Direct));
15281        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15282    }
15283
15284    /// A module that denies a capability receives no self-route exemption when it
15285    /// also attestedly provides that capability.
15286    #[tokio::test]
15287    async fn must_never_reach_self_route_is_capability_forbidden() {
15288        let registry = Arc::new(Registry::default());
15289        let forwarding = Arc::new(ForwardingTable::default());
15290        let supervisor = SupervisorHandle::new();
15291        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15292        let handler =
15293            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15294                .with_supervisor(supervisor);
15295        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15296        register_capability_manifest(
15297            &handler,
15298            &self_ctx,
15299            &mut self_rx,
15300            capability_manifest(
15301                "self-provider",
15302                &["credentials-provider/v1"],
15303                &["credentials-provider/v1"],
15304            ),
15305            1,
15306        )
15307        .await;
15308
15309        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15310        let replies = handler
15311            .handle_control_frame(
15312                &client_ctx,
15313                route_open_frame_with_admission_facts(
15314                    2,
15315                    "self-provider",
15316                    unique_project_root("admission-facts"),
15317                    Some(ConsumerIdentity {
15318                        module_id: "self-provider".to_string(),
15319                        launch_nonce: "self-nonce".to_string(),
15320                    }),
15321                    None,
15322                ),
15323            )
15324            .await
15325            .expect("self-route refusal returns a typed frame");
15326        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15327        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15328        assert!(
15329            self_rx.try_recv().is_err(),
15330            "self denial must not relay route.bind"
15331        );
15332    }
15333
15334    #[test]
15335    fn unsupported_channel_zero_frame_returns_error() {
15336        let handler = ControlHandler::default();
15337        let request = Frame::build(
15338            FrameType::Request,
15339            control_flags(),
15340            0,
15341            0,
15342            21,
15343            b"opaque".to_vec(),
15344        )
15345        .unwrap();
15346
15347        let response = handler
15348            .handle_control(ConnectionId::new(1), request)
15349            .unwrap();
15350
15351        assert_eq!(response[0].header.ty, FrameType::Error);
15352        assert_eq!(
15353            parse_error(&response[0])["code"],
15354            "unsupported_control_frame"
15355        );
15356    }
15357
15358    /// Blue/green swap at the control-plane boundary. The supervisor that opens
15359    /// a swap is not wired yet, so the candidate is registered here directly
15360    /// into the registry and forwarding candidate slots, the way the swap's
15361    /// HELLO admission will.
15362    mod swap {
15363        use super::*;
15364
15365        const INCUMBENT: ConnectionId = ConnectionId::new(30);
15366        const CANDIDATE: ConnectionId = ConnectionId::new(40);
15367
15368        struct Swap {
15369            registry: Arc<Registry>,
15370            forwarding: Arc<ForwardingTable>,
15371            handler: ControlHandler,
15372            incumbent_ctx: RouteCtx,
15373            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15374            candidate_ctx: RouteCtx,
15375            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15376        }
15377
15378        async fn swap_with_incumbent() -> Swap {
15379            let registry = Arc::new(Registry::default());
15380            let forwarding = Arc::new(ForwardingTable::default());
15381            let handler =
15382                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15383            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15384            hello_via_sink(
15385                &handler,
15386                &incumbent_ctx,
15387                &mut incumbent_rx,
15388                hello_frame("aft", PROTOCOL_VERSION, 7),
15389            )
15390            .await;
15391            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15392            Swap {
15393                registry,
15394                forwarding,
15395                handler,
15396                incumbent_ctx,
15397                incumbent_rx,
15398                candidate_ctx,
15399                candidate_rx,
15400            }
15401        }
15402
15403        fn register_candidate(swap: &Swap, ready: Option<bool>) {
15404            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15405            candidate_manifest.ready = ready;
15406            let registration = swap
15407                .registry
15408                .register_candidate_with_control_ops(
15409                    candidate_manifest,
15410                    PROTOCOL_VERSION,
15411                    CANDIDATE,
15412                    module_baseline_control_ops(),
15413                )
15414                .unwrap();
15415            swap.forwarding
15416                .register_candidate_module_connection(
15417                    CANDIDATE,
15418                    "aft".to_string(),
15419                    PROTOCOL_VERSION,
15420                    manifest_concurrency(&registration.manifest),
15421                    swap.candidate_ctx.egress.clone(),
15422                )
15423                .unwrap();
15424        }
15425
15426        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15427            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15428            swap.registry.promote_candidate("aft").unwrap().unwrap();
15429            cutover.incumbent.unwrap()
15430        }
15431
15432        /// A swap candidate's HELLO reserves its candidate registry entry
15433        /// before installing its candidate forwarding endpoint. The swap
15434        /// supervisor promotes a ready candidate as soon as it reads it as
15435        /// registered, and promotion cuts forwarding over to the candidate
15436        /// endpoint, so the lookup the supervisor reads
15437        /// (`Registry::published_registration`) must not return the candidate
15438        /// until that endpoint exists. Otherwise the cutover finds no candidate
15439        /// endpoint and the swap fails as if the candidate had disconnected.
15440        #[tokio::test]
15441        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15442            let mut swap = swap_with_incumbent().await;
15443            let hold = Arc::new(std::sync::Barrier::new(2));
15444            let handler = swap
15445                .handler
15446                .clone()
15447                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15448            handler
15449                .supervisor
15450                .open_swap("aft", "candidate-nonce".to_string());
15451            let hello = {
15452                let handler = handler.clone();
15453                let candidate_ctx = swap.candidate_ctx.clone();
15454                std::thread::spawn(move || {
15455                    handler
15456                        .handle_hello(
15457                            candidate_ctx.connection_id,
15458                            Some(candidate_ctx.egress.clone()),
15459                            hello_frame_with_nonce(
15460                                "aft",
15461                                PROTOCOL_VERSION,
15462                                8,
15463                                Some("candidate-nonce"),
15464                            ),
15465                        )
15466                        .unwrap()
15467                })
15468            };
15469
15470            hold.wait();
15471            // Parked: the candidate slot is claimed but not published, and
15472            // forwarding has no candidate endpoint.
15473            assert!(swap
15474                .registry
15475                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15476                .unwrap()
15477                .is_some());
15478            assert!(swap
15479                .registry
15480                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15481                .unwrap()
15482                .is_none());
15483            assert!(swap
15484                .forwarding
15485                .module_endpoint_for_connection(CANDIDATE)
15486                .unwrap()
15487                .is_none());
15488
15489            hold.wait();
15490            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15491            assert_eq!(
15492                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15493                FrameType::HelloAck
15494            );
15495            let published = swap
15496                .registry
15497                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15498                .unwrap()
15499                .expect("candidate is published once its HELLO completes");
15500            assert_eq!(published.connection_id, CANDIDATE);
15501            assert!(swap
15502                .forwarding
15503                .module_endpoint_for_connection(CANDIDATE)
15504                .unwrap()
15505                .is_some());
15506        }
15507
15508        fn keyed_total(counters: &Value, key: &str) -> u64 {
15509            counters[key]
15510                .as_object()
15511                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15512                .unwrap_or(0)
15513        }
15514
15515        #[tokio::test]
15516        async fn replacement_logs_old_end_and_new_admission_once() {
15517            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15518            let mut swap = swap_with_incumbent().await;
15519            swap.handler
15520                .supervisor
15521                .open_swap("aft", "candidate-nonce".to_string());
15522            hello_via_sink(
15523                &swap.handler,
15524                &swap.candidate_ctx,
15525                &mut swap.candidate_rx,
15526                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15527            )
15528            .await;
15529            cutover(&swap);
15530            swap.handler.cleanup_connection(INCUMBENT).unwrap();
15531
15532            let captured = crate::router::test_log::captured_logs(&logs);
15533            println!("captured replacement registry logs:\n{captured}");
15534            let new_admissions: Vec<_> = captured
15535                .lines()
15536                .filter(|line| {
15537                    line.contains("connection_id=40")
15538                        && line.contains("swap candidate registered; not routable until cutover")
15539                })
15540                .collect();
15541            assert_eq!(
15542                new_admissions.len(),
15543                1,
15544                "unexpected admission logs: {captured}"
15545            );
15546            assert!(new_admissions[0].contains("module_id=aft"));
15547            assert!(new_admissions[0].contains("connection_id=40"));
15548            assert!(
15549                new_admissions[0].contains("swap candidate registered; not routable until cutover")
15550            );
15551            assert!(new_admissions[0].contains("ready=true"));
15552
15553            let old_admissions: Vec<_> = captured
15554                .lines()
15555                .filter(|line| {
15556                    line.contains("module registered module_id=aft ")
15557                        && line.contains("connection_id=30")
15558                })
15559                .collect();
15560            assert_eq!(
15561                old_admissions.len(),
15562                1,
15563                "unexpected incumbent admission: {captured}"
15564            );
15565
15566            let old_ends: Vec<_> = captured
15567                .lines()
15568                .filter(|line| {
15569                    line.contains("connection_id=30") && line.contains("module registration ended")
15570                })
15571                .collect();
15572            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15573            assert!(old_ends[0].contains("module_id=aft"));
15574            assert!(old_ends[0].contains("connection_id=30"));
15575            assert!(old_ends[0].contains("reason=replaced"));
15576            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15577            assert_eq!(
15578                captured
15579                    .lines()
15580                    .filter(|line| line.contains("module registration promoted"))
15581                    .count(),
15582                1,
15583                "unexpected promotion logs: {captured}"
15584            );
15585        }
15586
15587        /// An ack from the incumbent for a bind it was sent before cutover,
15588        /// arriving before the incumbent is drained. The incumbent is the live
15589        /// connection carrying every other client's routes, so the ack must
15590        /// not end it: the waiting client is told to retry, the reservation is
15591        /// given back, and the incumbent is told to drop just that binding.
15592        #[tokio::test]
15593        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15594            let mut swap = swap_with_incumbent().await;
15595            let handler = swap.handler.clone();
15596
15597            // A co-tenant route, bound on the incumbent before the swap.
15598            let cotenant = ConnectionId::new(31);
15599            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15600            let (cotenant_task, cotenant_bind) = relay_route_open(
15601                &handler,
15602                cotenant,
15603                &cotenant_ctx.egress,
15604                &mut swap.incumbent_rx,
15605                100,
15606                "aft",
15607                "swap-cotenant",
15608            )
15609            .await;
15610            handler
15611                .handle_control_frame(
15612                    &swap.incumbent_ctx,
15613                    route_bind_ack(cotenant_bind.header.corr),
15614                )
15615                .await
15616                .unwrap();
15617            assert!(cotenant_task.await.unwrap().is_empty());
15618            let (cotenant_channel, cotenant_epoch) =
15619                published_route(&cotenant_rx.recv().await.unwrap());
15620
15621            // A second route.open, relayed to the incumbent and not yet acked.
15622            let caller = ConnectionId::new(32);
15623            let (caller_ctx, mut caller_rx) = route_ctx(caller);
15624            let (caller_task, caller_bind) = relay_route_open(
15625                &handler,
15626                caller,
15627                &caller_ctx.egress,
15628                &mut swap.incumbent_rx,
15629                101,
15630                "aft",
15631                "swap-caller",
15632            )
15633            .await;
15634            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15635
15636            register_candidate(&swap, None);
15637            cutover(&swap);
15638
15639            // The incumbent acks after promotion and before any drain.
15640            let ack = handler
15641                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15642                .await;
15643            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15644            if module_loop_error.is_some() {
15645                // What the connection loop does with an untranslated router
15646                // error: end the connection, releasing every route on it.
15647                handler.cleanup_connection(INCUMBENT).unwrap();
15648            }
15649
15650            // 1. The incumbent's other routes survive.
15651            assert!(
15652                cotenant_rx.try_recv().is_err(),
15653                "the co-tenant route on the incumbent was torn down by one late ack: \
15654                 {module_loop_error:?}"
15655            );
15656            assert!(matches!(
15657                swap.forwarding
15658                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15659                    .unwrap(),
15660                DataRoute::Client(DataRouteState::Bound(_))
15661            ));
15662            assert_eq!(module_loop_error, None);
15663            assert!(swap
15664                .registry
15665                .get_module_by_connection(INCUMBENT)
15666                .unwrap()
15667                .is_some());
15668
15669            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
15670            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15671                .await
15672                .expect("the incumbent is told to drop the abandoned binding")
15673                .unwrap()
15674                .frame;
15675            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15676            assert_eq!(goodbye.header.channel, abandoned_channel);
15677            assert_eq!(goodbye.header.epoch, abandoned_epoch);
15678            assert!(swap.incumbent_rx.try_recv().is_err());
15679
15680            // 3. The waiting client gets a retryable refusal and no route.
15681            let response = caller_task.await.unwrap();
15682            assert_eq!(response.len(), 1);
15683            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15684            assert!(caller_rx.try_recv().is_err());
15685
15686            // 4. The reservation pair is given back, and the pending bind
15687            //    settled exactly once: one accepted open (the co-tenant) and one
15688            //    refused open (the caller), nothing counted twice.
15689            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15690            let counters = handler.counters().snapshot();
15691            assert_eq!(
15692                keyed_total(&counters, "route_open_accepted_by_principal"),
15693                1
15694            );
15695            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15696            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15697        }
15698
15699        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
15700        /// id would resolve to the promoted candidate and every new route.open
15701        /// would be refused as reloading, leaving neither process routable.
15702        #[tokio::test]
15703        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15704            let mut swap = swap_with_incumbent().await;
15705            register_candidate(&swap, None);
15706            let incumbent = cutover(&swap);
15707            swap.forwarding
15708                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15709                .unwrap()
15710                .expect("the incumbent is still registered");
15711
15712            let client = ConnectionId::new(33);
15713            let (client_ctx, mut client_rx) = route_ctx(client);
15714            let route_handler = swap.handler.clone();
15715            let open_ctx = RouteCtx {
15716                connection_id: client,
15717                egress: client_ctx.egress.clone(),
15718            };
15719            let mut route_task = tokio::spawn(async move {
15720                route_handler
15721                    .handle_control_frame(
15722                        &open_ctx,
15723                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15724                    )
15725                    .await
15726                    .unwrap()
15727            });
15728            let bind = tokio::select! {
15729                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15730                response = &mut route_task => {
15731                    let response = response.unwrap();
15732                    panic!(
15733                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
15734                        parse_error(&response[0])["code"]
15735                    );
15736                }
15737            };
15738            swap.handler
15739                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15740                .await
15741                .unwrap();
15742            assert!(route_task.await.unwrap().is_empty());
15743            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15744            match swap
15745                .forwarding
15746                .lookup_data_route(client, channel, epoch)
15747                .unwrap()
15748            {
15749                DataRoute::Client(DataRouteState::Bound(route)) => {
15750                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15751                }
15752                other => panic!("expected a bound route on the candidate, got {other:?}"),
15753            }
15754            assert!(swap.incumbent_rx.try_recv().is_err());
15755        }
15756
15757        /// A candidate declares itself ready with `catalog.update` on its own
15758        /// connection. If the connection-keyed registry lookups searched only the
15759        /// active slot, this would answer `not_registered` and the candidate
15760        /// would never become ready.
15761        #[tokio::test]
15762        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15763            let swap = swap_with_incumbent().await;
15764            register_candidate(&swap, Some(false));
15765            let update = Frame::build(
15766                FrameType::Request,
15767                control_flags(),
15768                0,
15769                0,
15770                55,
15771                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15772                    provides: manifest("aft", PROTOCOL_VERSION).provides,
15773                    capabilities: None,
15774                    ready: Some(true),
15775                })
15776                .unwrap(),
15777            )
15778            .unwrap();
15779
15780            let replies = swap
15781                .handler
15782                .handle_control_frame(&swap.candidate_ctx, update)
15783                .await
15784                .unwrap();
15785
15786            assert_eq!(replies.len(), 1);
15787            assert_eq!(
15788                replies[0].header.ty,
15789                FrameType::Response,
15790                "candidate catalog.update was refused: {:?}",
15791                serde_json::from_slice::<Value>(&replies[0].body).ok()
15792            );
15793            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15794            assert_eq!(
15795                swap.registry
15796                    .get_module("aft")
15797                    .unwrap()
15798                    .unwrap()
15799                    .connection_id,
15800                INCUMBENT
15801            );
15802        }
15803    }
15804
15805    /// The HELLO gate while the supervisor has a swap open: only the nonce it
15806    /// minted for the candidate admits a second process, into the candidate
15807    /// slot, and that check runs ahead of the reserved-module gate.
15808    mod swap_admission {
15809        use super::*;
15810
15811        const INCUMBENT_NONCE: &str = "incumbent-nonce";
15812        const CANDIDATE_NONCE: &str = "candidate-nonce";
15813
15814        fn handler_with_incumbent(
15815            module_id: &str,
15816            reserved: bool,
15817        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15818            let registry = Arc::new(Registry::default());
15819            let supervisor = SupervisorHandle::new();
15820            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15821            if reserved {
15822                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15823            }
15824            let handler =
15825                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
15826            let incumbent = handler
15827                .handle_control(
15828                    ConnectionId::new(1),
15829                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15830                )
15831                .unwrap();
15832            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15833            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15834            (registry, supervisor, handler)
15835        }
15836
15837        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
15838        /// admits every nonce, so while a swap is open the swap gate is the only
15839        /// thing between a key-holder and the candidate slot. A nonce the
15840        /// supervisor did not mint, or none at all, is refused, and neither the
15841        /// incumbent's registration nor the candidate slot moves.
15842        #[test]
15843        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15844            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15845
15846            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15847                let replies = handler
15848                    .handle_control(
15849                        ConnectionId::new(connection),
15850                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15851                    )
15852                    .unwrap();
15853                assert_eq!(replies[0].header.ty, FrameType::Error);
15854                assert_eq!(
15855                    parse_error(&replies[0])["code"],
15856                    "swap_token_invalid",
15857                    "nonce {nonce:?}"
15858                );
15859            }
15860            assert!(registry.get_candidate("aft").unwrap().is_none());
15861            assert_eq!(
15862                registry.get_module("aft").unwrap().unwrap().connection_id,
15863                ConnectionId::new(1)
15864            );
15865
15866            // Control: the minted token is admitted, into the candidate slot,
15867            // and only once.
15868            let admitted = handler
15869                .handle_control(
15870                    ConnectionId::new(4),
15871                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15872                )
15873                .unwrap();
15874            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15875            assert_eq!(
15876                registry
15877                    .get_candidate("aft")
15878                    .unwrap()
15879                    .unwrap()
15880                    .connection_id,
15881                ConnectionId::new(4)
15882            );
15883            assert_eq!(
15884                registry.get_module("aft").unwrap().unwrap().connection_id,
15885                ConnectionId::new(1),
15886                "the candidate must not take the active slot"
15887            );
15888            let replayed = handler
15889                .handle_control(
15890                    ConnectionId::new(5),
15891                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15892                )
15893                .unwrap();
15894            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15895
15896            // The case only this gate covers: the incumbent has died mid-swap,
15897            // so its duplicate refusal is gone too, and without the gate a
15898            // key-holder would take the id's ACTIVE slot.
15899            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15900            let squatter = handler
15901                .handle_control(
15902                    ConnectionId::new(6),
15903                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15904                )
15905                .unwrap();
15906            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15907            assert!(
15908                registry.get_module("aft").unwrap().is_none(),
15909                "a squatter took the active slot of an id being swapped"
15910            );
15911        }
15912
15913        /// Design mutation arm (iii). A reserved module's candidate presents a
15914        /// nonce the reserved gate has never seen (that gate holds the
15915        /// incumbent's), so the swap gate must run first or the candidate is
15916        /// refused `reserved_module` and a reserved module can never be swapped.
15917        #[test]
15918        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15919            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15920
15921            let replies = handler
15922                .handle_control(
15923                    ConnectionId::new(2),
15924                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15925                )
15926                .unwrap();
15927
15928            assert_eq!(
15929                replies[0].header.ty,
15930                FrameType::HelloAck,
15931                "reserved candidate refused: {:?}",
15932                serde_json::from_slice::<Value>(&replies[0].body).ok()
15933            );
15934            assert_eq!(
15935                registry
15936                    .get_candidate("vault")
15937                    .unwrap()
15938                    .unwrap()
15939                    .connection_id,
15940                ConnectionId::new(2)
15941            );
15942        }
15943
15944        /// With no swap open the gate is inert: the incumbent's reserved gate
15945        /// and duplicate refusal behave exactly as before.
15946        #[test]
15947        fn without_an_open_swap_the_ordinary_gates_decide() {
15948            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15949            supervisor.close_swap("vault");
15950
15951            let candidate = handler
15952                .handle_control(
15953                    ConnectionId::new(2),
15954                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15955                )
15956                .unwrap();
15957            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15958            let duplicate = handler
15959                .handle_control(
15960                    ConnectionId::new(3),
15961                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15962                )
15963                .unwrap();
15964            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15965            assert!(registry.get_candidate("vault").unwrap().is_none());
15966        }
15967    }
15968
15969    /// `scope.sync` and `scope.describe` through the real control handler: who
15970    /// may sync is decided by the registration and launch nonce of the module
15971    /// connection, never by the request body.
15972    mod scopes {
15973        mod idle_tests {
15974            use super::*;
15975            include!("scope_idle_tests.rs");
15976        }
15977        mod route_open_timings {
15978            use super::*;
15979            include!("route_open_timing_tests.rs");
15980        }
15981        mod route_open_refusal_log {
15982            use super::*;
15983            include!("route_open_refusal_log_tests.rs");
15984        }
15985        mod activity {
15986            include!("activity_scope_tests.rs");
15987        }
15988        mod delta {
15989            include!("scope_handler_delta_tests.rs");
15990        }
15991        mod call_modes {
15992            include!("scope_call_mode_tests.rs");
15993        }
15994        #[tokio::test]
15995        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15996            let rig = rig().await;
15997            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15998                "s".into(),
15999                1,
16000                "tool".into(),
16001            ))
16002            .unwrap();
16003            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16004            let (direct, _rx) = wide_ctx(999);
16005            for (ctx, expected) in [
16006                (&rig.owner, FrameType::Response),
16007                (&direct, FrameType::Error),
16008            ] {
16009                let frame =
16010                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16011                        .unwrap();
16012                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16013                assert_eq!(replies[0].header.ty, expected);
16014                if expected == FrameType::Response {
16015                    let reply: subc_control::ScopeServedByReply =
16016                        serde_json::from_slice(&replies[0].body).unwrap();
16017                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16018                } else {
16019                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16020                }
16021            }
16022            let reply = call(
16023                &rig.handler,
16024                &rig.owner,
16025                &ModuleControlRequestFromModule::ScopeDescribe {
16026                    owner: Principal::Reserved {
16027                        module_id: OWNER.into(),
16028                    },
16029                    scope_ref: "s".into(),
16030                },
16031            )
16032            .await;
16033            assert!(matches!(
16034                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16035                ModuleControlResponseToModule::ScopeDescribe { .. }
16036            ));
16037        }
16038
16039        use subc_protocol::scope::{
16040            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16041            ScopeStatus,
16042        };
16043
16044        use super::*;
16045
16046        const OWNER: &str = "prefrontal-core";
16047
16048        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16049            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16050        }
16051
16052        async fn call(
16053            handler: &ControlHandler,
16054            ctx: &RouteCtx,
16055            request: &ModuleControlRequestFromModule,
16056        ) -> Frame {
16057            let body = serde_json::to_vec(request).unwrap();
16058            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16059            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16060            assert_eq!(replies.len(), 1, "{replies:?}");
16061            replies.pop().unwrap()
16062        }
16063
16064        async fn sync(
16065            handler: &ControlHandler,
16066            ctx: &RouteCtx,
16067            generation: u64,
16068            scopes: Vec<ScopeRecord>,
16069        ) -> Result<ModuleControlResponseToModule, String> {
16070            let reply = call(
16071                handler,
16072                ctx,
16073                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16074            )
16075            .await;
16076            match reply.header.ty {
16077                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16078                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16079            }
16080        }
16081
16082        async fn describe(
16083            handler: &ControlHandler,
16084            ctx: &RouteCtx,
16085            owner: &str,
16086            scope_ref: &str,
16087        ) -> ModuleControlResponseToModule {
16088            let reply = call(
16089                handler,
16090                ctx,
16091                &ModuleControlRequestFromModule::ScopeDescribe {
16092                    owner: Principal::Reserved {
16093                        module_id: owner.to_string(),
16094                    },
16095                    scope_ref: scope_ref.to_string(),
16096                },
16097            )
16098            .await;
16099            assert_eq!(
16100                reply.header.ty,
16101                FrameType::Response,
16102                "{:?}",
16103                parse_error(&reply)
16104            );
16105            serde_json::from_slice(&reply.body).unwrap()
16106        }
16107
16108        #[tokio::test]
16109        async fn scope_describe_never_echoes_bind_proof_or_hash() {
16110            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16111            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16112            let mut rig = rig().await;
16113            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16114                .await;
16115            let _bound = rig
16116                .bound(
16117                    Some(AFT),
16118                    PLEXUS,
16119                    Some(
16120                        rig.selector("s", Some(1))
16121                            .with_bind_proof(Some(PROOF.into())),
16122                    ),
16123                )
16124                .await;
16125            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16126            let output = serde_json::to_string(&reply).unwrap();
16127            assert!(output.contains("\"status\":\"live\""));
16128            assert!(!output.contains(PROOF));
16129            assert!(!output.contains(HASH));
16130        }
16131
16132        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
16133        async fn module(
16134            handler: &ControlHandler,
16135            connection: u64,
16136            module_id: &str,
16137            nonce: Option<&str>,
16138        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16139            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16140            hello_via_sink(
16141                handler,
16142                &ctx,
16143                &mut rx,
16144                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16145            )
16146            .await;
16147            (ctx, rx)
16148        }
16149
16150        /// `direct` and every other client connection has no registration, so
16151        /// it can neither sync nor own a scope.
16152        #[tokio::test]
16153        async fn a_client_connection_cannot_sync_or_describe() {
16154            let handler = ControlHandler::new(Arc::new(Registry::default()));
16155            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16156            for request in [
16157                ModuleControlRequestFromModule::ScopeSync {
16158                    generation: 1,
16159                    scopes: vec![head("s", 1)],
16160                },
16161                ModuleControlRequestFromModule::ScopeDescribe {
16162                    owner: Principal::Direct,
16163                    scope_ref: "s".to_string(),
16164                },
16165            ] {
16166                let reply = call(&handler, &ctx, &request).await;
16167                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16168            }
16169            assert!(
16170                !handler
16171                    .scopes
16172                    .read()
16173                    .unwrap()
16174                    .describe(
16175                        &Principal::Reserved {
16176                            module_id: OWNER.to_string()
16177                        },
16178                        "s"
16179                    )
16180                    .owner_synced
16181            );
16182        }
16183
16184        /// A module the supervisor did not spawn registers without a launch
16185        /// nonce, so it is never an owner's current launch.
16186        #[tokio::test]
16187        async fn a_module_without_a_supervised_launch_cannot_sync() {
16188            let handler = ControlHandler::new(Arc::new(Registry::default()));
16189            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16190            assert_eq!(
16191                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16192                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16193            );
16194        }
16195
16196        #[tokio::test]
16197        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16198            let supervisor = SupervisorHandle::new();
16199            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16200            let handler = ControlHandler::new(Arc::new(Registry::default()))
16201                .with_supervisor(supervisor.clone());
16202            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16203            sync(&handler, &incumbent, 1, vec![head("s", 1)])
16204                .await
16205                .expect("the current launch syncs");
16206
16207            // A swap candidate registers with the swap token and is refused
16208            // while the incumbent keeps syncing.
16209            supervisor.open_swap(OWNER, "n2".to_string());
16210            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16211            assert_eq!(
16212                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16213                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16214            );
16215            sync(&handler, &incumbent, 2, vec![head("s", 1)])
16216                .await
16217                .expect("the serving owner syncs during the swap");
16218
16219            // The swap fails and is rolled back. The candidate never held sync
16220            // authority, and still cannot sync.
16221            supervisor.close_swap(OWNER);
16222            assert_eq!(
16223                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16224                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16225            );
16226            sync(&handler, &incumbent, 3, vec![head("s", 1)])
16227                .await
16228                .expect("the serving owner syncs after the rollback");
16229            handler.cleanup_connection(candidate.connection_id).unwrap();
16230
16231            // A swap that cuts over. Promotion records the candidate's nonce as
16232            // the module's spawn nonce, which is what `set_spawn_nonce` does
16233            // here; the promoted connection then takes authority at any
16234            // generation and the superseded incumbent is refused.
16235            supervisor.open_swap(OWNER, "n3".to_string());
16236            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16237            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16238            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16239                .await
16240                .expect("the promoted launch takes authority");
16241            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16242                panic!("unexpected reply {reply:?}");
16243            };
16244            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16245            assert_eq!(
16246                sync(&handler, &incumbent, 4, Vec::new()).await,
16247                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16248            );
16249        }
16250
16251        /// Authority dies with its connection: the cleanup path releases it,
16252        /// so the owner's next connection takes it at any generation.
16253        #[tokio::test]
16254        async fn closing_the_authority_connection_frees_sync_authority() {
16255            let supervisor = SupervisorHandle::new();
16256            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16257            let handler = ControlHandler::new(Arc::new(Registry::default()))
16258                .with_supervisor(supervisor.clone());
16259            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16260            sync(&handler, &first, 10, vec![head("s", 1)])
16261                .await
16262                .unwrap();
16263            handler.cleanup_connection(first.connection_id).unwrap();
16264
16265            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16266            sync(&handler, &second, 1, vec![head("s", 1)])
16267                .await
16268                .expect("the next connection takes the released authority");
16269        }
16270
16271        #[tokio::test]
16272        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16273            let supervisor = SupervisorHandle::new();
16274            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16275            let handler =
16276                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16277            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16278            sync(&handler, &first, 10, vec![head("s", 1)])
16279                .await
16280                .unwrap();
16281            handler
16282                .handle_control_frame(
16283                    &first,
16284                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16285                )
16286                .await
16287                .unwrap();
16288            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16289            sync(&handler, &second, 1, vec![head("s", 1)])
16290                .await
16291                .expect("GOODBYE releases authority even if the old socket remains open");
16292        }
16293
16294        #[tokio::test]
16295        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16296            let registry = Arc::new(Registry::default());
16297            let supervisor_handle = SupervisorHandle::new();
16298            let supervisor =
16299                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16300                    .with_handle(supervisor_handle.clone())
16301                    .with_daemon_incarnation("incarnation-7".to_string());
16302            // Configured with enabled: false, so the supervisor lists the
16303            // module without spawning a process for it.
16304            supervisor
16305                .supervise_configured(
16306                    ModuleSpec {
16307                        module_id: OWNER.to_string(),
16308                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16309                        args: Vec::new(),
16310                        env: Vec::new(),
16311                        reserved: false,
16312                        reserved_prefixes: Vec::new(),
16313                        protocol: ModuleProtocol::Subc,
16314                        overlap: Default::default(),
16315                    },
16316                    false,
16317                )
16318                .unwrap();
16319            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16320            let handler =
16321                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
16322            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16323
16324            // Configured but not yet synced: a reader waits for the owner.
16325            let ModuleControlResponseToModule::ScopeDescribe {
16326                status,
16327                daemon_incarnation,
16328                owner_synced,
16329                owner_configured,
16330                scope,
16331                ..
16332            } = describe(&handler, &reader, OWNER, "s").await
16333            else {
16334                panic!("not a describe reply");
16335            };
16336            assert_eq!(status, ScopeStatus::NotLive);
16337            assert_eq!(daemon_incarnation, "incarnation-7");
16338            assert!(!owner_synced);
16339            assert!(owner_configured);
16340            assert!(scope.is_none());
16341
16342            // Not a supervised module: the owner will never sync, and a reader
16343            // refuses rather than waits.
16344            let ModuleControlResponseToModule::ScopeDescribe {
16345                status,
16346                owner_configured,
16347                ..
16348            } = describe(&handler, &reader, "ghost", "s").await
16349            else {
16350                panic!("not a describe reply");
16351            };
16352            assert_eq!(status, ScopeStatus::NotLive);
16353            assert!(!owner_configured);
16354
16355            // Live, with the stamp fields and the computed owner_authorized.
16356            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16357            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16358            let ModuleControlResponseToModule::ScopeDescribe {
16359                status,
16360                scope_epoch,
16361                owner_synced,
16362                scope,
16363                ..
16364            } = describe(&handler, &reader, OWNER, "s").await
16365            else {
16366                panic!("not a describe reply");
16367            };
16368            assert_eq!(status, ScopeStatus::Live);
16369            assert_eq!(scope_epoch, Some(4));
16370            assert!(owner_synced);
16371            let stamp = scope.expect("a live scope carries its stamp");
16372            assert!(
16373                stamp.owner_authorized,
16374                "prefrontal-core is the default authority"
16375            );
16376            assert_eq!(stamp.kind, ScopeKind::Head);
16377        }
16378
16379        #[tokio::test]
16380        async fn scope_authority_owners_decides_owner_authorized() {
16381            let supervisor = SupervisorHandle::new();
16382            supervisor.set_spawn_nonce("broca", "b1".to_string());
16383            let handler = ControlHandler::new(Arc::new(Registry::default()))
16384                .with_supervisor(supervisor)
16385                .with_scope_authority_owners(vec!["broca".to_string()]);
16386            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16387            let mut gated = head("s", 1);
16388            gated.attributes.agent_id = Some("agent".to_string());
16389            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16390            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16391                describe(&handler, &broca, "broca", "s").await
16392            else {
16393                panic!("not a describe reply");
16394            };
16395            assert!(scope.unwrap().owner_authorized);
16396        }
16397
16398        /// With route admission, the stamp, the commit re-check and drains in
16399        /// place, the feature is advertised: the module ops in HELLO_ACK, and
16400        /// `scopes/v1` in HELLO_ACK and `server.describe`.
16401        #[tokio::test]
16402        async fn scope_ops_and_the_scopes_capability_are_advertised() {
16403            let handler = ControlHandler::new(Arc::new(Registry::default()));
16404            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16405            let ack = hello_via_sink(
16406                &handler,
16407                &ctx,
16408                &mut rx,
16409                hello_frame("m", PROTOCOL_VERSION, 1),
16410            )
16411            .await;
16412            let ack = parse_ack(&ack);
16413            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16414                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16415            }
16416            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16417
16418            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16419            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16420            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16421            let reply = handler
16422                .handle_control_frame(&client, frame)
16423                .await
16424                .unwrap()
16425                .pop()
16426                .unwrap();
16427            let ClientControlResponse::ServerDescribe { capabilities, .. } =
16428                serde_json::from_slice(&reply.body).unwrap()
16429            else {
16430                panic!("not a server.describe reply");
16431            };
16432            assert!(
16433                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16434                "{capabilities:?}"
16435            );
16436        }
16437
16438        // ---- route admission, stamps, commit re-check and drains ----------
16439
16440        const PLEXUS: &str = "plexus";
16441        const OTHER: &str = "other";
16442        const AFT: &str = "aft";
16443        const BROCA: &str = "broca";
16444        const MAGIC: &str = "magic-context";
16445
16446        fn nonce(module_id: &str) -> String {
16447            format!("nonce-{module_id}")
16448        }
16449
16450        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16451            let (tx, rx) = mpsc::channel(64);
16452            (
16453                RouteCtx {
16454                    connection_id: ConnectionId::new(connection),
16455                    egress: FrameSink::new(tx),
16456                },
16457                rx,
16458            )
16459        }
16460
16461        /// A daemon with a configured owner (prefrontal-core) registered on its
16462        /// own module connection, two routable targets (plexus, other), and
16463        /// launch nonces minted for the modules that open routes as carriers.
16464        struct Rig {
16465            handler: ControlHandler,
16466            forwarding: Arc<ForwardingTable>,
16467            owner: RouteCtx,
16468            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16469            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16470            generation: u64,
16471            next_connection: u64,
16472            _supervisor: Supervisor,
16473        }
16474
16475        async fn rig() -> Rig {
16476            rig_with_flow_support(true).await
16477        }
16478
16479        async fn rig_with_flow_support(flow_support: bool) -> Rig {
16480            rig_with_scope_capabilities(flow_support, false).await
16481        }
16482
16483        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16484            let mut provides = Vec::new();
16485            if flow_support {
16486                provides.push("flow-scopes/v1");
16487            }
16488            if run_support {
16489                provides.push("agent-run-scopes/v1");
16490            }
16491            rig_with_target_capabilities(&provides).await
16492        }
16493
16494        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16495            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16496            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16497            // Decoder versions alone cannot admit scoped calls: only a capability
16498            // declaration promises the required run or flow behaviour.
16499            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16500            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16501            Frame::build(
16502                FrameType::Hello,
16503                control_flags(),
16504                0,
16505                0,
16506                connection,
16507                serde_json::to_vec(&body).unwrap(),
16508            )
16509            .unwrap()
16510        }
16511
16512        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16513            let registry = Arc::new(Registry::default());
16514            let forwarding = Arc::new(ForwardingTable::default());
16515            let supervisor_handle = SupervisorHandle::new();
16516            let supervisor =
16517                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16518                    .with_handle(supervisor_handle.clone());
16519            supervisor
16520                .supervise_configured(
16521                    ModuleSpec {
16522                        module_id: OWNER.to_string(),
16523                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16524                        args: Vec::new(),
16525                        env: Vec::new(),
16526                        reserved: false,
16527                        reserved_prefixes: Vec::new(),
16528                        protocol: ModuleProtocol::Subc,
16529                        overlap: Default::default(),
16530                    },
16531                    false,
16532                )
16533                .unwrap();
16534            for module_id in [OWNER, AFT, BROCA, MAGIC] {
16535                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16536            }
16537            let handler =
16538                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16539                    .with_supervisor(supervisor_handle);
16540            let (owner, mut owner_rx) = wide_ctx(1);
16541            hello_via_sink(
16542                &handler,
16543                &owner,
16544                &mut owner_rx,
16545                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16546            )
16547            .await;
16548            let mut modules = BTreeMap::new();
16549            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16550                let (ctx, mut rx) = wide_ctx(connection);
16551                let hello = hello_with_capabilities(module_id, connection, provides);
16552                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16553                modules.insert(module_id.to_string(), (ctx, rx));
16554            }
16555            Rig {
16556                handler,
16557                forwarding,
16558                owner,
16559                _owner_rx: owner_rx,
16560                modules,
16561                generation: 0,
16562                next_connection: 100,
16563                _supervisor: supervisor,
16564            }
16565        }
16566
16567        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16568            ScopeCarrier::new(Principal::Reserved {
16569                module_id: module_id.to_string(),
16570            })
16571            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16572        }
16573
16574        /// The scope most tests open under: aft carries to any module, broca
16575        /// only to plexus and other, and the owner delegates as agent-1.
16576        fn session(scope_epoch: u64) -> ScopeRecord {
16577            let mut record = head("s", scope_epoch);
16578            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16579            record.attributes.agent_id = Some("agent-1".to_string());
16580            record.attributes.delegates = true;
16581            record
16582        }
16583
16584        impl Rig {
16585            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16586                self.generation += 1;
16587                sync(&self.handler, &self.owner, self.generation, scopes)
16588                    .await
16589                    .expect("the owner's sync is accepted");
16590            }
16591
16592            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16593                ScopeSelector::new(
16594                    Principal::Reserved {
16595                        module_id: OWNER.to_string(),
16596                    },
16597                    scope_ref.to_string(),
16598                    scope_epoch,
16599                )
16600            }
16601
16602            fn open_frame(
16603                &mut self,
16604                opener: Option<&str>,
16605                target: &str,
16606                scope: Option<ScopeSelector>,
16607            ) -> (
16608                RouteCtx,
16609                mpsc::Receiver<crate::router::OutboundFrame>,
16610                Frame,
16611            ) {
16612                self.next_connection += 1;
16613                let (ctx, rx) = wide_ctx(self.next_connection);
16614                let root = unique_project_root("scoped-open");
16615                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16616                    target: RouteTarget::ToolProvider {
16617                        module_id: target.to_string(),
16618                    },
16619                    identity: BindIdentity::new(
16620                        root.path().to_path_buf(),
16621                        "unit".to_string(),
16622                        "session".to_string(),
16623                    ),
16624                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
16625                        module_id: module_id.to_string(),
16626                        launch_nonce: nonce(module_id),
16627                    }),
16628                    consumer_capabilities: None,
16629                    role_versions: None,
16630                    admission_facts: None,
16631                    scope,
16632                })
16633                .unwrap();
16634                let frame = Frame::build(
16635                    FrameType::Request,
16636                    control_flags(),
16637                    0,
16638                    0,
16639                    self.next_connection,
16640                    body,
16641                )
16642                .unwrap();
16643                (ctx, rx, frame)
16644            }
16645
16646            /// Open and expect a refusal before anything is relayed.
16647            async fn refused(
16648                &mut self,
16649                opener: Option<&str>,
16650                target: &str,
16651                scope: Option<ScopeSelector>,
16652            ) -> String {
16653                self.refusal_body(opener, target, scope).await["code"]
16654                    .as_str()
16655                    .unwrap()
16656                    .to_string()
16657            }
16658
16659            async fn refusal_body(
16660                &mut self,
16661                opener: Option<&str>,
16662                target: &str,
16663                scope: Option<ScopeSelector>,
16664            ) -> Value {
16665                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16666                let replies = tokio::time::timeout(
16667                    Duration::from_secs(2),
16668                    self.handler.handle_control_frame(&ctx, frame),
16669                )
16670                .await
16671                .expect("the open must be refused before waiting for a bind ack")
16672                .unwrap();
16673                assert_eq!(replies.len(), 1, "{replies:?}");
16674                assert_eq!(replies[0].header.ty, FrameType::Error);
16675                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16676                assert!(
16677                    module_rx.try_recv().is_err(),
16678                    "a refused open relays nothing"
16679                );
16680                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16681                parse_error(&replies[0])
16682            }
16683
16684            /// Start an open and return its task and the bind the target got.
16685            async fn relayed(
16686                &mut self,
16687                opener: Option<&str>,
16688                target: &str,
16689                scope: Option<ScopeSelector>,
16690            ) -> Relayed {
16691                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16692                let handler = self.handler.clone();
16693                let task_ctx = ctx.clone();
16694                let task = tokio::spawn(async move {
16695                    handler
16696                        .handle_control_frame(&task_ctx, frame)
16697                        .await
16698                        .unwrap()
16699                });
16700                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16701                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16702                    .await
16703                    .expect("the target receives the relayed route.bind")
16704                    .unwrap()
16705                    .frame;
16706                Relayed {
16707                    target: target.to_string(),
16708                    client: ctx,
16709                    client_rx: rx,
16710                    task,
16711                    bind,
16712                }
16713            }
16714
16715            async fn ack(&self, relayed: &Relayed) {
16716                let (module, _) = &self.modules[&relayed.target];
16717                self.handler
16718                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16719                    .await
16720                    .unwrap();
16721            }
16722
16723            /// Open, ack and return the bound route.
16724            async fn bound(
16725                &mut self,
16726                opener: Option<&str>,
16727                target: &str,
16728                scope: Option<ScopeSelector>,
16729            ) -> Bound {
16730                let relayed = self.relayed(opener, target, scope).await;
16731                self.ack(&relayed).await;
16732                let Relayed {
16733                    target,
16734                    client,
16735                    mut client_rx,
16736                    task,
16737                    bind,
16738                } = relayed;
16739                assert!(
16740                    task.await.unwrap().is_empty(),
16741                    "the open is answered by commit"
16742                );
16743                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16744                Bound {
16745                    target,
16746                    client,
16747                    client_rx,
16748                    channel,
16749                    epoch,
16750                    bind,
16751                }
16752            }
16753
16754            fn live(&self, route: &Bound) -> bool {
16755                matches!(
16756                    self.forwarding
16757                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16758                        .unwrap(),
16759                    DataRoute::Client(DataRouteState::Bound(_))
16760                )
16761            }
16762        }
16763
16764        struct Relayed {
16765            target: String,
16766            client: RouteCtx,
16767            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16768            task: tokio::task::JoinHandle<Vec<Frame>>,
16769            bind: Frame,
16770        }
16771
16772        struct Bound {
16773            target: String,
16774            client: RouteCtx,
16775            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16776            channel: u16,
16777            epoch: u32,
16778            bind: Frame,
16779        }
16780
16781        impl Bound {
16782            /// The reason of the `route.closed` this client was sent, after
16783            /// checking it also got a GOODBYE on exactly this route.
16784            fn closed_reason(&mut self) -> RouteCloseReason {
16785                let mut reason = None;
16786                let mut goodbye = false;
16787                while let Ok(outbound) = self.client_rx.try_recv() {
16788                    let frame = outbound.frame;
16789                    match frame.header.ty {
16790                        FrameType::Goodbye => {
16791                            assert_eq!(
16792                                (frame.header.channel, frame.header.epoch),
16793                                (self.channel, self.epoch)
16794                            );
16795                            goodbye = true;
16796                        }
16797                        FrameType::Push => {
16798                            let ClientControlPush::RouteClosed {
16799                                reason: r,
16800                                module_id,
16801                                ..
16802                            } = serde_json::from_slice(&frame.body).unwrap()
16803                            else {
16804                                panic!("unexpected push");
16805                            };
16806                            assert_eq!(module_id, self.target);
16807                            reason = Some(r);
16808                        }
16809                        other => panic!("unexpected frame {other:?}"),
16810                    }
16811                }
16812                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16813                reason.expect("the client is told why the route closed")
16814            }
16815
16816            fn untouched(&mut self) -> bool {
16817                self.client_rx.try_recv().is_err()
16818            }
16819
16820            fn stamp(&self) -> Option<ScopeStamp> {
16821                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16822                    ModuleControlRequest::RouteBind { scope, .. } => scope,
16823                    other => panic!("expected a route.bind, got {other:?}"),
16824                }
16825            }
16826        }
16827
16828        #[tokio::test]
16829        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16830        ) {
16831            let mut rig = rig().await;
16832            let mut record = session(1);
16833            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16834            record.child_owners = vec![Principal::Reserved {
16835                module_id: MAGIC.to_string(),
16836            }];
16837            rig.sync(vec![record]).await;
16838            let scope = || Some(rig_selector("s", Some(1)));
16839
16840            // Admitted: the owner, a bare carrier to any module, a targeted
16841            // carrier to its listed module.
16842            rig.bound(Some(OWNER), PLEXUS, scope()).await;
16843            rig.bound(Some(AFT), OTHER, scope()).await;
16844            rig.bound(Some(BROCA), PLEXUS, scope()).await;
16845
16846            // Refused scope_not_carrier: a targeted carrier to an unlisted
16847            // module, a module that is not listed at all (a child owner is not
16848            // a carrier), and a direct key-holder.
16849            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16850                assert_eq!(
16851                    rig.refused(opener, target, scope()).await,
16852                    error_codes::SCOPE_NOT_CARRIER,
16853                    "{opener:?} -> {target}"
16854                );
16855            }
16856        }
16857
16858        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16859            ScopeSelector::new(
16860                Principal::Reserved {
16861                    module_id: OWNER.to_string(),
16862                },
16863                scope_ref.to_string(),
16864                scope_epoch,
16865            )
16866        }
16867
16868        #[tokio::test]
16869        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16870            let mut rig = rig().await;
16871            rig.sync(vec![session(1)]).await;
16872            for opener in [OWNER, AFT] {
16873                assert_eq!(
16874                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16875                        .await,
16876                    error_codes::SCOPE_EPOCH_REQUIRED,
16877                    "{opener}"
16878                );
16879            }
16880        }
16881
16882        #[tokio::test]
16883        async fn admission_separates_not_synced_not_live_and_ended() {
16884            let mut rig = rig().await;
16885            // Before the configured owner's first sync: retryable.
16886            let code = rig
16887                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16888                .await;
16889            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16890            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16891
16892            // An owner that is not configured will never sync: terminal.
16893            let ghost = ScopeSelector::new(
16894                Principal::Reserved {
16895                    module_id: "ghost".to_string(),
16896                },
16897                "s".to_string(),
16898                Some(1),
16899            );
16900            assert_eq!(
16901                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16902                error_codes::SCOPE_NOT_LIVE
16903            );
16904
16905            rig.sync(vec![session(2)]).await;
16906            assert_eq!(
16907                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16908                    .await,
16909                error_codes::SCOPE_NOT_LIVE
16910            );
16911            for epoch in [1, 3] {
16912                assert_eq!(
16913                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16914                        .await,
16915                    error_codes::SCOPE_ENDED,
16916                    "epoch {epoch}"
16917                );
16918            }
16919            // Control: the live epoch is admitted.
16920            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16921                .await;
16922        }
16923
16924        #[tokio::test]
16925        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16926            let mut rig = rig().await;
16927            rig.sync(vec![session(1)]).await;
16928            let route = rig
16929                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16930                .await;
16931            let stamp = route.stamp().expect("a scoped bind carries the stamp");
16932            assert_eq!(stamp.scope_ref, "s");
16933            assert_eq!(stamp.scope_epoch, 1);
16934            assert_eq!(stamp.kind, ScopeKind::Head);
16935            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16936            assert!(stamp.attributes.delegates);
16937            assert!(stamp.owner_authorized);
16938            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16939            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16940
16941            // broca owns a scope of its own on its own module connection; it is
16942            // not in scope_authority_owners, so its stamp is not authorized.
16943            let (broca, mut broca_rx) = wide_ctx(50);
16944            hello_via_sink(
16945                &rig.handler,
16946                &broca,
16947                &mut broca_rx,
16948                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16949            )
16950            .await;
16951            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16952                .await
16953                .unwrap();
16954            let own = ScopeSelector::new(
16955                Principal::Reserved {
16956                    module_id: BROCA.to_string(),
16957                },
16958                "b".to_string(),
16959                Some(1),
16960            );
16961            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16962            assert!(!route.stamp().unwrap().owner_authorized);
16963        }
16964
16965        #[tokio::test]
16966        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16967            let mut rig = rig().await;
16968            let mut record = head("s", 1);
16969            record.carriers = vec![carrier(AFT, None)];
16970            let flow_id = "Flow:run-7/step_2!~";
16971            record.attributes.flow_id = Some(flow_id.to_string());
16972            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16973                .await
16974                .unwrap();
16975            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16976                panic!("not a sync reply");
16977            };
16978            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16979            let route = rig
16980                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16981                .await;
16982            assert!(rig.live(&route), "the stamped bind committed");
16983            let stamp = route.stamp().expect("a flow scope carries a stamp");
16984            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16985            assert_eq!(stamp.attributes.agent_id, None);
16986            assert!(!stamp.attributes.delegates);
16987            assert!(stamp.owner_authorized);
16988            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16989            assert_eq!(unscoped.stamp(), None);
16990        }
16991
16992        #[tokio::test]
16993        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16994            let mut rig = rig_with_flow_support(false).await;
16995            let mut record = session(1);
16996            record.attributes.flow_id = Some("flow:7".to_string());
16997            rig.sync(vec![record]).await;
16998            for opener in [OWNER, AFT] {
16999                let body = rig
17000                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17001                    .await;
17002                assert_eq!(body["code"], "target_flow_unsupported");
17003                let message = body["message"].as_str().unwrap();
17004                for required in [PLEXUS, "flow-scopes/v1"] {
17005                    assert!(message.contains(required), "{message}");
17006                }
17007            }
17008        }
17009
17010        #[tokio::test]
17011        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17012            let mut rig = rig_with_flow_support(true).await;
17013            let mut record = session(1);
17014            record.attributes.flow_id = Some("flow:7".to_string());
17015            rig.sync(vec![record]).await;
17016            for opener in [OWNER, AFT] {
17017                let route = rig
17018                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17019                    .await;
17020                assert!(rig.live(&route));
17021                assert_eq!(
17022                    route.stamp().unwrap().attributes.flow_id.as_deref(),
17023                    Some("flow:7")
17024                );
17025            }
17026        }
17027
17028        #[tokio::test]
17029        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17030            use std::future::Future;
17031
17032            let mut rig = rig_with_flow_support(true).await;
17033            let mut record = session(1);
17034            record.attributes.flow_id = Some("flow:7".to_string());
17035            rig.sync(vec![record]).await;
17036            let (client, mut client_rx, frame) =
17037                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17038            // Hold the route-open response permit so admission sees the first
17039            // target but relay reservation cannot capture an endpoint yet.
17040            for _ in 0..64 {
17041                client.egress.try_send(route_bind_ack(1)).unwrap();
17042            }
17043            let handler = rig.handler.clone();
17044            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17045            std::future::poll_fn(|cx| {
17046                assert!(open.as_mut().poll(cx).is_pending());
17047                std::task::Poll::Ready(())
17048            })
17049            .await;
17050            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17051
17052            let old_connection = rig.modules[PLEXUS].0.connection_id;
17053            rig.handler.cleanup_connection(old_connection).unwrap();
17054            let (replacement, mut replacement_rx) = wide_ctx(200);
17055            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17056            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17057            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17058            let hello = Frame::build(
17059                FrameType::Hello,
17060                control_flags(),
17061                0,
17062                0,
17063                200,
17064                serde_json::to_vec(&body).unwrap(),
17065            )
17066            .unwrap();
17067            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17068
17069            client_rx.try_recv().unwrap();
17070            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17071                .await
17072                .expect("the replacement is refused without waiting for a bind ack")
17073                .unwrap();
17074            assert_eq!(replies.len(), 1);
17075            let body = parse_error(&replies[0]);
17076            assert_eq!(body["code"], "target_flow_unsupported");
17077            for required in [PLEXUS, "flow-scopes/v1"] {
17078                assert!(body["message"].as_str().unwrap().contains(required));
17079            }
17080            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17081            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17082            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17083            assert!(rig
17084                .handler
17085                .registry
17086                .get_module_by_connection(replacement.connection_id)
17087                .unwrap()
17088                .is_some());
17089        }
17090
17091        #[tokio::test]
17092        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17093            use std::future::Future;
17094            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17095            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17096            const NEXT_HASH: &str =
17097                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17098            let mut rig = rig().await;
17099            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17100            record.carriers.push(ScopeCarrier::new(Principal::Direct));
17101            rig.sync(vec![record.clone()]).await;
17102            let (client, mut rx, frame) = rig.open_frame(
17103                None,
17104                PLEXUS,
17105                Some(
17106                    rig.selector("s", Some(1))
17107                        .with_bind_proof(Some(PROOF.into())),
17108                ),
17109            );
17110            // Hold egress reservation while the scope owner changes its bind-proof
17111            // hash, so a proof valid at admission must be checked again at relay.
17112            for _ in 0..64 {
17113                client.egress.try_send(route_bind_ack(1)).unwrap();
17114            }
17115            let handler = rig.handler.clone();
17116            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17117            std::future::poll_fn(|cx| {
17118                assert!(open.as_mut().poll(cx).is_pending());
17119                std::task::Poll::Ready(())
17120            })
17121            .await;
17122            record.bind_proof_sha256 = Some(NEXT_HASH.into());
17123            rig.sync(vec![record]).await;
17124            rx.try_recv().unwrap();
17125            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17126                .await
17127                .unwrap()
17128                .unwrap();
17129            assert_eq!(
17130                parse_error(&replies[0])["code"],
17131                error_codes::SCOPE_BIND_PROOF_MISMATCH
17132            );
17133            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17134            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17135            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17136        }
17137
17138        #[tokio::test]
17139        async fn scope_bind_proof_reserved_openers_ignore_proof() {
17140            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17141            let mut rig = rig().await;
17142            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17143                .await;
17144            for opener in [OWNER, AFT] {
17145                for proof in [None, Some("malformed+proof=")] {
17146                    let route = rig
17147                        .bound(
17148                            Some(opener),
17149                            PLEXUS,
17150                            Some(
17151                                rig.selector("s", Some(1))
17152                                    .with_bind_proof(proof.map(str::to_string)),
17153                            ),
17154                        )
17155                        .await;
17156                    assert!(rig.live(&route));
17157                }
17158            }
17159        }
17160
17161        #[tokio::test]
17162        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17163            use std::future::Future;
17164            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17165            const NEXT_HASH: &str =
17166                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17167            let mut rig = rig().await;
17168            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17169                .await;
17170            let (client, mut rx, frame) =
17171                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17172            for _ in 0..64 {
17173                client.egress.try_send(route_bind_ack(1)).unwrap();
17174            }
17175            let handler = rig.handler.clone();
17176            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17177            std::future::poll_fn(|cx| {
17178                assert!(open.as_mut().poll(cx).is_pending());
17179                std::task::Poll::Ready(())
17180            })
17181            .await;
17182            rig.sync(vec![
17183                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17184            ])
17185            .await;
17186            rx.try_recv().unwrap();
17187            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17188            let bind = std::future::poll_fn(|cx| {
17189                assert!(open.as_mut().poll(cx).is_pending());
17190                module_rx.poll_recv(cx)
17191            })
17192            .await
17193            .unwrap()
17194            .frame;
17195            handler
17196                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17197                .await
17198                .unwrap();
17199            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17200                .await
17201                .unwrap()
17202                .unwrap();
17203            assert_eq!(replies.len(), 1);
17204            // Proof rechecking allowed the relay. Independent tag settlement
17205            // still asks the opener to retry after any same-epoch content change.
17206            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17207            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17208            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17209            let route = rig
17210                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17211                .await;
17212            assert!(rig.live(&route));
17213        }
17214
17215        #[tokio::test]
17216        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17217        {
17218            let mut rig = rig_with_flow_support(false).await;
17219            rig.sync(vec![session(1)]).await;
17220            let route = rig
17221                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17222                .await;
17223            assert!(rig.live(&route));
17224            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17225            let mut record = session(1);
17226            record.attributes.flow_id = Some("flow:7".to_string());
17227            rig.sync(vec![record]).await;
17228            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17229            assert!(rig.live(&unscoped));
17230            assert_eq!(unscoped.stamp(), None);
17231        }
17232
17233        #[tokio::test]
17234        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17235            let mut rig = rig().await;
17236            let mut record = session(1);
17237            record.attributes.flow_id = Some("flow:7".to_string());
17238            rig.sync(vec![record.clone()]).await;
17239            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17240            let mut owner_route = rig
17241                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17242                .await;
17243            let mut carrier_route = rig
17244                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17245                .await;
17246            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17247            rig.sync(vec![record.clone()]).await;
17248            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17249            assert!(rig.live(&owner_route) && owner_route.untouched());
17250            assert!(rig.live(&carrier_route) && carrier_route.untouched());
17251
17252            record.attributes.flow_id = Some("flow:8".to_string());
17253            rig.sync(vec![record]).await;
17254            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17255            let before = before.unwrap();
17256            assert_eq!(after.scope_epoch, before.scope_epoch);
17257            assert!(after.version > before.version);
17258            for route in [&mut owner_route, &mut carrier_route] {
17259                assert!(!rig.live(route));
17260                assert_eq!(
17261                    route.closed_reason(),
17262                    RouteCloseReason::ScopeDelegationChanged
17263                );
17264            }
17265            assert!(rig.live(&unscoped) && unscoped.untouched());
17266            // Each provider also receives a GOODBYE for its drained route;
17267            // consume it before expecting the next route.bind on that sink.
17268            for target in [PLEXUS, OTHER] {
17269                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17270                let goodbye = module_rx
17271                    .try_recv()
17272                    .expect("the provider sees the drain")
17273                    .frame;
17274                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17275                assert!(module_rx.try_recv().is_err());
17276            }
17277            let rebound = rig
17278                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17279                .await;
17280            assert_eq!(
17281                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17282                Some("flow:8")
17283            );
17284        }
17285
17286        /// The owner's sync lands between admission and the module's ack. The
17287        /// open is refused by name, the module's other routes stay up, and the
17288        /// reserved pair is released. Changed content is retryable; an ended
17289        /// scope is not.
17290        #[tokio::test]
17291        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17292            let mut rig = rig().await;
17293            rig.sync(vec![session(1)]).await;
17294            let mut cotenant = rig
17295                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17296                .await;
17297
17298            let mut changed = session(1);
17299            changed.child_owners.push(Principal::Reserved {
17300                module_id: MAGIC.to_string(),
17301            });
17302            let mut ended = None;
17303            for (code, next) in [
17304                (error_codes::SCOPE_CHANGED, vec![changed]),
17305                (error_codes::SCOPE_ENDED, Vec::new()),
17306            ] {
17307                let relayed = rig
17308                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17309                    .await;
17310                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17311                ended = Some(next.is_empty());
17312                rig.sync(next).await;
17313                rig.ack(&relayed).await;
17314                let replies = relayed.task.await.unwrap();
17315                assert_eq!(replies.len(), 1, "{replies:?}");
17316                assert_eq!(parse_error(&replies[0])["code"], code);
17317                assert_eq!(
17318                    subc_protocol::error_codes::is_retryable_route_open(code),
17319                    code == error_codes::SCOPE_CHANGED
17320                );
17321                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17322                // The module is told to drop just the binding it created.
17323                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17324                // Collected, because ending the scope also closes the co-tenant
17325                // route, whose GOODBYE comes first.
17326                let mut goodbyes = Vec::new();
17327                while let Ok(outbound) = plexus_rx.try_recv() {
17328                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17329                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17330                }
17331                assert!(
17332                    goodbyes.contains(&(bind_channel, bind_epoch)),
17333                    "{goodbyes:?}"
17334                );
17335                assert!(rig
17336                    .handler
17337                    .registry
17338                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17339                    .unwrap()
17340                    .is_some());
17341            }
17342            assert_eq!(ended, Some(true));
17343            // The co-tenant stayed up through the change, and closed only when
17344            // the scope ended, by the drain rule rather than by the commit.
17345            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17346        }
17347
17348        /// Each row of the drain table on one set of routes: the owner's, a
17349        /// bare carrier's, and a targeted carrier's to each of its targets.
17350        #[tokio::test]
17351        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17352            struct Case {
17353                name: &'static str,
17354                change: fn(&mut ScopeRecord),
17355                /// Closed routes by index: owner->plexus, aft->plexus,
17356                /// broca->plexus, broca->other.
17357                closed: [Option<RouteCloseReason>; 4],
17358            }
17359            use RouteCloseReason::*;
17360            let cases = [
17361                Case {
17362                    name: "a carrier entry removed",
17363                    change: |r| {
17364                        r.carriers.retain(|c| {
17365                            c.principal
17366                                != Principal::Reserved {
17367                                    module_id: AFT.to_string(),
17368                                }
17369                        })
17370                    },
17371                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17372                },
17373                Case {
17374                    name: "a target removed from a carrier",
17375                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17376                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
17377                },
17378                Case {
17379                    name: "a bare carrier narrowed to targets",
17380                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17381                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17382                },
17383                Case {
17384                    name: "delegates turned off",
17385                    change: |r| r.attributes.delegates = false,
17386                    closed: [Some(ScopeDelegationChanged); 4],
17387                },
17388                Case {
17389                    name: "agent_id changed",
17390                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17391                    closed: [Some(ScopeDelegationChanged); 4],
17392                },
17393                Case {
17394                    name: "a carrier added, child owners changed, the record re-sent",
17395                    change: |r| {
17396                        r.carriers.push(carrier(MAGIC, None));
17397                        r.child_owners.push(Principal::Reserved {
17398                            module_id: MAGIC.to_string(),
17399                        });
17400                    },
17401                    closed: [None; 4],
17402                },
17403                Case {
17404                    name: "a target added",
17405                    change: |r| {
17406                        r.carriers[1]
17407                            .targets
17408                            .as_mut()
17409                            .unwrap()
17410                            .push("third".to_string())
17411                    },
17412                    closed: [None; 4],
17413                },
17414                Case {
17415                    name: "delegates turned on",
17416                    change: |r| r.attributes.delegates = true,
17417                    closed: [None; 4],
17418                },
17419            ];
17420            for case in cases {
17421                let mut rig = rig().await;
17422                rig.sync(vec![session(1)]).await;
17423                let scope = || Some(rig_selector("s", Some(1)));
17424                let mut routes = [
17425                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
17426                    rig.bound(Some(AFT), PLEXUS, scope()).await,
17427                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
17428                    rig.bound(Some(BROCA), OTHER, scope()).await,
17429                ];
17430                let mut record = session(1);
17431                (case.change)(&mut record);
17432                rig.sync(vec![record]).await;
17433                for (index, expected) in case.closed.iter().enumerate() {
17434                    let route = &mut routes[index];
17435                    match expected {
17436                        Some(reason) => {
17437                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
17438                            assert_eq!(
17439                                route.closed_reason(),
17440                                *reason,
17441                                "{}: route {index}",
17442                                case.name
17443                            );
17444                        }
17445                        None => {
17446                            assert!(rig.live(route), "{}: route {index} closed", case.name);
17447                            assert!(
17448                                route.untouched(),
17449                                "{}: route {index} was told something",
17450                                case.name
17451                            );
17452                        }
17453                    }
17454                }
17455            }
17456        }
17457
17458        #[tokio::test]
17459        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17460            // Removed, and replaced by a higher epoch.
17461            for next in [Vec::new(), vec![session(2)]] {
17462                let mut rig = rig().await;
17463                rig.sync(vec![session(1)]).await;
17464                let mut route = rig
17465                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17466                    .await;
17467                rig.sync(next).await;
17468                assert!(!rig.live(&route));
17469                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17470            }
17471
17472            // A child whose parent ends: its routes close as parent-ended, the
17473            // child stays live, and routes under the parent close as ended.
17474            let mut rig = rig().await;
17475            let mut child = session(1);
17476            child.scope_ref = "child".to_string();
17477            child.kind = ScopeKind::Worker;
17478            child.parent = Some(ScopeParent::new(
17479                Principal::Reserved {
17480                    module_id: OWNER.to_string(),
17481                },
17482                "s".to_string(),
17483                1,
17484            ));
17485            rig.sync(vec![session(1), child.clone()]).await;
17486            let mut child_route = rig
17487                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17488                .await;
17489            assert_eq!(
17490                child_route.stamp().unwrap().parent_state,
17491                Some(ParentState::Linked)
17492            );
17493            rig.sync(vec![child]).await;
17494            assert!(!rig.live(&child_route));
17495            assert_eq!(
17496                child_route.closed_reason(),
17497                RouteCloseReason::ScopeParentEnded
17498            );
17499        }
17500
17501        #[tokio::test]
17502        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17503        ) {
17504            let mut rig = rig().await;
17505            rig.sync(vec![session(1)]).await;
17506            let mut route = rig
17507                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17508                .await;
17509            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17510            rig.sync(vec![session(1)]).await;
17511            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17512            assert!(rig.live(&route) && route.untouched());
17513
17514            // A call in flight on the route when another carrier is added. A
17515            // forwarded REQUEST holds one credit on the route's flow until the
17516            // module answers; the router takes it exactly like this.
17517            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17518                .forwarding
17519                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17520                .unwrap()
17521            else {
17522                panic!("the route is bound");
17523            };
17524            binding.flow.acquire_tagged(9, false).await.unwrap();
17525            let mut widened = session(1);
17526            widened.carriers.push(carrier(MAGIC, None));
17527            rig.sync(vec![widened]).await;
17528            assert!(rig.live(&route) && route.untouched());
17529            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17530            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17531            // The call's credit is still held on an open flow, so its answer
17532            // will be delivered: closing the route would have closed the flow.
17533            assert_eq!(binding.flow.in_flight(), 1);
17534            binding
17535                .flow
17536                .acquire_tagged(10, false)
17537                .await
17538                .expect("the flow is still open");
17539        }
17540
17541        /// A swap's superseded endpoint keeps its routes until drained; ending
17542        /// the scope closes them there too.
17543        #[tokio::test]
17544        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17545            let mut rig = rig().await;
17546            rig.sync(vec![session(1)]).await;
17547            let mut on_incumbent = rig
17548                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17549                .await;
17550
17551            // Swap plexus: register a candidate and cut over, leaving the
17552            // incumbent superseded with the route still on it.
17553            let (candidate, _candidate_rx) = wide_ctx(9);
17554            let registration = rig
17555                .handler
17556                .registry
17557                .register_candidate_with_control_ops(
17558                    manifest(PLEXUS, PROTOCOL_VERSION),
17559                    PROTOCOL_VERSION,
17560                    candidate.connection_id,
17561                    module_baseline_control_ops(),
17562                )
17563                .unwrap();
17564            rig.forwarding
17565                .register_candidate_module_connection(
17566                    candidate.connection_id,
17567                    PLEXUS.to_string(),
17568                    PROTOCOL_VERSION,
17569                    manifest_concurrency(&registration.manifest),
17570                    candidate.egress.clone(),
17571                )
17572                .unwrap();
17573            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17574            rig.handler
17575                .registry
17576                .promote_candidate(PLEXUS)
17577                .unwrap()
17578                .unwrap();
17579            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17580
17581            rig.sync(Vec::new()).await;
17582            assert!(!rig.live(&on_incumbent));
17583            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17584            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17585            let goodbye = incumbent_rx
17586                .try_recv()
17587                .expect("the superseded endpoint is told")
17588                .frame;
17589            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17590        }
17591    }
17592}
17593
17594#[cfg(test)]
17595mod concurrency_default_exposure_tests {
17596    use super::*;
17597
17598    fn hello_body(role_json: &str) -> Vec<u8> {
17599        format!(
17600            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17601        )
17602        .into_bytes()
17603    }
17604
17605    fn manifest_from(body: &[u8]) -> ModuleManifest {
17606        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17607        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17608            .expect("manifest parses")
17609    }
17610
17611    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17612
17613    #[test]
17614    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17615        let body = hello_body(&format!(
17616            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17617        ));
17618        let manifest = manifest_from(&body);
17619        // Precondition: serde really resolved it to the default, so the typed
17620        // manifest alone cannot answer the question this probe exists for.
17621        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17622        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17623    }
17624
17625    #[test]
17626    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17627        let body = hello_body(&format!(
17628            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17629        ));
17630        let manifest = manifest_from(&body);
17631        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17632        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17633    }
17634
17635    #[test]
17636    fn non_management_roles_are_never_reported() {
17637        let body = hello_body(
17638            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17639        );
17640        let manifest = manifest_from(&body);
17641        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17642    }
17643}