Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
26        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
27        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
28    },
29    scope::{
30        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32        SCOPE_SYNC_OP,
33    },
34    session::{
35        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38        ROLE_VERSIONS_FIELD,
39    },
40    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41    Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47    caller_identity::CallerIdentity,
48    capability_requirements::{
49        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51        RuntimeModule,
52    },
53    child_roster::DaemonShutdownFlag,
54    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55    forwarding::{
56        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59    },
60    observability::{
61        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62    },
63    privileged_prompt::{prompt_text, PrivilegedOperation},
64    provenance::{
65        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66    },
67    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68    router::{RouteCtx, RouterError},
69    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71    stderr_tail::{CaptureState, TailEntry},
72    supervise::{
73        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74        SupervisorHandle, SwapHelloAdmission,
75    },
76    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79/// Lowest envelope version this subc build will negotiate.
80///
81/// Module HELLO negotiation is exact: peers must use the daemon's locked
82/// protocol version. Older and newer peers receive `version_unsupported` and
83/// are not registered.
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93    ops::ACTIVITY_RECORD,
94    ops::ACTIVITY_QUERY,
95    ops::ACTIVITY_SUBSCRIBE,
96    ops::SERVER_DESCRIBE,
97    ops::CATALOG_LIST,
98    ops::ROUTE_OPEN,
99    ops::ROUTE_POLL,
100    ops::ROUTE_CLOSING,
101    ops::ROUTE_CLOSED,
102    ops::SUPERVISOR_LIST,
103    ops::SUPERVISOR_RESTART,
104    ops::SUPERVISOR_SWAP,
105    ops::SUPERVISOR_RELOAD,
106    ops::SUPERVISOR_RESCAN,
107    ops::SUPERVISOR_RELEASE_RESERVED,
108    ops::SUPERVISOR_SET_ENABLED,
109    ops::SUPERVISOR_HEALTH_PROBE,
110    ops::SUPERVISOR_HEALTH,
111    ops::SUPERVISOR_STDERR_TAIL,
112    ops::SUPERVISOR_TERMINALS,
113    ops::SUPERVISOR_ROUTES,
114    ops::SUPERVISOR_PROVENANCE,
115    ops::SUPERVISOR_SPAWN_SNAPSHOT,
116    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
120/// announced by a capability in `server.describe` instead, and clients check
121/// that capability before sending it. `ck daemon --json` prints `subc_ops`
122/// verbatim and scripts read that output, so a new name there would change it.
123/// A malformed body for one of these is still refused `invalid_control_body`
124/// rather than as an unknown op.
125const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132    "supervisor.live_roots",
133    SCOPE_SYNC_OP,
134    SCOPE_APPLY_OP,
135    SCOPE_DESCRIBE_OP,
136    "operator.confirm",
137];
138
139/// Module-originated ops the daemon answers but does not advertise in
140/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
141/// belongs to is incomplete, so no module is told it works before it does.
142const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146/// How long subc waits for a module to ack a relayed route.bind before returning
147/// `module_timeout`. The ack waits on the module's own configure, which for AFT
148/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
149/// and DB-open work — on a cold page cache or a large repo that legitimately
150/// exceeds a couple of seconds. The default is generous because rejecting a VALID
151/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
152/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
153pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155/// How many CONSECUTIVE full-budget relay timeouts against one target module
156/// open that module's bind-relay breaker.
157///
158/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
159/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
160/// so three consecutive full-budget timeouts take ~36s to observe: every client
161/// whose open contributed to opening the breaker had already given up on its
162/// own. That is what makes opening the breaker unable to turn a call that would
163/// have succeeded into a refusal — it can only make an already-failing module
164/// fail faster.
165///
166/// Two would be reachable inside one default deadline. One would convict a
167/// module on a single cold-cache bind, which is exactly the valid-but-slow case
168/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
169pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171/// How long a module's bind-relay breaker stays open before exactly one
172/// `route.open` is let through as a probe.
173///
174/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
175/// re-pays a full-budget stall almost continuously, and the breaker stops being
176/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
177/// deadline: a client that starts retrying after the module recovers has to get
178/// a probe opportunity inside its own deadline, or the breaker converts a
179/// recovered module into a failed call — the failure it exists to prevent,
180/// pointed the other way.
181///
182/// 20s sits between those with room on both sides, and it caps what a wedged
183/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
184/// rather than one per `route.open` per connection. The stall that motivated
185/// this, with its measurements, is written up in
186/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
187/// waited the whole budget out.
188pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194    configured: &Path,
195    spawned_from: Option<&Path>,
196    image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198    let path = match spawned_from {
199        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200            ReloadPathAgreement::Match
201        }
202        Some(spawned_from) => ReloadPathAgreement::Mismatch {
203            configured: configured.to_path_buf(),
204            spawned_from: spawned_from.to_path_buf(),
205        },
206        None => ReloadPathAgreement::Unavailable {
207            reason: if matches!(
208                image,
209                subc_control::RunningImageAgreement::Unavailable {
210                    reason: subc_control::RunningImageUnavailableReason::NotRunning
211                }
212            ) {
213                ReloadPathUnavailableReason::NotRunning
214            } else {
215                ReloadPathUnavailableReason::SpawnedPathUnavailable
216            },
217        },
218    };
219    PendingReloadVerdict {
220        path,
221        image,
222        spec: None,
223    }
224}
225
226fn launch_spec_agreement(
227    configured: &crate::supervise::ModuleSpec,
228    running: bool,
229    spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232    if !running {
233        return Agreement::Unavailable {
234            reason: Reason::NotRunning,
235        };
236    }
237    match spawned {
238        None => Agreement::Unavailable {
239            reason: Reason::SpawnedSpecUnavailable,
240        },
241        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242            Agreement::Match
243        }
244        Some(_) => Agreement::Mismatch,
245    }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249    if configured == spawned_from {
250        return true;
251    }
252    #[cfg(windows)]
253    {
254        // Windows spawn records the resolved verbatim path (\\?\...), while
255        // configuration keeps its original spelling. Resolve both with the OS
256        // instead of stripping prefixes or folding case, which could conflate
257        // distinct names in case-sensitive directories. Different resolved
258        // paths still differ even if their files have identical content.
259        match (
260            std::fs::canonicalize(configured),
261            std::fs::canonicalize(spawned_from),
262        ) {
263            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264            _ => false,
265        }
266    }
267    #[cfg(not(windows))]
268    {
269        false
270    }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275    build: DaemonBuildProvenance,
276    pid: Option<u32>,
277    started_at_ms: Option<u64>,
278    start_clock: Option<crate::clock::StartClock>,
279    executable_path: Option<PathBuf>,
280    executable_identity: Option<SpawnedFileIdentity>,
281    process_start_time: Option<u64>,
282    probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286    fn default() -> Self {
287        Self {
288            build: DaemonBuildProvenance {
289                build_git_sha: None,
290                build_lock_digest: None,
291            },
292            pid: None,
293            started_at_ms: None,
294            start_clock: None,
295            executable_path: None,
296            executable_identity: None,
297            process_start_time: None,
298            probe: ExecutableIdentityProbe::default(),
299        }
300    }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305    supervisor: Supervisor,
306    config_path: PathBuf,
307    configured_port: Option<u16>,
308    storage_config: Option<crate::daemon_config::StorageConfig>,
309    admission_facts_carrier_module_id: Option<String>,
310    admission_facts_targets: Option<Vec<String>>,
311    scope_authority_owners: Vec<String>,
312    privileged_operator_prompt: PrivilegedOperatorPrompt,
313    /// Counts executed rescans that set out to change the supervised modules'
314    /// staged or applied configuration, counted before the first change so a
315    /// rescan that fails partway still counts. With the daemon incarnation it
316    /// names the state a `supervisor.rescan_checked` preview was computed
317    /// against.
318    generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321/// Refusal labels passed to `observe_route_open_refusal` that mean the target
322/// module is not serving right now, and so open or extend an outage in the
323/// route outage tracker. Every one of them is only reachable after the target
324/// was found in the registry, which is what keeps an arbitrary client-chosen
325/// id from ever creating tracker state.
326///
327/// Deliberately absent: `not_registered` and `removed` (the id may be
328/// anything a client sent, and a removed module is gone on purpose),
329/// `protocol_none` (such a module never serves routes, so nothing is out),
330/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
331/// capability and admission-facts refusals (they refuse the caller, not a
332/// module outage), and `relay_reservation_failed` (its code ranges over
333/// capacity limits as well as a vanished connection). Capacity, breaker,
334/// relay-timeout and module-rejection refusals do not pass through that
335/// function at all; the breaker logs its own transitions.
336///
337/// The two not-serving refusals that bypass that function record themselves
338/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
339/// `required_capability_unprovided` is not tracked: the module itself is up,
340/// and the outage belongs to the missing provider.
341const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342    "reloading",
343    "supervisor_not_live",
344    "registration_not_active",
345    "no_forwarding_connection",
346    "relay_send_failed",
347];
348
349/// Real channel-0 control handler for subc itself.
350#[derive(Clone)]
351pub struct ControlHandler {
352    registry: Arc<Registry>,
353    activity: Option<crate::activity::ActivityLedger>,
354    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355    forwarding: Arc<ForwardingTable>,
356    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357    supervisor: SupervisorHandle,
358    subc_capabilities: Arc<[String]>,
359    /// Daemon-wide route.bind relay budget. Used as the fallback when the
360    /// target module has no per-module override in
361    /// `route_bind_relay_timeouts`.
362    route_bind_relay_timeout: Duration,
363    /// Per-module route.bind relay budget overrides, keyed by module id. When
364    /// `handle_route_open` resolves the deadline for a target module, a
365    /// per-module entry wins over the daemon-wide value above.
366    route_bind_relay_timeouts: BTreeMap<String, Duration>,
367    /// Per-target-module bind-relay breaker state. Shared with the forwarding
368    /// table, which is where a new module connection resets it.
369    route_bind_breakers: RouteBindBreakers,
370    /// Live relay admissions keyed by target module. Shared through the
371    /// forwarding table so cloned or separately built handlers enforce one cap.
372    route_bind_concurrency: RouteBindConcurrency,
373    /// Start and end of each module's not-serving period as seen by
374    /// `route.open`, so an outage gets one line at each edge instead of only
375    /// the per-refusal INFO lines. Taken from the forwarding table, so every
376    /// handler built over one table shares it.
377    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378    /// Consecutive relay timeouts that open a module's breaker.
379    route_bind_breaker_threshold: u32,
380    /// How long a breaker stays open before one probe is admitted.
381    route_bind_breaker_cooldown: Duration,
382    health_probe_timeout: Duration,
383    /// Central storage policy. When set, each registering module receives its
384    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
385    storage_config: Option<crate::daemon_config::StorageConfig>,
386    /// The machine id established at boot, served on every HELLO_ACK and on
387    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
388    /// changes the file, never this value. `None` serves no id.
389    machine_id: Option<crate::machine_id::MachineId>,
390    admission_facts_carrier_module_id: Option<String>,
391    admission_facts_targets: Option<Vec<String>>,
392    /// Scope records with their sync authorities and tombstones; see
393    /// `crate::scopes`. Shared by clones of this handler, so every connection
394    /// reads and writes one table.
395    scopes: Arc<RwLock<ScopeTable>>,
396    /// Wakes the expiry loop when an accepted scope change may have added a
397    /// deadline, so the loop can wait without ticking while no live scope has
398    /// one. A permit is stored when nothing waits, so no wake is lost.
399    scope_deadline_added: Arc<tokio::sync::Notify>,
400    /// The configured `scope_authority_owners`, kept so a rescan can report a
401    /// changed value as needing a daemon restart; rescan never applies it.
402    scope_authority_owners: Vec<String>,
403    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
404    privileged_operator_prompt: PrivilegedOperatorPrompt,
405    /// The local and peer address of each accepted TCP client connection, read
406    /// from the daemon's own socket, so a privileged request can name its caller.
407    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408    #[cfg(test)]
409    caller_lookup: Option<TestCallerLookup>,
410    /// The launch nonce each module connection presented at HELLO, which is how
411    /// a `scope.sync` is matched to the owner's current launch.
412    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413    rescan: Option<SupervisorRescanContext>,
414    connected_clients: ConnectedClients,
415    counters: DaemonCounters,
416    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
418    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
419    /// their usual severity (see [`log_requirement_events`]). A handler built
420    /// without one has a flag that is never set.
421    daemon_shutdown: DaemonShutdownFlag,
422    daemon_provenance: DaemonProvenanceFacts,
423    #[cfg(test)]
424    control_dispatch_delay: Option<Duration>,
425    #[cfg(test)]
426    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427    /// Holds `route.open` after the route.bind is queued to the module and
428    /// before it is marked enqueued, so a test can make the module's reply (or
429    /// a client teardown) land exactly inside that window.
430    #[cfg(test)]
431    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432    /// Holds a HELLO after its registry entry is reserved and before its
433    /// forwarding endpoint is installed, so a test can read the catalog and
434    /// open routes while the module is registered but not yet routable. A
435    /// blocking barrier because the HELLO handler is synchronous; the test
436    /// runs the HELLO on its own thread.
437    #[cfg(test)]
438    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443        f.debug_struct("ControlHandler")
444            .field("registry", &self.registry)
445            .field("forwarding", &self.forwarding)
446            .field("process_liveness", &self.process_liveness.is_some())
447            .field("supervisor", &self.supervisor)
448            .field("subc_capabilities", &self.subc_capabilities)
449            .finish()
450    }
451}
452
453struct RouteOpenRequest {
454    target: RouteTarget,
455    identity: BindIdentity,
456    consumer_identity: Option<ConsumerIdentity>,
457    consumer_capabilities: Option<Vec<String>>,
458    role_versions: Option<BTreeMap<String, String>>,
459    admission_facts: Option<serde_json::Value>,
460    scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465    Admission,
466    ProjectRoot,
467    EgressReservation,
468    BindSend,
469    BindAckWait,
470}
471
472/// Store elapsed time for each route-open phase, including attempts that end
473/// in an early refusal or backend error. Admission also includes rechecks and
474/// commit bookkeeping performed between the other timed phases.
475struct RouteOpenTimings {
476    module_id: String,
477    principal: String,
478    admission: Duration,
479    project_root: Duration,
480    egress_reservation: Duration,
481    bind_send: Duration,
482    bind_ack_wait: Duration,
483    phase: Option<RouteOpenPhase>,
484    phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488    fn new() -> Self {
489        Self {
490            module_id: String::new(),
491            // Refusals before attestation must not claim a verified principal.
492            principal: "unresolved".to_string(),
493            admission: Duration::ZERO,
494            project_root: Duration::ZERO,
495            egress_reservation: Duration::ZERO,
496            bind_send: Duration::ZERO,
497            bind_ack_wait: Duration::ZERO,
498            phase: Some(RouteOpenPhase::Admission),
499            phase_started_at: Instant::now(),
500        }
501    }
502
503    fn finish(&mut self) {
504        let elapsed = self.phase_started_at.elapsed();
505        match self.phase.take() {
506            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511            None => {}
512        }
513    }
514
515    fn enter(&mut self, phase: RouteOpenPhase) {
516        self.finish();
517        self.phase = Some(phase);
518        self.phase_started_at = Instant::now();
519    }
520}
521
522struct RouteBindReservationGuard {
523    forwarding: Arc<ForwardingTable>,
524    endpoint: ModuleEndpointId,
525    relay_corr: u64,
526    armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530    forwarding: Arc<ForwardingTable>,
531    endpoint: ModuleEndpointId,
532    corr: u64,
533    armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538        Self {
539            forwarding,
540            endpoint,
541            corr,
542            armed: true,
543        }
544    }
545
546    fn disarm(&mut self) {
547        self.armed = false;
548    }
549}
550
551impl Drop for ModuleControlRpcGuard {
552    fn drop(&mut self) {
553        if self.armed {
554            let _ = self
555                .forwarding
556                .cancel_module_control_rpc(self.endpoint, self.corr);
557        }
558    }
559}
560
561impl RouteBindReservationGuard {
562    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563        Self {
564            forwarding,
565            endpoint,
566            relay_corr,
567            armed: true,
568        }
569    }
570
571    fn release_and_disarm(&mut self) {
572        if !self.armed {
573            return;
574        }
575        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576            self.endpoint,
577            self.relay_corr,
578            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579        ) {
580            send_goodbye_target_best_effort(
581                &self.forwarding.counters(),
582                &target,
583                "canceled route.bind",
584            );
585        }
586        self.armed = false;
587    }
588
589    fn disarm(&mut self) {
590        self.armed = false;
591    }
592}
593
594impl Drop for RouteBindReservationGuard {
595    fn drop(&mut self) {
596        self.release_and_disarm();
597    }
598}
599
600/// Per-target-module circuit breaker around the `route.bind` relay.
601///
602/// The connection reader is serial per connection, so a module whose `on_bind`
603/// sits on the ack blocks every LATER frame on the connections that call it,
604/// including calls to unrelated modules. This does not make any module's bind
605/// fast; it stops the daemon paying the full budget again and again for a
606/// condition it has already observed.
607///
608/// State is keyed by TARGET MODULE and shared by every connection: a wedged
609/// module wedges everyone, so what one connection learned should protect the
610/// rest.
611///
612/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
613/// relay to that module has actually timed out, and is removed again when a
614/// relay is accepted or the module reconnects, so it cannot grow with traffic
615/// or with modules that behave.
616///
617/// # Why a `std` mutex here is not the head-of-line defect again
618///
619/// Acquisition never awaits. The critical section is a hash lookup plus a few
620/// integer updates, with no I/O and no `.await` inside it, so a reader task
621/// cannot be descheduled behind it the way it can behind
622/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
623/// primitive, held for the same kind of work, as the refusal counter this very
624/// path already increments.
625///
626/// It is also NOT on the data-plane splice path: only `route.open` and module
627/// registration touch it, so bound-route frames gain no state check and no
628/// contention.
629#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636    modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640    concurrency: RouteBindConcurrency,
641    module_id: String,
642}
643
644impl RouteBindConcurrency {
645    /// Admit without waiting. Waiting here would move the bind stall from the
646    /// module reply to a semaphore and restore reader head-of-line blocking.
647    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648        let mut modules = self
649            .modules
650            .lock()
651            .expect("route.bind concurrency mutex poisoned");
652        let in_flight = modules.entry(module_id.to_string()).or_default();
653        if *in_flight >= limit {
654            return Err(*in_flight);
655        }
656        *in_flight += 1;
657        Ok(RouteBindConcurrencyGuard {
658            concurrency: self.clone(),
659            module_id: module_id.to_string(),
660        })
661    }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665    fn drop(&mut self) {
666        let mut modules = self
667            .concurrency
668            .modules
669            .lock()
670            .expect("route.bind concurrency mutex poisoned");
671        let remove = {
672            let in_flight = modules
673                .get_mut(&self.module_id)
674                .expect("admitted route.bind has a concurrency entry");
675            *in_flight -= 1;
676            *in_flight == 0
677        };
678        if remove {
679            modules.remove(&self.module_id);
680        }
681    }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686    /// Relay timeouts observed with no accepted relay in between.
687    consecutive_timeouts: u32,
688    /// `Some` while the breaker is open: the instant the cooldown expires and
689    /// the next arrival may probe. `None` means closed.
690    cooldown_until: Option<Instant>,
691    /// A half-open probe has been admitted and has not settled yet. This is
692    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
693    /// that read the cooldown, so concurrent opens arriving at the moment the
694    /// cooldown expires cannot all decide that they are the probe.
695    probe_in_flight: Option<Arc<()>>,
696}
697
698/// What the breaker decided for one `route.open`, before any relay work.
699enum RouteBindAdmission<'a> {
700    Admitted {
701        guard: RouteBindBreakerGuard<'a>,
702        /// This open is the single half-open probe, so the transition is worth
703        /// one log line.
704        probe: bool,
705    },
706    Refused {
707        consecutive_timeouts: u32,
708        /// What is left of the cooldown. Zero when the refusal is because the
709        /// one probe is already in flight rather than because the cooldown has
710        /// not elapsed.
711        retry_in: Duration,
712        probe_in_flight: bool,
713    },
714}
715
716/// An outstanding admission, which must be told how its relay settled.
717///
718/// `Drop` settles it as inconclusive, so an early return between admission and
719/// the relay -- or the whole handler being cancelled when the client
720/// disconnects -- releases a half-open probe slot instead of leaving the
721/// breaker wedged half-open with no further probes.
722struct RouteBindBreakerGuard<'a> {
723    breakers: RouteBindBreakers,
724    module_id: &'a str,
725    probe_token: Option<Arc<()>>,
726    settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730    /// The module answered within the budget and took the bind. THE ONLY
731    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
732    /// breaker, which is a transition worth logging.
733    fn record_accepted(&mut self) -> bool {
734        self.settled = true;
735        self.breakers.record_accepted(self.module_id)
736    }
737
738    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
739    /// COUNTS TOWARD OPENING.
740    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741        self.settled = true;
742        self.breakers.record_timeout(
743            self.module_id,
744            self.probe_token.as_ref(),
745            threshold,
746            cooldown,
747        )
748    }
749
750    /// Everything else: the module REJECTED the bind, its connection went away
751    /// mid-relay, or the waiter was cancelled.
752    ///
753    /// None of these is evidence that a module is slow, and each already has
754    /// its own refusal with its own code. A module that rejects a bind in
755    /// microseconds is healthy and must never be convicted for it; a module
756    /// that died has said nothing about the module that replaces it. So these
757    /// neither increment nor reset the count -- they only release a probe slot.
758    fn record_inconclusive(&mut self) {
759        self.settled = true;
760        self.breakers
761            .record_inconclusive(self.module_id, self.probe_token.as_ref());
762    }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766    fn drop(&mut self) {
767        if !self.settled {
768            self.breakers
769                .record_inconclusive(self.module_id, self.probe_token.as_ref());
770        }
771    }
772}
773
774/// The breaker moved to open, reported so the caller can log it outside the
775/// lock. Opening is rare and load-bearing; the refusals that follow are
776/// frequent and are counted rather than logged.
777struct BreakerOpened {
778    consecutive_timeouts: u32,
779    /// True when a failed probe re-opened an already-open breaker, which reads
780    /// very differently in a log from a first opening.
781    reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786        self.modules
787            .lock()
788            .expect("route.bind breaker mutex poisoned")
789    }
790
791    /// Decide whether this `route.open` may attempt its relay. Takes the map
792    /// lock and nothing else, and never awaits.
793    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795            probe: probe_token.is_some(),
796            guard: RouteBindBreakerGuard {
797                breakers: self.clone(),
798                module_id,
799                probe_token,
800                settled: false,
801            },
802        };
803
804        let mut modules = self.lock();
805        let Some(state) = modules.get_mut(module_id) else {
806            return admitted(None);
807        };
808        let Some(cooldown_until) = state.cooldown_until else {
809            return admitted(None);
810        };
811        if state.probe_in_flight.is_some() {
812            return RouteBindAdmission::Refused {
813                consecutive_timeouts: state.consecutive_timeouts,
814                retry_in: Duration::ZERO,
815                probe_in_flight: true,
816            };
817        }
818        let now = Instant::now();
819        if now < cooldown_until {
820            return RouteBindAdmission::Refused {
821                consecutive_timeouts: state.consecutive_timeouts,
822                retry_in: cooldown_until - now,
823                probe_in_flight: false,
824            };
825        }
826        let token = Arc::new(());
827        state.probe_in_flight = Some(Arc::clone(&token));
828        admitted(Some(token))
829    }
830
831    fn record_accepted(&self, module_id: &str) -> bool {
832        self.lock()
833            .remove(module_id)
834            .is_some_and(|state| state.cooldown_until.is_some())
835    }
836
837    fn record_timeout(
838        &self,
839        module_id: &str,
840        probe_token: Option<&Arc<()>>,
841        threshold: u32,
842        cooldown: Duration,
843    ) -> Option<BreakerOpened> {
844        let mut modules = self.lock();
845        let state = modules.entry(module_id.to_string()).or_default();
846        let was_open = state.cooldown_until.is_some();
847        let was_probe = Self::owns_probe(state, probe_token);
848        if was_probe {
849            state.probe_in_flight = None;
850        }
851        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852        if state.consecutive_timeouts < threshold {
853            return None;
854        }
855        state.cooldown_until = Some(Instant::now() + cooldown);
856        Some(BreakerOpened {
857            consecutive_timeouts: state.consecutive_timeouts,
858            reopened_after_probe: was_open && was_probe,
859        })
860    }
861
862    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863        // A relay can finish after the breaker was reset or after it opened
864        // again and started a new probe. Only the guard whose token matches the
865        // active probe may release it, so a late relay never frees a newer probe.
866        state
867            .probe_in_flight
868            .as_ref()
869            .zip(token)
870            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871    }
872
873    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874        if let Some(state) = self.lock().get_mut(module_id) {
875            if Self::owns_probe(state, probe_token) {
876                state.probe_in_flight = None;
877            }
878        }
879    }
880
881    /// Discard what was learned about a module, because the process it was
882    /// learned about is gone. Returns the discarded count when it was non-zero.
883    ///
884    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
885    /// `module_id` is a configuration identity that outlives any particular
886    /// child; what the breaker observed was the process behind the module
887    /// connection of the moment. When a new connection registers under that id
888    /// the verdict's subject no longer exists, so the verdict is stale by
889    /// construction rather than merely likely to be wrong. Keeping it would
890    /// apply a dead process's record to a live one, which is the same defect
891    /// class this breaker exists to stop the daemon committing.
892    ///
893    /// A half-open probe in flight is discarded with the rest: it was a
894    /// question about the old process.
895    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896        self.lock()
897            .remove(module_id)
898            .map(|state| state.consecutive_timeouts)
899            .filter(|discarded| *discarded > 0)
900    }
901
902    /// Open breakers, for the `server.describe` counters object. `None` when
903    /// none is open, so the key stays absent rather than present-and-empty.
904    ///
905    /// This is the operator's answer to "is this module refusing instantly or
906    /// is it fine?", which look identical from a client that retries and then
907    /// succeeds.
908    fn open_snapshot(&self) -> Option<serde_json::Value> {
909        let now = Instant::now();
910        let modules = self.lock();
911        let open = modules
912            .iter()
913            .filter_map(|(module_id, state)| {
914                let cooldown_until = state.cooldown_until?;
915                Some((
916                    module_id.clone(),
917                    serde_json::json!({
918                        "consecutive_timeouts": state.consecutive_timeouts,
919                        "cooldown_remaining_ms":
920                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
921                        "probe_in_flight": state.probe_in_flight.is_some(),
922                    }),
923                ))
924            })
925            .collect::<serde_json::Map<String, serde_json::Value>>();
926        (!open.is_empty()).then_some(serde_json::Value::Object(open))
927    }
928}
929
930impl ControlHandler {
931    pub fn new(registry: Arc<Registry>) -> Self {
932        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933    }
934
935    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936        let counters = forwarding.counters();
937        // Taken from the forwarding table rather than created here, so that the
938        // breaker a `route.open` consults is the same one a module's
939        // registration resets, however many handlers are built over one table.
940        let route_bind_breakers = forwarding.route_bind_breakers();
941        let route_bind_concurrency = forwarding.route_bind_concurrency();
942        let route_outages = forwarding.route_outages();
943        Self {
944            registry,
945            activity: None,
946            activity_streams: Arc::new(Mutex::new(HashMap::new())),
947            forwarding,
948            process_liveness: None,
949            supervisor: SupervisorHandle::new(),
950            subc_capabilities: Arc::from([
951                CAP_MANIFEST_REGISTRATION.to_string(),
952                CAP_CHANNEL_LIFECYCLE.to_string(),
953                CAP_PING_PONG.to_string(),
954                CAP_SESSION_ATTACH.to_string(),
955                CAP_ADMISSION_FACTS_RELAY.to_string(),
956                CAP_SCOPES_V1.to_string(),
957                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966            ]),
967            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968            route_bind_relay_timeouts: BTreeMap::new(),
969            route_bind_breakers,
970            route_bind_concurrency,
971            route_outages,
972            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975            storage_config: None,
976            machine_id: None,
977            admission_facts_carrier_module_id: None,
978            admission_facts_targets: None,
979            scopes: Arc::new(RwLock::new(ScopeTable::new(
980                crate::daemon_config::default_scope_authority_owners(),
981            ))),
982            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986            #[cfg(test)]
987            caller_lookup: None,
988            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989            rescan: None,
990            connected_clients: ConnectedClients::new(),
991            counters,
992            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993            daemon_shutdown: DaemonShutdownFlag::default(),
994            daemon_provenance: DaemonProvenanceFacts::default(),
995            #[cfg(test)]
996            control_dispatch_delay: None,
997            #[cfg(test)]
998            provenance_probe_override: None,
999            #[cfg(test)]
1000            route_bind_relay_sent_hold: None,
1001            #[cfg(test)]
1002            hello_registry_to_forwarding_hold: None,
1003        }
1004    }
1005
1006    /// Set the central storage policy: registering modules then receive their
1007    /// resolved storage descriptor in HELLO_ACK.
1008    pub fn with_storage_config(
1009        mut self,
1010        storage_config: Option<crate::daemon_config::StorageConfig>,
1011    ) -> Self {
1012        self.storage_config = storage_config;
1013        self
1014    }
1015
1016    /// Open the ledger on its own writer thread; an unavailable sink does not
1017    /// prevent the daemon from serving routing or supervisor requests.
1018    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019        self.activity = Some(crate::activity::ActivityLedger::open(
1020            run_dir.join("activity.jsonl"),
1021            incarnation,
1022        ));
1023        self.supervisor.set_activity(self.activity.clone());
1024        self
1025    }
1026
1027    /// Set the machine id served to every registering module (HELLO_ACK) and on
1028    /// `server.describe`.
1029    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030        self.machine_id = machine_id;
1031        self
1032    }
1033
1034    /// Configure the exact reserved module and target ids permitted to relay
1035    /// opaque admission facts. Config-file loading validates this authority;
1036    /// this builder keeps the same policy available to embedded test daemons.
1037    pub fn with_admission_facts_config(
1038        mut self,
1039        carrier_module_id: Option<String>,
1040        targets: Option<Vec<String>>,
1041    ) -> Self {
1042        self.admission_facts_carrier_module_id = carrier_module_id;
1043        self.admission_facts_targets = targets;
1044        self
1045    }
1046
1047    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1048    /// Replaces the scope table with an empty one under the new list, so call it
1049    /// while building the handler, before any module can sync.
1050    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052        self.scope_authority_owners = owners;
1053        self
1054    }
1055
1056    /// Set the startup policy before installing the rescan context or serving requests.
1057    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058        self.privileged_operator_prompt = mode;
1059        self
1060    }
1061
1062    /// Override the route.bind relay timeout. Used by tests that assert the
1063    /// timeout path so they don't block on the production-safe default.
1064    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065        self.route_bind_relay_timeout = timeout;
1066        self
1067    }
1068
1069    /// Install per-module route.bind relay budget overrides. A module id
1070    /// listed here wins over the daemon-wide default set via
1071    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1072    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1073    /// the same `Duration` the bind path will use.
1074    pub fn with_route_bind_relay_timeouts(
1075        mut self,
1076        timeouts: impl IntoIterator<Item = (String, Duration)>,
1077    ) -> Self {
1078        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079        self
1080    }
1081
1082    /// Resolve the route.bind relay budget for a specific target module id.
1083    /// Per-module overrides win; the daemon-wide value (set via
1084    /// `with_route_bind_relay_timeout` or the built-in default) is the
1085    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1086    /// the same resolution `handle_route_open` will use.
1087    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088        self.route_bind_relay_timeouts
1089            .get(module_id)
1090            .copied()
1091            .unwrap_or(self.route_bind_relay_timeout)
1092    }
1093
1094    /// Override the per-module bind-relay breaker policy.
1095    ///
1096    /// Used by tests, which cannot spend three production budgets opening a
1097    /// breaker or twenty seconds waiting for its cooldown. The production
1098    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1099    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1100    /// reasoning for the numbers.
1101    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102        self.route_bind_breaker_threshold = threshold.max(1);
1103        self.route_bind_breaker_cooldown = cooldown;
1104        self
1105    }
1106
1107    #[cfg(test)]
1108    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109        self.health_probe_timeout = timeout;
1110        self
1111    }
1112
1113    #[cfg(test)]
1114    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115        self.control_dispatch_delay = Some(delay);
1116        self
1117    }
1118
1119    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1120    /// announce it is parked, once to be released. Build the barrier for two
1121    /// parties, the test being the other one.
1122    #[cfg(test)]
1123    pub(crate) fn with_route_bind_relay_sent_hold(
1124        mut self,
1125        hold: Arc<tokio::sync::Barrier>,
1126    ) -> Self {
1127        self.route_bind_relay_sent_hold = Some(hold);
1128        self
1129    }
1130
1131    /// A HELLO waits on `hold` twice between reserving its registry entry and
1132    /// installing its forwarding endpoint: once to announce it is parked, once
1133    /// to be released. Build the barrier for two parties, the test being the
1134    /// other one, and run the HELLO on a thread of its own.
1135    #[cfg(test)]
1136    pub(crate) fn with_hello_registry_to_forwarding_hold(
1137        mut self,
1138        hold: Arc<std::sync::Barrier>,
1139    ) -> Self {
1140        self.hello_registry_to_forwarding_hold = Some(hold);
1141        self
1142    }
1143
1144    #[cfg(test)]
1145    fn hold_between_registry_and_forwarding(&self) {
1146        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147            hold.wait();
1148            hold.wait();
1149        }
1150    }
1151
1152    pub fn with_process_liveness(
1153        mut self,
1154        process_liveness: Arc<dyn ModuleProcessLiveness>,
1155    ) -> Self {
1156        self.process_liveness = Some(process_liveness);
1157        self
1158    }
1159
1160    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161        self.supervisor = supervisor;
1162        self.supervisor.set_activity(self.activity.clone());
1163        self
1164    }
1165
1166    pub fn with_daemon_provenance(
1167        mut self,
1168        pid: u32,
1169        started_at_ms: u64,
1170        executable_path: Option<PathBuf>,
1171        build_git_sha: Option<String>,
1172        build_lock_digest: Option<String>,
1173    ) -> Self {
1174        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175        let process_start_time = process_start_time(pid);
1176        self.daemon_provenance = DaemonProvenanceFacts {
1177            build: DaemonBuildProvenance {
1178                build_git_sha,
1179                build_lock_digest,
1180            },
1181            pid: Some(pid),
1182            started_at_ms: Some(started_at_ms),
1183            start_clock: None,
1184            executable_path,
1185            executable_identity,
1186            process_start_time,
1187            probe: ExecutableIdentityProbe::default(),
1188        };
1189        self
1190    }
1191
1192    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193        self.daemon_provenance.start_clock = Some(clock);
1194        self
1195    }
1196
1197    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1198    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199        self.scopes
1200            .write()
1201            .unwrap_or_else(|poisoned| poisoned.into_inner())
1202            .set_wall_clock(Arc::new(clock));
1203        self
1204    }
1205
1206    #[cfg(test)]
1207    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208        self.provenance_probe_override = Some(result);
1209        self
1210    }
1211
1212    /// Install the configured module set and its reserved capability bindings.
1213    /// Bindings are configuration-scoped and may point at a provider that has not
1214    /// been installed yet, so this does not require the bound module to exist.
1215    pub fn with_capability_config(
1216        self,
1217        modules: impl IntoIterator<Item = (String, bool)>,
1218        reserved_capabilities: BTreeMap<String, String>,
1219    ) -> Self {
1220        self.capability_evaluator
1221            .configure(modules, reserved_capabilities);
1222        self
1223    }
1224
1225    /// Share the supervisor's daemon-shutdown flag with this handler, so
1226    /// requirement changes after the daemon has begun its announced shutdown
1227    /// are not reported as faults.
1228    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229        self.daemon_shutdown = flag;
1230        self
1231    }
1232
1233    pub fn with_supervisor_rescan(
1234        mut self,
1235        supervisor: Supervisor,
1236        config_path: impl Into<PathBuf>,
1237        configured_port: Option<u16>,
1238    ) -> Self {
1239        self.rescan = Some(SupervisorRescanContext {
1240            supervisor,
1241            config_path: config_path.into(),
1242            configured_port,
1243            storage_config: self.storage_config.clone(),
1244            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245            admission_facts_targets: self.admission_facts_targets.clone(),
1246            scope_authority_owners: self.scope_authority_owners.clone(),
1247            privileged_operator_prompt: self.privileged_operator_prompt,
1248            generation: Arc::default(),
1249        });
1250        self
1251    }
1252
1253    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254        self.connected_clients = connected_clients;
1255        self
1256    }
1257
1258    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259        Arc::clone(&self.forwarding)
1260    }
1261
1262    pub(crate) fn counters(&self) -> DaemonCounters {
1263        self.counters.clone()
1264    }
1265
1266    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1267    /// requirement event without depending on an operator polling a status command.
1268    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269        tokio::spawn(async move {
1270            loop {
1271                self.capability_evaluator
1272                    .wait_for_change_or_deadline()
1273                    .await;
1274                self.refresh_capability_requirements();
1275            }
1276        });
1277    }
1278
1279    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1280    /// deadline passed during sleep is enforced on the first tick after wake.
1281    /// While no live scope carries a deadline the loop waits for an accepted
1282    /// scope change instead of ticking, so a daemon without deadlines never
1283    /// wakes for expiry.
1284    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285        tokio::spawn(async move {
1286            loop {
1287                let has_deadlines = self
1288                    .scopes
1289                    .read()
1290                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1291                    .has_deadlines();
1292                if !has_deadlines {
1293                    self.scope_deadline_added.notified().await;
1294                    continue;
1295                }
1296                tokio::time::sleep(Duration::from_secs(1)).await;
1297                if let Err(error) = self.sweep_expired_scopes() {
1298                    warn!(%error, "scope expiry sweep failed");
1299                }
1300            }
1301        });
1302    }
1303
1304    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305        let mut table = self
1306            .scopes
1307            .write()
1308            .unwrap_or_else(|poisoned| poisoned.into_inner());
1309        let swept = table.sweep_expired();
1310        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311        drop(table);
1312        self.close_scope_drained_routes(drained);
1313        Ok(())
1314    }
1315
1316    fn publish_scope_changes(
1317        &self,
1318        changes: &[crate::scopes::ScopeTagChange],
1319        expired: &[crate::scopes::ScopeExpired],
1320    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321        if changes.is_empty() {
1322            return Ok(Vec::new());
1323        }
1324        let drained = self
1325            .forwarding
1326            .publish_scope_changes(changes)
1327            .map_err(RouterError::Forwarding)?;
1328        let mut counts = HashMap::new();
1329        for route in &drained {
1330            *counts
1331                .entry((
1332                    route.scope.owner.as_str(),
1333                    route.scope.scope_ref.as_str(),
1334                    route.scope.tag.scope_epoch,
1335                ))
1336                .or_insert(0usize) += 1;
1337        }
1338        for scope in expired {
1339            let routes_closed = counts
1340                .get(&(
1341                    scope.owner.as_str(),
1342                    scope.scope_ref.as_str(),
1343                    scope.scope_epoch,
1344                ))
1345                .copied()
1346                .unwrap_or(0);
1347            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349                routes_closed, "scope expired");
1350        }
1351        Ok(drained)
1352    }
1353
1354    fn runtime_capability_snapshot(
1355        &self,
1356    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357        let runtime = self
1358            .supervisor
1359            .list()
1360            .into_iter()
1361            .map(|module| {
1362                let status = module.status().map_err(|err| {
1363                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364                })?;
1365                Ok(RuntimeModule {
1366                    module_id: status.module_id,
1367                    state: status.state,
1368                    enabled: status.enabled,
1369                })
1370            })
1371            .collect::<Result<Vec<_>, RouterError>>()?;
1372        // Include every registration, published or not. The capability
1373        // evaluator counts a module as providing its capabilities as soon as it
1374        // has registered, even before `catalog.list` shows it, so a HELLO in
1375        // progress does not change other modules' required-capability
1376        // verdicts. See `Registry::list_registered_modules`.
1377        let registrations = self.registry.list_registered_modules().map_err(|err| {
1378            RouterError::backend(
1379                0,
1380                0,
1381                format!("failed to list capability registrations: {err}"),
1382            )
1383        })?;
1384        let registrations = registrations
1385            .into_iter()
1386            .map(|registration| RegisteredModule {
1387                module_id: registration.manifest.module_id,
1388                module_version: registration.manifest.module_version,
1389                capabilities: registration.manifest.capabilities,
1390            })
1391            .collect();
1392        Ok((runtime, registrations))
1393    }
1394
1395    /// The capability side effects of a module becoming the active registration
1396    /// for its id: cache its manifest (warning if its claims drifted), run the
1397    /// deny census when its declarations call for one, and recompute the
1398    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1399    /// candidate's does not, and the supervisor does it at promotion instead,
1400    /// through [`crate::supervise::SwapPromotionObserver`].
1401    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402        let cached_registration = RegisteredModule {
1403            module_id: registration.manifest.module_id.clone(),
1404            module_version: registration.manifest.module_version.clone(),
1405            capabilities: registration.manifest.capabilities.clone(),
1406        };
1407        if self.capability_evaluator.record_hello(&cached_registration) {
1408            warn!(
1409                module_id = %cached_registration.module_id,
1410                "capability claims drifted from the cached manifest"
1411            );
1412        }
1413        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414            self.enforce_capability_denies();
1415        }
1416        self.refresh_capability_requirements();
1417    }
1418
1419    /// Point the shared supervisor handle at this handler for swap promotions.
1420    /// Called wherever a handler is put behind the `Arc` the router serves, so
1421    /// it can be held weakly.
1422    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425        self.supervisor.set_swap_promotion_observer(observer);
1426    }
1427
1428    pub fn refresh_capability_requirements(&self) {
1429        match self.runtime_capability_snapshot() {
1430            Ok((runtime, registrations)) => {
1431                log_requirement_events(
1432                    self.capability_evaluator
1433                        .evaluate_now(&runtime, &registrations),
1434                    &self.daemon_shutdown,
1435                );
1436            }
1437            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438        }
1439    }
1440
1441    /// Reconcile only live, attested route bindings after a capability deny edge
1442    /// or target claim was added. This is deliberately a control-plane census:
1443    /// the opaque forwarding hot path must not grow a per-frame capability check.
1444    fn enforce_capability_denies(&self) {
1445        let registrations = match self.registry.list_registered_modules() {
1446            Ok(snapshot) => snapshot,
1447            Err(err) => {
1448                warn!(error = %err, "failed to read registrations for capability deny census");
1449                return;
1450            }
1451        };
1452        let manifests = registrations
1453            .into_iter()
1454            .map(|registration| {
1455                (
1456                    registration.manifest.module_id.clone(),
1457                    registration.manifest,
1458                )
1459            })
1460            .collect::<BTreeMap<_, _>>();
1461        let census = match self.forwarding.route_census(None) {
1462            Ok(census) => census,
1463            Err(err) => {
1464                warn!(error = %err, "failed to read route census for capability deny enforcement");
1465                return;
1466            }
1467        };
1468
1469        for (target_module_id, routes) in census {
1470            let Some(target_manifest) = manifests.get(&target_module_id) else {
1471                continue;
1472            };
1473            let mut closed_routes = Vec::new();
1474            let mut goodbyes = Vec::new();
1475            for route in routes {
1476                let Principal::Reserved {
1477                    module_id: opening_module_id,
1478                } = &route.principal
1479                else {
1480                    continue;
1481                };
1482                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483                    continue;
1484                };
1485                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486                    continue;
1487                };
1488
1489                match self.forwarding.release_client_route(
1490                    route.goodbye_target.connection_id,
1491                    route.goodbye_target.channel,
1492                    route.goodbye_target.epoch,
1493                ) {
1494                    Ok(RouteRelease::Removed(module_goodbye)) => {
1495                        warn!(
1496                            opening_module_id,
1497                            target_module_id,
1498                            capability,
1499                            "force-closing route because an attested capability deny edge now matches"
1500                        );
1501                        goodbyes.push(module_goodbye);
1502                        goodbyes.push(route.goodbye_target.clone());
1503                        closed_routes.push(route);
1504                    }
1505                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1506                    Err(err) => warn!(
1507                        opening_module_id,
1508                        target_module_id,
1509                        capability,
1510                        error = %err,
1511                        "failed to force-close capability-denied route"
1512                    ),
1513                }
1514            }
1515
1516            if closed_routes.is_empty() {
1517                continue;
1518            }
1519            send_route_control_pushes(
1520                &self.forwarding,
1521                closed_routes,
1522                ClientControlPush::RouteClosed {
1523                    module_id: target_module_id,
1524                    channels: Vec::new(),
1525                    reason: RouteCloseReason::CapabilityDenied,
1526                    drained: false,
1527                    abandoned: 0,
1528                    excluded_subscriptions: 0,
1529                    terminal: Some(false),
1530                },
1531            );
1532            // route.closed records the reason; the route GOODBYE ends the
1533            // client's handle and settles any calls still pending on it.
1534            self.emit_route_goodbyes(goodbyes);
1535        }
1536    }
1537
1538    /// Why a registered module is not accepting new route binds, or `None` when
1539    /// it is. This is the module's effective readiness: its declared readiness
1540    /// first, then every `need: required` capability it declares evaluating to
1541    /// `provided`. `route.open` and `catalog.list` both read it here so the
1542    /// catalog never reports a module routable that `route.open` would refuse.
1543    fn not_ready_reason(
1544        &self,
1545        registration: &crate::registry::ModuleRegistration,
1546    ) -> Option<NotReadyReason> {
1547        if !registration.ready {
1548            return Some(NotReadyReason {
1549                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1550                capability: None,
1551            });
1552        }
1553        self.first_unprovided_required_capability(registration)
1554            .map(|capability| NotReadyReason {
1555                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1556                capability: Some(capability),
1557            })
1558    }
1559
1560    /// The lexicographically first capability this registration declares
1561    /// `need: required` whose evaluator verdict is not `provided`.
1562    ///
1563    /// The verdicts are the capability evaluator's own; nothing here decides
1564    /// what "provided" means. The evaluator counts a capability provided as
1565    /// soon as a module claiming it has REGISTERED, not once that module is
1566    /// ready. That distinction is what keeps two modules that require each
1567    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1568    /// is ready", each would wait for the other to become ready first and
1569    /// neither ever would. Do not tighten it to readiness.
1570    ///
1571    /// A required capability with no verdict at all means this registration's
1572    /// HELLO or catalog.update landed after the last recompute; recompute once
1573    /// rather than let a missing verdict read as either answer. If it is still
1574    /// missing (the recompute itself failed) the capability counts as
1575    /// unprovided: the refusal is retryable, and routing a module whose
1576    /// required provider is unknown is the outcome this check exists to stop.
1577    fn first_unprovided_required_capability(
1578        &self,
1579        registration: &crate::registry::ModuleRegistration,
1580    ) -> Option<String> {
1581        let required = registration
1582            .manifest
1583            .capabilities
1584            .iter()
1585            .flat_map(|declarations| declarations.requires.iter())
1586            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1587            .map(|requirement| requirement.capability.as_str())
1588            .collect::<BTreeSet<_>>();
1589        if required.is_empty() {
1590            return None;
1591        }
1592        let module_id = registration.manifest.module_id.as_str();
1593        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1594        if required
1595            .iter()
1596            .any(|capability| verdict(capability).is_none())
1597        {
1598            self.refresh_capability_requirements();
1599        }
1600        required
1601            .into_iter()
1602            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1603            .map(str::to_string)
1604    }
1605
1606    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1607        self.capability_evaluator
1608            .statuses()
1609            .into_iter()
1610            .map(capability_requirement_status)
1611            .collect()
1612    }
1613
1614    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1615    /// registration-release watch. The signal is what the supervisor waits on
1616    /// before spawning a replacement, so it must only fire once forwarding
1617    /// teardown is also done (see [`Self::cleanup_connection`] /
1618    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1619    /// admitted but its forwarding endpoint could not be installed.
1620    fn deregister_connection(
1621        &self,
1622        connection_id: ConnectionId,
1623        reason: RegistrationEndReason,
1624    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1625        self.registry
1626            .deregister_connection_with_reason(connection_id, reason)
1627    }
1628
1629    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1630        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1631            return None;
1632        }
1633        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1634            parse_client_control_request(&frame.body)
1635        else {
1636            return None;
1637        };
1638        Some(target_module_id(&target).to_string())
1639    }
1640
1641    pub(crate) fn route_open_capacity_refusal(
1642        &self,
1643        ctx: &RouteCtx,
1644        frame: &Frame,
1645        target_module_id: &str,
1646        in_flight: usize,
1647        limit: usize,
1648    ) -> Result<Frame, RouterError> {
1649        // Sent before the request reaches `handle_route_open`, so the caller
1650        // fields come from the raw frame here.
1651        let caller = self.route_open_caller_from_frame(frame);
1652        self.route_open_admission_refusal_frame(
1653            ctx,
1654            &caller,
1655            frame,
1656            target_module_id,
1657            "open_admission_full",
1658            (in_flight, limit),
1659            format!(
1660                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1661            ),
1662        )
1663    }
1664
1665    fn route_open_target_capacity_refusal(
1666        &self,
1667        ctx: &RouteCtx,
1668        caller: &RouteOpenCaller,
1669        frame: &Frame,
1670        target_module_id: &str,
1671        in_flight: usize,
1672    ) -> Result<Frame, RouterError> {
1673        self.route_open_admission_refusal_frame(
1674            ctx,
1675            caller,
1676            frame,
1677            target_module_id,
1678            "target_binds_full",
1679            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1680            format!(
1681                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1682            ),
1683        )
1684    }
1685
1686    /// Admission pressure clears as existing binds settle, so its refusal must
1687    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1688    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1689    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1690    /// cannot currently reach its target; `module_timeout` would falsely claim
1691    /// that a wait expired. A new, cleaner code would be terminal to deployed
1692    /// clients, so it requires a client-tolerance rollout before daemon emission.
1693    #[allow(clippy::too_many_arguments)]
1694    fn route_open_admission_refusal_frame(
1695        &self,
1696        ctx: &RouteCtx,
1697        caller: &RouteOpenCaller,
1698        frame: &Frame,
1699        target_module_id: &str,
1700        reason: &'static str,
1701        (in_flight, limit): (usize, usize),
1702        message: impl Into<String>,
1703    ) -> Result<Frame, RouterError> {
1704        let code = error_codes::TARGET_UNAVAILABLE;
1705        self.counters.increment_route_open_refused(code);
1706        info!(
1707            target: "control",
1708            code,
1709            reason,
1710            module_id = ?target_module_id,
1711            connection_id = ctx.connection_id.get(),
1712            principal = %caller.principal,
1713            scope = %caller.scope,
1714            harness = %caller.harness,
1715            in_flight,
1716            limit,
1717            "route.open refused"
1718        );
1719        control_error_frame(frame, code, message.into())
1720    }
1721
1722    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1723    ///
1724    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1725    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1726    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1727    #[cfg(test)]
1728    pub fn handle_control(
1729        &self,
1730        connection_id: ConnectionId,
1731        frame: Frame,
1732    ) -> Result<Vec<Frame>, RouterError> {
1733        match frame.header.ty {
1734            FrameType::Ping => Ok(vec![pong(&frame)?]),
1735            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1736            FrameType::Goodbye => self.handle_goodbye(connection_id),
1737            ty => Ok(vec![control_error_frame(
1738                &frame,
1739                "unsupported_control_frame",
1740                format!("unsupported channel-0 frame {ty:?}"),
1741            )?]),
1742        }
1743    }
1744
1745    pub async fn handle_control_frame(
1746        &self,
1747        ctx: &RouteCtx,
1748        frame: Frame,
1749    ) -> Result<Vec<Frame>, RouterError> {
1750        self.handle_control_frame_timed(ctx, frame, None).await
1751    }
1752
1753    pub(crate) fn record_caller_endpoints(
1754        &self,
1755        connection: ConnectionId,
1756        endpoints: Option<(SocketAddr, SocketAddr)>,
1757    ) {
1758        let mut callers = self
1759            .caller_endpoints
1760            .lock()
1761            .unwrap_or_else(|p| p.into_inner());
1762        callers.remove(&connection);
1763        if let Some(endpoints) = endpoints {
1764            callers.insert(connection, endpoints);
1765        }
1766    }
1767
1768    #[cfg(test)]
1769    pub(crate) fn with_caller_lookup_for_test(
1770        mut self,
1771        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1772    ) -> Self {
1773        self.caller_lookup = Some(Arc::new(lookup));
1774        self
1775    }
1776
1777    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1778        let endpoints = self
1779            .caller_endpoints
1780            .lock()
1781            .unwrap_or_else(|p| p.into_inner())
1782            .get(&connection)
1783            .copied();
1784        #[cfg(test)]
1785        let lookup = self.caller_lookup.clone();
1786        tokio::task::spawn_blocking(move || {
1787            #[cfg(test)]
1788            if let Some(lookup) = lookup {
1789                return lookup(endpoints);
1790            }
1791            endpoints
1792                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1793                .unwrap_or_default()
1794        })
1795        .await
1796        .unwrap_or_default()
1797    }
1798
1799    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1800        frame.header.channel == 0
1801            && frame.header.ty == FrameType::Request
1802            && self
1803                .forwarding
1804                .module_endpoint_for_connection(ctx.connection_id)
1805                .is_ok_and(|endpoint| endpoint.is_none())
1806            && parse_client_control_request(&frame.body)
1807                .ok()
1808                .is_some_and(|request| privileged_operation(&request).is_some())
1809    }
1810
1811    pub(crate) async fn handle_control_frame_timed(
1812        &self,
1813        ctx: &RouteCtx,
1814        frame: Frame,
1815        dispatch_started_at: Option<StdInstant>,
1816    ) -> Result<Vec<Frame>, RouterError> {
1817        match frame.header.ty {
1818            FrameType::Ping => Ok(vec![pong(&frame)?]),
1819            FrameType::Hello => {
1820                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1821            }
1822            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1823            FrameType::Cancel => {
1824                // A Cancel on channel 0 names a waiting operator.confirm or a
1825                // subscription. Successful cancellation has no ordinary reply.
1826                if self
1827                    .forwarding
1828                    .operator_confirms()
1829                    .cancel(ctx.connection_id, frame.header.corr)
1830                    || self
1831                        .supervisor
1832                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1833                    || self.cancel_activity_subscription(ctx, &frame)?
1834                {
1835                    Ok(Vec::new())
1836                } else {
1837                    Ok(vec![control_error_frame(
1838                        &frame,
1839                        "unknown_subscription",
1840                        "no subscription has this correlation id",
1841                    )?])
1842                }
1843            }
1844            FrameType::Request => {
1845                // This additive operation is not part of the existing exhaustive
1846                // module-control enum. Probe the op before decoding that enum.
1847                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1848                if op
1849                    .as_ref()
1850                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1851                {
1852                    return self.handle_activity_record(ctx, frame).await;
1853                }
1854                if op.as_ref().is_some_and(|probe| {
1855                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1856                }) {
1857                    return self.handle_activity_read(ctx, frame).await;
1858                }
1859                if op
1860                    .as_ref()
1861                    .is_some_and(|probe| probe.op == "operator.confirm")
1862                {
1863                    return self.handle_operator_confirm(ctx, frame);
1864                }
1865                if op
1866                    .as_ref()
1867                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1868                {
1869                    return self.handle_scope_served_by(ctx.connection_id, frame);
1870                }
1871                if self
1872                    .forwarding
1873                    .module_endpoint_for_connection(ctx.connection_id)
1874                    .map_err(RouterError::Forwarding)?
1875                    .is_some()
1876                {
1877                    if !is_known_module_request_op(&frame.body) {
1878                        return Ok(vec![control_error_frame(
1879                            &frame,
1880                            "unsupported_control_frame",
1881                            "module-originated channel-0 REQUEST is not supported",
1882                        )?]);
1883                    }
1884                    let request = match parse_module_control_request_from_module(&frame.body) {
1885                        Ok(request) => request,
1886                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1887                            return Ok(vec![control_error_frame(
1888                                &frame,
1889                                "unsupported_control_frame",
1890                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1891                            )?])
1892                        }
1893                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1894                            return Ok(vec![control_error_frame(
1895                                &frame,
1896                                "invalid_control_body",
1897                                format!("malformed module control body: {err}"),
1898                            )?])
1899                        }
1900                    };
1901                    let op = module_control_request_op(&request);
1902                    let corr = frame.header.corr;
1903                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1904                    let result =
1905                        self.handle_module_control_request(ctx.connection_id, frame, request);
1906                    log_slow_control_dispatch(
1907                        dispatch_started_at,
1908                        op,
1909                        ctx.connection_id,
1910                        corr,
1911                        None,
1912                    );
1913                    return result;
1914                }
1915
1916                if is_known_module_request_op(&frame.body) {
1917                    return Ok(vec![control_error_frame(
1918                        &frame,
1919                        "not_registered",
1920                        "catalog.update requires an active module registration owned by this connection",
1921                    )?]);
1922                }
1923
1924                let request = match parse_client_control_request(&frame.body) {
1925                    Ok(request) => request,
1926                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1927                        return Ok(vec![control_error_frame(
1928                            &frame,
1929                            "unknown_control_op",
1930                            format!("unknown client control op: {err}"),
1931                        )?])
1932                    }
1933                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1934                        return Ok(vec![control_error_frame(
1935                            &frame,
1936                            "invalid_control_body",
1937                            format!("malformed client control body: {err}"),
1938                        )?])
1939                    }
1940                };
1941                let op = client_control_request_op(&request);
1942                let corr = frame.header.corr;
1943                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1944                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1945                #[cfg(test)]
1946                if let Some(delay) = self.control_dispatch_delay {
1947                    tokio::time::sleep(delay).await;
1948                }
1949                let result = self
1950                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1951                    .await;
1952                if let Some(timings) = route_open_timings.as_mut() {
1953                    timings.finish();
1954                }
1955                log_slow_control_dispatch(
1956                    dispatch_started_at,
1957                    op,
1958                    ctx.connection_id,
1959                    corr,
1960                    route_open_timings.as_ref(),
1961                );
1962                result
1963            }
1964            FrameType::Push => {
1965                let Some(endpoint) = self
1966                    .forwarding
1967                    .module_endpoint_for_connection(ctx.connection_id)
1968                    .map_err(RouterError::Forwarding)?
1969                else {
1970                    return Ok(vec![control_error_frame(
1971                        &frame,
1972                        "unsupported_control_frame",
1973                        "client-originated channel-0 PUSH is not supported",
1974                    )?]);
1975                };
1976                self.handle_status_update(endpoint, frame)
1977            }
1978            FrameType::Response | FrameType::Error
1979                if self
1980                    .forwarding
1981                    .module_endpoint_for_connection(ctx.connection_id)
1982                    .map_err(RouterError::Forwarding)?
1983                    .is_some() =>
1984            {
1985                self.handle_module_relay_response(ctx.connection_id, frame)
1986            }
1987            ty => Ok(vec![control_error_frame(
1988                &frame,
1989                "unsupported_control_frame",
1990                format!("unsupported channel-0 frame {ty:?}"),
1991            )?]),
1992        }
1993    }
1994
1995    pub fn cleanup_connection(
1996        &self,
1997        connection_id: ConnectionId,
1998    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1999        self.caller_endpoints
2000            .lock()
2001            .unwrap_or_else(|p| p.into_inner())
2002            .remove(&connection_id);
2003        self.forwarding
2004            .operator_confirms()
2005            .connection_closed(connection_id);
2006        self.cleanup_connection_with_end_reason(
2007            connection_id,
2008            RegistrationEndReason::ConnectionClosed,
2009        )
2010    }
2011
2012    fn cleanup_connection_with_end_reason(
2013        &self,
2014        connection_id: ConnectionId,
2015        requested_reason: RegistrationEndReason,
2016    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2017        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2018            self.registry
2019                .get_module_by_connection(connection_id)?
2020                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2021                .and_then(|module| module.registration_end_reason().ok().flatten())
2022                .unwrap_or(requested_reason)
2023        } else {
2024            requested_reason
2025        };
2026        let crash_closed = self
2027            .registry
2028            .get_module_by_connection(connection_id)?
2029            .and_then(|registration| {
2030                self.forwarding
2031                    .module_endpoint_for_connection(connection_id)
2032                    .ok()
2033                    .flatten()
2034                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2035                    .map(|routes| (registration.manifest.module_id, routes))
2036            });
2037        let crash_closed = crash_closed.map(|(module_id, routes)| {
2038            let terminal = match self.supervisor.get(&module_id) {
2039                None => false,
2040                Some(module) => match module.will_recover_after_connection_loss() {
2041                    Ok(will_recover) => !will_recover,
2042                    Err(err) => {
2043                        warn!(
2044                            %module_id,
2045                            error = %err,
2046                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2047                        );
2048                        false
2049                    }
2050                },
2051            };
2052            // The forwarding table gates all providers at the start of daemon
2053            // shutdown, before their connections are closed. An ordinary
2054            // module disconnect still reports crash if that gate is not set.
2055            let (reason, drained) = match self
2056                .forwarding
2057                .daemon_drain_result_for_connection(connection_id)
2058            {
2059                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2060                Ok(None) => (RouteCloseReason::Crash, false),
2061                Err(err) => {
2062                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2063                    (RouteCloseReason::Crash, false)
2064                }
2065            };
2066            (module_id, routes, reason, drained, terminal)
2067        });
2068        let registrations = self.deregister_connection(connection_id, end_reason);
2069        let cleanup = if crash_closed.is_some() {
2070            self.forwarding.cleanup_connection_counted(connection_id)
2071        } else {
2072            // A module connection's teardown needs the count of abandoned
2073            // route.bind relays for its route.closed notice below. Any other
2074            // connection, such as a client's, sends no such notice and needs
2075            // only its routes released, so it uses the route-only wrapper and
2076            // reports zero.
2077            self.forwarding
2078                .cleanup_connection(connection_id)
2079                .map(|released| crate::forwarding::ConnectionCleanup {
2080                    released,
2081                    abandoned_relays: 0,
2082                })
2083        };
2084        // The route.closed push waits for forwarding teardown because only
2085        // teardown knows how many pending route.bind relays it aborted. It still
2086        // goes out before the GOODBYEs for the released routes, and its targets
2087        // were captured above, before teardown removed those routes.
2088        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2089            let abandoned = cleanup
2090                .as_ref()
2091                .map_or(0, |cleanup| cleanup.abandoned_relays);
2092            send_route_control_pushes(
2093                &self.forwarding,
2094                routes,
2095                ClientControlPush::RouteClosed {
2096                    module_id,
2097                    channels: Vec::new(),
2098                    reason,
2099                    drained,
2100                    abandoned,
2101                    excluded_subscriptions: 0,
2102                    terminal: Some(terminal),
2103                },
2104            );
2105        }
2106        if let Ok(cleanup) = cleanup {
2107            self.emit_route_goodbyes(cleanup.released);
2108        }
2109        // Signal the registration-release watch only now that BOTH registry and
2110        // forwarding teardown are done, so a supervisor waiting to spawn a
2111        // replacement never observes release while old routes still exist.
2112        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2113            crate::supervise::notify_registration_release();
2114            self.capability_evaluator.wake_deadline_loop();
2115            self.refresh_capability_requirements();
2116        }
2117        self.supervisor.remove_spawn_subscribers(connection_id);
2118        self.activity_streams
2119            .lock()
2120            .unwrap_or_else(|p| p.into_inner())
2121            .retain(|(id, _), task| {
2122                if *id == connection_id {
2123                    task.abort();
2124                    false
2125                } else {
2126                    true
2127                }
2128            });
2129        // Sync authority dies with its connection, so the owner's next
2130        // connection can take it; the owner's scopes stay as they are.
2131        self.hello_launch_nonces
2132            .lock()
2133            .unwrap_or_else(|poisoned| poisoned.into_inner())
2134            .forget(connection_id);
2135        self.scopes
2136            .write()
2137            .unwrap_or_else(|poisoned| poisoned.into_inner())
2138            .release_connection(connection_id);
2139        registrations
2140    }
2141
2142    pub(crate) fn handle_route_goodbye(
2143        &self,
2144        connection_id: ConnectionId,
2145        route_channel: u16,
2146        route_epoch: u32,
2147    ) -> Result<bool, RouterError> {
2148        debug!(
2149            connection_id = connection_id.get(),
2150            route_channel, route_epoch, "handling route GOODBYE"
2151        );
2152        let RouteRelease::Removed(released_route) = self
2153            .forwarding
2154            .release_client_route(connection_id, route_channel, route_epoch)
2155            .map_err(RouterError::Forwarding)?
2156        else {
2157            return Ok(false);
2158        };
2159        self.emit_route_goodbyes(vec![released_route]);
2160        Ok(true)
2161    }
2162
2163    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2164        for released in released_routes {
2165            let frame = match Frame::build_with_version(
2166                released.negotiated_ver,
2167                FrameType::Goodbye,
2168                control_flags(),
2169                released.channel,
2170                released.epoch,
2171                0,
2172                Vec::new(),
2173            ) {
2174                Ok(frame) => frame,
2175                Err(err) => {
2176                    warn!(
2177                        route_channel = released.channel,
2178                        error = %err,
2179                        "failed to build route GOODBYE frame"
2180                    );
2181                    continue;
2182                }
2183            };
2184            if !released.close_on_delivery_failure() {
2185                crate::forwarding::send_module_route_goodbye(
2186                    &self.counters,
2187                    &released.sink,
2188                    frame,
2189                    released.module_id.as_deref(),
2190                    "client route released",
2191                );
2192                continue;
2193            }
2194            if let Err(err) = released.sink.try_send(frame) {
2195                warn!(
2196                    target_connection_id = released.connection_id.get(),
2197                    route_channel = released.channel,
2198                    error = %err,
2199                    "route GOODBYE was not delivered to client; closing target connection"
2200                );
2201                if self
2202                    .forwarding
2203                    .escalate_client_delivery_failure(
2204                        released.connection_id,
2205                        released.channel,
2206                        released.epoch,
2207                        CloseReason::new(
2208                            "route_goodbye_delivery_failed",
2209                            format!(
2210                                "failed to enqueue route GOODBYE for channel {}: {err}",
2211                                released.channel
2212                            ),
2213                        ),
2214                        crate::forwarding::UndeliveredFrame {
2215                            module_id: released.module_id.as_deref(),
2216                            sink: &released.sink,
2217                        },
2218                    )
2219                    .unwrap_or(false)
2220                {
2221                    self.counters.increment_goodbye_relay_client_failed();
2222                }
2223            }
2224        }
2225    }
2226
2227    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2228    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2229    /// own commit failed after the module had already accepted). Without this, a
2230    /// module that accepts late keeps a binding subc has torn down, so a later
2231    /// frame on that module channel could misdeliver if the channel is reused.
2232    ///
2233    /// Never closes the shared module connection on failure: a dropped notification
2234    /// only wastes a bounded amount of warm module-side state, which the module's
2235    /// own idle reaper reclaims. Only call this once the route.bind relay was
2236    /// actually enqueued to the module — if the relay send itself failed, the
2237    /// module never created a binding and there is nothing to tear down.
2238    fn send_abandoned_route_bind_goodbye(
2239        &self,
2240        module_sink: &crate::FrameSink,
2241        negotiated_ver: u8,
2242        module_channel: u16,
2243        module_epoch: u32,
2244    ) {
2245        let frame = match Frame::build_with_version(
2246            negotiated_ver,
2247            FrameType::Goodbye,
2248            control_flags(),
2249            module_channel,
2250            module_epoch,
2251            0,
2252            Vec::new(),
2253        ) {
2254            Ok(frame) => frame,
2255            Err(err) => {
2256                warn!(
2257                    route_channel = module_channel,
2258                    error = %err,
2259                    "failed to build GOODBYE for abandoned route.bind"
2260                );
2261                return;
2262            }
2263        };
2264        crate::forwarding::send_module_route_goodbye(
2265            &self.counters,
2266            module_sink,
2267            frame,
2268            None,
2269            "abandoned route.bind",
2270        );
2271    }
2272
2273    fn handle_hello(
2274        &self,
2275        connection_id: ConnectionId,
2276        sink: Option<crate::FrameSink>,
2277        frame: Frame,
2278    ) -> Result<Vec<Frame>, RouterError> {
2279        debug!(
2280            connection_id = connection_id.get(),
2281            corr = frame.header.corr,
2282            "handling HELLO"
2283        );
2284        // A module connection has one identity for its entire lifetime. A second
2285        // registration would leave the old registry owner behind while replacing
2286        // its forwarding endpoint and launch nonce.
2287        if self
2288            .registry
2289            .get_module_by_connection(connection_id)
2290            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2291            .is_some()
2292        {
2293            return Ok(vec![control_error_frame(
2294                &frame,
2295                "invalid_hello",
2296                "connection is already registered as a module",
2297            )?]);
2298        }
2299        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2300            Ok(value) => value,
2301            Err(err) => {
2302                return Ok(vec![control_error_frame(
2303                    &frame,
2304                    "invalid_hello",
2305                    format!("malformed HELLO body: {err}"),
2306                )?])
2307            }
2308        };
2309        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2310            return Ok(vec![control_error_frame(
2311                &frame,
2312                "invalid_capability_grammar",
2313                err.to_string(),
2314            )?]);
2315        }
2316        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2317            return Ok(vec![control_error_frame(
2318                &frame,
2319                "invalid_manifest",
2320                err.to_string(),
2321            )?]);
2322        }
2323        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2324            return Ok(vec![control_error_frame(
2325                &frame,
2326                "invalid_event_declaration",
2327                err.to_string(),
2328            )?]);
2329        }
2330        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2331            return Ok(vec![control_error_frame(
2332                &frame,
2333                "invalid_manifest",
2334                err.to_string(),
2335            )?]);
2336        }
2337        if let Some(provenance) = hello_value
2338            .get("manifest")
2339            .and_then(|manifest| manifest.get("provenance"))
2340        {
2341            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2342                return Ok(vec![control_error_frame(
2343                    &frame,
2344                    "invalid_manifest",
2345                    format!("malformed manifest provenance: {err}"),
2346                )?]);
2347            }
2348        }
2349        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2350            let module_id = hello_value["manifest"]["module_id"]
2351                .as_str()
2352                .unwrap_or("<unknown>");
2353            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2354        }
2355        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2356            Ok(hello) => hello,
2357            Err(err) => {
2358                return Ok(vec![control_error_frame(
2359                    &frame,
2360                    "invalid_hello",
2361                    format!("malformed HELLO body: {err}"),
2362                )?])
2363            }
2364        };
2365
2366        if hello.protocol_ver != hello.manifest.protocol_ver {
2367            return Ok(vec![control_error_frame(
2368                &frame,
2369                "invalid_manifest",
2370                format!(
2371                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2372                    hello.protocol_ver, hello.manifest.protocol_ver
2373                ),
2374            )?]);
2375        }
2376
2377        if hello.manifest.module_id.trim().is_empty() {
2378            return Ok(vec![control_error_frame(
2379                &frame,
2380                "invalid_manifest",
2381                "manifest module_id must not be empty",
2382            )?]);
2383        }
2384
2385        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2386            Ok(negotiated_ver) => negotiated_ver,
2387            Err(message) => {
2388                return Ok(vec![control_error_frame(
2389                    &frame,
2390                    "version_unsupported",
2391                    message,
2392                )?])
2393            }
2394        };
2395
2396        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2397        // swap is open for this id, the only HELLO admitted as a second process
2398        // is the one carrying the candidate's launch nonce (the swap token), and
2399        // it registers into the candidate slot rather than being refused as a
2400        // duplicate. Run after the reserved gate, a reserved module's candidate
2401        // would be refused `reserved_module` for presenting a nonce that gate
2402        // does not know. See `SupervisorHandle::swap_hello_admission`.
2403        let swap_admission = self
2404            .supervisor
2405            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2406        if swap_admission == SwapHelloAdmission::Refused {
2407            warn!(
2408                module_id = %hello.manifest.module_id,
2409                connection_id = connection_id.get(),
2410                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2411            );
2412            return Ok(vec![control_error_frame(
2413                &frame,
2414                "swap_token_invalid",
2415                format!(
2416                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2417                    hello.manifest.module_id
2418                ),
2419            )?]);
2420        }
2421        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2422
2423        // A swap candidate proves its identity with the candidate nonce above.
2424        // Other registrations receive this catalog fact only when reserved-module
2425        // policy required and accepted a nonce; unreserved registrations require
2426        // no nonce check.
2427        let nonce_verified = if swap_candidate {
2428            true
2429        } else {
2430            match self.supervisor.reserved_hello_verification(
2431                &hello.manifest.module_id,
2432                hello.launch_nonce.as_deref(),
2433            ) {
2434                Ok(verified) => verified,
2435                Err(rejection) => {
2436                    let message = match rejection {
2437                        ReservedHelloRejection::Exact { module_id } => format!(
2438                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2439                        ),
2440                        ReservedHelloRejection::Prefix {
2441                            prefix,
2442                            owner_module_id,
2443                        } => format!(
2444                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2445                            hello.manifest.module_id
2446                        ),
2447                    };
2448                    return Ok(vec![control_error_frame(
2449                        &frame,
2450                        "reserved_module",
2451                        message,
2452                    )?]);
2453                }
2454            }
2455        };
2456
2457        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2458            &hello.manifest.module_id,
2459            hello.manifest.capabilities.as_ref(),
2460        );
2461        if let Some(refusal) = reserved_capability_refusals.first() {
2462            let capability = refusal.capability.clone();
2463            let bound_module = refusal.claimants[0].clone();
2464            log_duplicate_claim_events(reserved_capability_refusals);
2465            return Ok(vec![control_error_frame(
2466                &frame,
2467                "reserved_capability",
2468                format!(
2469                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2470                    capability, bound_module, hello.manifest.module_id
2471                ),
2472            )?]);
2473        }
2474
2475        // A connection that already opened client routes must not also register as
2476        // a module: cleanup would then release only one side and leak the other.
2477        if self
2478            .forwarding
2479            .connection_has_client_routes(connection_id)
2480            .map_err(RouterError::Forwarding)?
2481        {
2482            return Ok(vec![control_error_frame(
2483                &frame,
2484                "invalid_hello",
2485                "connection has open client routes and cannot also register as a module",
2486            )?]);
2487        }
2488
2489        // Kept for scope sync authority, which goes only to the connection that
2490        // presented the module's current launch nonce. Recorded before the
2491        // registration is attempted: a connection whose registration then fails
2492        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2493        self.hello_launch_nonces
2494            .lock()
2495            .unwrap_or_else(|poisoned| poisoned.into_inner())
2496            .record(connection_id, hello.launch_nonce.as_deref());
2497        let control_ops = effective_module_control_ops(hello.control_ops);
2498        // Built before anything is registered so an encoding failure leaves no
2499        // registry or forwarding state behind.
2500        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2501        if swap_candidate {
2502            return self.register_swap_candidate(
2503                connection_id,
2504                sink,
2505                &frame,
2506                hello.manifest,
2507                negotiated_ver,
2508                control_ops,
2509                hello_ack,
2510            );
2511        }
2512        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2513        // is refused, but `catalog.list` does not show the module and the
2514        // catalog generation does not move until it is published below, once
2515        // its forwarding endpoint exists. Listing it now would advertise a
2516        // module that `route.open` still refuses for having no forwarding
2517        // connection.
2518        let registration = match self
2519            .registry
2520            .register_unpublished_verified_with_control_ops(
2521                hello.manifest,
2522                negotiated_ver,
2523                connection_id,
2524                control_ops,
2525                nonce_verified,
2526            ) {
2527            Ok(registration) => registration,
2528            Err(RegistryError::DuplicateModuleId { module_id }) => {
2529                return Ok(vec![control_error_frame(
2530                    &frame,
2531                    "duplicate_module_id",
2532                    format!(
2533                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2534                    ),
2535                )?])
2536            }
2537            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2538                return Ok(vec![control_error_frame(
2539                    &frame,
2540                    "invalid_module_id",
2541                    err.to_string(),
2542                )?])
2543            }
2544            Err(err) => {
2545                return Ok(vec![control_error_frame(
2546                    &frame,
2547                    "registry_error",
2548                    err.to_string(),
2549                )?])
2550            }
2551        };
2552        #[cfg(test)]
2553        self.hold_between_registry_and_forwarding();
2554
2555        let reply = if let Some(sink) = sink {
2556            // The forwarding table's module store is also the daemon-to-module
2557            // control-RPC lane, so every HELLO gets a live endpoint even when the
2558            // manifest has no routable provider role. Non-routable modules still
2559            // cannot receive route.bind in production: `handle_route_open` checks
2560            // the registry manifest with `target_has_required_role` before the
2561            // only production call to `begin_route_bind_relay_for` below that
2562            // route.open path. The remaining direct relay callers are unit tests
2563            // and benchmark harnesses that construct forwarding state explicitly.
2564            //
2565            // The HELLO_ACK is queued by the forwarding table itself, before the
2566            // endpoint becomes visible, and is NOT returned as a reply. A module
2567            // reads HELLO_ACK first and exits on anything else; a reply is only
2568            // written after this handler returns, by which time a route.open on
2569            // another connection could already have queued a route.bind request
2570            // for this module ahead of it.
2571            let concurrency = manifest_concurrency(&registration.manifest);
2572            if let Err(err) = self.forwarding.register_module_connection_acked(
2573                connection_id,
2574                registration.manifest.module_id.clone(),
2575                negotiated_ver,
2576                concurrency,
2577                sink,
2578                hello_ack,
2579            ) {
2580                // Forwarding registration failed, so there is no forwarding
2581                // state to tear down. Remove the registry entry, which was never
2582                // published and so never listed or counted in the catalog
2583                // generation, and signal the release watch directly.
2584                if matches!(
2585                    self.deregister_connection(
2586                        connection_id,
2587                        RegistrationEndReason::RegistrationFailed,
2588                    ),
2589                    Ok(r) if !r.is_empty()
2590                ) {
2591                    crate::supervise::notify_registration_release();
2592                }
2593                return Ok(vec![control_error_frame(
2594                    &frame,
2595                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2596                        "invalid_hello"
2597                    } else {
2598                        forwarding_error_code(&err)
2599                    },
2600                    err.to_string(),
2601                )?]);
2602            }
2603            Vec::new()
2604        } else {
2605            // No sink means no forwarding endpoint, so nothing can be routed
2606            // ahead of the ack; it goes out as the reply.
2607            vec![hello_ack]
2608        };
2609
2610        // The module is routable now (or, without a sink, as routable as it
2611        // will ever be), so the catalog may list it. Publishing bumps the
2612        // catalog generation in the same registry critical section that makes
2613        // the module listed.
2614        if !self
2615            .registry
2616            .publish_registration(connection_id)
2617            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2618        {
2619            debug!(
2620                module_id = %registration.manifest.module_id,
2621                connection_id = connection_id.get(),
2622                "registration was removed before it could be published"
2623            );
2624        }
2625
2626        // Exposure over assumption: Concurrency's serde default is pinned to the
2627        // pre-field behavior (ModuleManaged), so a management surface that is
2628        // genuinely Serial and just never declared it inherits concurrent
2629        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2630        // turns "no module has been bitten yet" into the checkable claim "no
2631        // module is exposed" -- one read of the boot log instead of a fleet
2632        // audit. Detected from the raw HELLO bytes because the serde default
2633        // deliberately erases the absent/declared distinction from the type.
2634        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2635            info!(
2636                module_id = %registration.manifest.module_id,
2637                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2638            );
2639        }
2640
2641        self.apply_registration_capabilities(&registration);
2642
2643        self.supervisor
2644            .record_activity_registration(&registration.manifest.module_id);
2645
2646        info!(
2647            module_id = %registration.manifest.module_id,
2648            module_version = %registration.manifest.module_version,
2649            negotiated_ver,
2650            routable_provider = manifest_provides_routable_role(&registration.manifest),
2651            connection_id = connection_id.get(),
2652            "module registered"
2653        );
2654
2655        Ok(reply)
2656    }
2657
2658    /// Register a HELLO the swap gate admitted into the candidate slot of the
2659    /// registry and of forwarding, where it is reachable over its own
2660    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2661    /// nothing routes to it until the supervisor cuts over.
2662    ///
2663    /// Registry first (unpublished), then forwarding, then publication, the
2664    /// same order as an ordinary HELLO; a forwarding failure removes the
2665    /// registry entry again. The swap supervisor treats a published, ready
2666    /// candidate as registered and promotes it, and promotion needs the
2667    /// candidate's forwarding endpoint, so the candidate is published only
2668    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2669    /// recompute capability verdicts or run the capability deny census
2670    /// (`enforce_capability_denies`): both are about the module in the active
2671    /// slot, which `route.open` can target. A candidate sits in the candidate
2672    /// slot, so it neither provides capabilities to other modules nor can be
2673    /// routed to until promotion moves it into the active slot.
2674    #[allow(clippy::too_many_arguments)]
2675    fn register_swap_candidate(
2676        &self,
2677        connection_id: ConnectionId,
2678        sink: Option<crate::FrameSink>,
2679        frame: &Frame,
2680        manifest: ModuleManifest,
2681        negotiated_ver: u8,
2682        control_ops: Vec<String>,
2683        hello_ack: Frame,
2684    ) -> Result<Vec<Frame>, RouterError> {
2685        let module_id = manifest.module_id.clone();
2686        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2687            manifest,
2688            negotiated_ver,
2689            connection_id,
2690            control_ops,
2691        ) {
2692            Ok(registration) => registration,
2693            Err(RegistryError::DuplicateModuleId { module_id }) => {
2694                return Ok(vec![control_error_frame(
2695                    frame,
2696                    "duplicate_module_id",
2697                    format!(
2698                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2699                    ),
2700                )?])
2701            }
2702            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2703                return Ok(vec![control_error_frame(
2704                    frame,
2705                    "invalid_module_id",
2706                    err.to_string(),
2707                )?])
2708            }
2709            Err(err) => {
2710                return Ok(vec![control_error_frame(
2711                    frame,
2712                    "registry_error",
2713                    err.to_string(),
2714                )?])
2715            }
2716        };
2717        #[cfg(test)]
2718        self.hold_between_registry_and_forwarding();
2719        let reply = if let Some(sink) = sink {
2720            // Same ordering as an ordinary HELLO: the forwarding table queues
2721            // the HELLO_ACK before the candidate endpoint is inserted, because
2722            // a module exits if its first frame after HELLO is anything else.
2723            let concurrency = manifest_concurrency(&registration.manifest);
2724            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2725                connection_id,
2726                module_id.clone(),
2727                negotiated_ver,
2728                concurrency,
2729                sink,
2730                hello_ack,
2731            ) {
2732                if matches!(
2733                    self.deregister_connection(
2734                        connection_id,
2735                        RegistrationEndReason::RegistrationFailed,
2736                    ),
2737                    Ok(r) if !r.is_empty()
2738                ) {
2739                    crate::supervise::notify_registration_release();
2740                }
2741                return Ok(vec![control_error_frame(
2742                    frame,
2743                    forwarding_error_code(&err),
2744                    err.to_string(),
2745                )?]);
2746            }
2747            Vec::new()
2748        } else {
2749            vec![hello_ack]
2750        };
2751        if !self
2752            .registry
2753            .publish_registration(connection_id)
2754            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2755        {
2756            debug!(
2757                module_id = %module_id,
2758                connection_id = connection_id.get(),
2759                "swap candidate registration was removed before it could be published"
2760            );
2761        }
2762        self.supervisor.mark_swap_candidate_admitted(&module_id);
2763        info!(
2764            module_id = %module_id,
2765            module_version = %registration.manifest.module_version,
2766            negotiated_ver,
2767            ready = registration.ready,
2768            connection_id = connection_id.get(),
2769            "swap candidate registered; not routable until cutover"
2770        );
2771        Ok(reply)
2772    }
2773
2774    fn build_hello_ack(
2775        &self,
2776        frame: &Frame,
2777        negotiated_ver: u8,
2778        module_id: &str,
2779    ) -> Result<Frame, RouterError> {
2780        let ack = ModuleHelloAckBody {
2781            negotiated_ver,
2782            subc_ops: module_subc_ops(),
2783            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2784            storage: self
2785                .storage_config
2786                .as_ref()
2787                .map(|cfg| cfg.descriptor_for(module_id)),
2788            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2789        };
2790        let body = serde_json::to_vec(&ack).map_err(|err| {
2791            RouterError::backend(
2792                0,
2793                frame.header.corr,
2794                format!("failed to encode HELLO_ACK: {err}"),
2795            )
2796        })?;
2797
2798        Frame::build_with_version(
2799            negotiated_ver,
2800            FrameType::HelloAck,
2801            control_flags(),
2802            0,
2803            0,
2804            frame.header.corr,
2805            body,
2806        )
2807        .map_err(RouterError::FrameBuild)
2808    }
2809
2810    async fn handle_client_control_request(
2811        &self,
2812        ctx: &RouteCtx,
2813        frame: Frame,
2814        request: ClientControlRequest,
2815        route_open_timings: &mut Option<RouteOpenTimings>,
2816    ) -> Result<Vec<Frame>, RouterError> {
2817        if let Some(operation) = privileged_operation(&request) {
2818            let caller = self.resolve_caller(ctx.connection_id).await;
2819            let op = client_control_request_op(&request);
2820            let text = prompt_text(operation, &caller);
2821            match self.privileged_operator_prompt {
2822                PrivilegedOperatorPrompt::Log => {
2823                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2824                        target = ?operation, caller = %caller.description(), summary = %text,
2825                        "privileged request would prompt the operator");
2826                }
2827                PrivilegedOperatorPrompt::Enforce => {
2828                    let key =
2829                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2830                    let outcome = self
2831                        .forwarding
2832                        .operator_confirms()
2833                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2834                        .await;
2835                    if outcome.code != "confirmed" {
2836                        return Ok(vec![outcome.frame(&frame)]);
2837                    }
2838                }
2839            }
2840        }
2841        match request {
2842            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2843            ClientControlRequest::CatalogList { module_id } => {
2844                self.handle_catalog_list(frame, module_id)
2845            }
2846            ClientControlRequest::RouteOpen {
2847                target,
2848                identity,
2849                consumer_identity,
2850                consumer_capabilities,
2851                role_versions,
2852                admission_facts,
2853                scope,
2854            } => {
2855                self.handle_route_open(
2856                    ctx,
2857                    frame,
2858                    RouteOpenRequest {
2859                        target,
2860                        identity,
2861                        consumer_identity,
2862                        consumer_capabilities,
2863                        role_versions,
2864                        admission_facts,
2865                        scope,
2866                    },
2867                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2868                )
2869                .await
2870            }
2871            ClientControlRequest::RoutePoll {
2872                route_channel,
2873                route_epoch,
2874                kind,
2875            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2876            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2877            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2878                self.handle_supervisor_spawn_snapshot(frame)
2879            }
2880            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2881                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2882            }
2883            ClientControlRequest::SupervisorRestart {
2884                module_id,
2885                drain_timeout_ms,
2886            } => {
2887                self.log_supervisor_request_received(
2888                    ctx,
2889                    frame.header.corr,
2890                    ops::SUPERVISOR_RESTART,
2891                    Some(&module_id),
2892                    None,
2893                )?;
2894                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2895                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2896                    .await
2897            }
2898            ClientControlRequest::SupervisorSwap {
2899                module_id,
2900                ready_timeout_ms,
2901            } => {
2902                self.log_supervisor_request_received(
2903                    ctx,
2904                    frame.header.corr,
2905                    ops::SUPERVISOR_SWAP,
2906                    Some(&module_id),
2907                    None,
2908                )?;
2909                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2910                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2911                    .await
2912            }
2913            ClientControlRequest::SupervisorReload { module_id } => {
2914                self.log_supervisor_request_received(
2915                    ctx,
2916                    frame.header.corr,
2917                    ops::SUPERVISOR_RELOAD,
2918                    Some(&module_id),
2919                    None,
2920                )?;
2921                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2922                self.handle_supervisor_reload(frame, module_id, requester)
2923                    .await
2924            }
2925            ClientControlRequest::SupervisorRescan { preview } => {
2926                if !preview {
2927                    self.log_supervisor_request_received(
2928                        ctx,
2929                        frame.header.corr,
2930                        ops::SUPERVISOR_RESCAN,
2931                        None,
2932                        None,
2933                    )?;
2934                }
2935                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2936                self.handle_supervisor_rescan(
2937                    frame,
2938                    if preview {
2939                        RescanMode::Preview
2940                    } else {
2941                        RescanMode::Execute
2942                    },
2943                    requester,
2944                )
2945                .await
2946            }
2947            ClientControlRequest::SupervisorRescanChecked {
2948                preview,
2949                candidate,
2950                expect,
2951            } => {
2952                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2953                    .await
2954            }
2955            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2956                self.log_supervisor_request_received(
2957                    ctx,
2958                    frame.header.corr,
2959                    ops::SUPERVISOR_RELEASE_RESERVED,
2960                    Some(&module_id),
2961                    None,
2962                )?;
2963                self.handle_supervisor_release_reserved(frame, module_id)
2964                    .await
2965            }
2966            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2967                self.log_supervisor_request_received(
2968                    ctx,
2969                    frame.header.corr,
2970                    ops::SUPERVISOR_SET_ENABLED,
2971                    Some(&module_id),
2972                    Some(enabled),
2973                )?;
2974                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2975                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2976                    .await
2977            }
2978            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2979                self.handle_supervisor_health_probe(frame, module_id).await
2980            }
2981            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2982            ClientControlRequest::SupervisorRoutes { module_id } => {
2983                self.handle_supervisor_routes(frame, module_id)
2984            }
2985            ClientControlRequest::SupervisorProvenance { module_id } => {
2986                self.handle_supervisor_provenance(frame, module_id).await
2987            }
2988            ClientControlRequest::SupervisorStderrTail {
2989                module_id,
2990                max_lines,
2991                max_bytes,
2992            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2993            ClientControlRequest::SupervisorTerminals { module_id } => {
2994                self.handle_supervisor_terminals(frame, module_id).await
2995            }
2996        }
2997    }
2998
2999    fn log_supervisor_request_received(
3000        &self,
3001        ctx: &RouteCtx,
3002        corr: u64,
3003        op: &'static str,
3004        module_id: Option<&str>,
3005        enabled: Option<bool>,
3006    ) -> Result<(), RouterError> {
3007        let caller = self.supervisor_requester(ctx, corr)?;
3008        let caller = principal_label(&caller);
3009        // `Option` fields are recorded only when present, so a request with no
3010        // module id (rescan) or no enabled flag simply omits that field.
3011        info!(
3012            target: "control",
3013            op,
3014            module_id,
3015            enabled,
3016            connection_id = ctx.connection_id.get(),
3017            caller = %caller,
3018            "supervisor request received"
3019        );
3020        Ok(())
3021    }
3022
3023    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3024        Ok(self
3025            .registry
3026            .get_module_by_connection(ctx.connection_id)
3027            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3028            .map(|registration| Principal::Reserved {
3029                module_id: registration.manifest.module_id,
3030            })
3031            .unwrap_or(Principal::Direct))
3032    }
3033
3034    fn handle_module_control_request(
3035        &self,
3036        connection_id: ConnectionId,
3037        frame: Frame,
3038        request: ModuleControlRequestFromModule,
3039    ) -> Result<Vec<Frame>, RouterError> {
3040        match request {
3041            ModuleControlRequestFromModule::CatalogUpdate {
3042                provides,
3043                capabilities,
3044                ready,
3045            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3046            ModuleControlRequestFromModule::LiveRoots {} => {
3047                let registered = self
3048                    .registry
3049                    .get_module_by_connection(connection_id)
3050                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3051                let Some(registration) = registered else {
3052                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3053                };
3054                let response = self
3055                    .forwarding
3056                    .live_roots(&registration.manifest.module_id)
3057                    .map_err(RouterError::Forwarding)?;
3058                Ok(vec![control_response_body_frame(
3059                    &frame,
3060                    &response,
3061                    "ModuleControlResponseToModule::LiveRoots",
3062                )?])
3063            }
3064            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3065                self.handle_scope_sync(connection_id, frame, generation, scopes)
3066            }
3067            ModuleControlRequestFromModule::ScopeApply {
3068                generation,
3069                upsert,
3070                end,
3071            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3072            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3073                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3074            }
3075        }
3076    }
3077
3078    fn handle_scope_served_by(
3079        &self,
3080        connection_id: ConnectionId,
3081        frame: Frame,
3082    ) -> Result<Vec<Frame>, RouterError> {
3083        let registered = self
3084            .registry
3085            .get_module_by_connection(connection_id)
3086            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3087        let Some(registered) = registered else {
3088            return Ok(vec![control_error_frame(
3089                &frame,
3090                "not_registered",
3091                "scope.served_by requires an active module registration owned by this connection",
3092            )?]);
3093        };
3094        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3095        {
3096            Ok(request) => request,
3097            Err(err) => {
3098                return Ok(vec![control_error_frame(
3099                    &frame,
3100                    "invalid_control_body",
3101                    format!("malformed scope.served_by body: {err}"),
3102                )?])
3103            }
3104        };
3105        let answer = self
3106            .scopes
3107            .read()
3108            .unwrap_or_else(|poisoned| poisoned.into_inner())
3109            .served_by(
3110                &registered.manifest.module_id,
3111                &request.scope_ref,
3112                request.scope_epoch,
3113                &request.module_id,
3114            );
3115        let reply = subc_control::ScopeServedByReply::new(
3116            answer,
3117            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3118        );
3119        Ok(vec![control_response_body_frame(
3120            &frame,
3121            &reply,
3122            "ScopeServedByReply",
3123        )?])
3124    }
3125
3126    fn handle_operator_confirm(
3127        &self,
3128        ctx: &RouteCtx,
3129        frame: Frame,
3130    ) -> Result<Vec<Frame>, RouterError> {
3131        use crate::operator_confirm::{audit, Outcome};
3132        let registration = self
3133            .registry
3134            .get_module_by_connection(ctx.connection_id)
3135            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3136        let Some(registration) = registration else {
3137            let outcome = Outcome::refusal("not_registered");
3138            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3139            return Ok(vec![outcome.frame(&frame)]);
3140        };
3141        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3142            Ok(request) => request,
3143            Err(_) => {
3144                let outcome = Outcome::refusal("invalid_control_body");
3145                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3146                return Ok(vec![outcome.frame(&frame)]);
3147            }
3148        };
3149        let module_id = registration.manifest.module_id;
3150        // Read the launch nonce (under its own lock) before taking the forwarding
3151        // table's lock below: holding forwarding while waiting on another daemon
3152        // lock risks a lock-order deadlock with paths that take them the other way.
3153        let nonce = self
3154            .hello_launch_nonces
3155            .lock()
3156            .unwrap_or_else(|p| p.into_inner())
3157            .nonce(ctx.connection_id)
3158            .map(str::to_owned);
3159        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3160            self.supervisor
3161                .spawned_consumer_authorized(&module_id, nonce)
3162        });
3163        let confirms = self.forwarding.operator_confirms();
3164        self.forwarding
3165            .with_operator_route(
3166                ctx.connection_id,
3167                request.route_channel,
3168                request.route_epoch,
3169                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3170            )
3171            .map_err(RouterError::Forwarding)
3172    }
3173
3174    /// Resolve folders before taking the scope-table lock: a slow filesystem
3175    /// must not hold up admissions for unrelated scoped routes.
3176    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3177        let mut scopes = ScopeRecords::from(scopes);
3178        for (index, record) in scopes.records.iter_mut().enumerate() {
3179            if let Some(refusal) = Self::canonicalize_workdir(record) {
3180                scopes.refusals.insert(index, refusal);
3181            }
3182        }
3183        scopes
3184    }
3185
3186    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3187        let workdir = record.workdir.as_mut()?;
3188        let path = std::path::Path::new(workdir);
3189        if !path.is_absolute() {
3190            return Some((
3191                error_codes::INVALID_WORKDIR,
3192                "workdir must be absolute".to_string(),
3193            ));
3194        }
3195        match ProjectRootId::from_path_allowing_missing(path) {
3196            Ok(root) => match root.as_path().to_str() {
3197                Some(canonical) => {
3198                    *workdir = canonical.to_string();
3199                    None
3200                }
3201                None => Some((
3202                    error_codes::INVALID_WORKDIR,
3203                    "canonical workdir is not UTF-8".to_string(),
3204                )),
3205            },
3206            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3207        }
3208    }
3209
3210    /// `scope.sync`: the owner is the module registered on this connection.
3211    /// A connection with no registration (every client connection, `direct`
3212    /// included) is refused `not_registered` before the table is consulted.
3213    fn handle_scope_sync(
3214        &self,
3215        connection_id: ConnectionId,
3216        frame: Frame,
3217        generation: u64,
3218        scopes: Vec<ScopeRecord>,
3219    ) -> Result<Vec<Frame>, RouterError> {
3220        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3221    }
3222
3223    fn handle_scope_change(
3224        &self,
3225        connection_id: ConnectionId,
3226        frame: Frame,
3227        generation: u64,
3228        scopes: Vec<ScopeRecord>,
3229        end: Option<Vec<ScopeEnd>>,
3230    ) -> Result<Vec<Frame>, RouterError> {
3231        let is_apply = end.is_some();
3232        let Some(registration) = self
3233            .registry
3234            .get_module_by_connection(connection_id)
3235            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3236        else {
3237            return Ok(vec![control_error_frame(
3238                &frame,
3239                "not_registered",
3240                "scope.sync and scope.apply require an active module registration owned by this connection",
3241            )?]);
3242        };
3243        let owner = registration.manifest.module_id;
3244        let scopes = Self::prepare_scope_records(scopes);
3245        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3246        let is_current_launch = |connection: ConnectionId| {
3247            self.hello_launch_nonces
3248                .lock()
3249                .unwrap_or_else(|poisoned| poisoned.into_inner())
3250                .presented(connection, current_nonce.as_deref())
3251        };
3252        // Lock order is the scope table, then the forwarding table: the new
3253        // tags are published, and the routes the change closes are selected,
3254        // while the scope table is still write-locked, so no admission can read
3255        // a record whose tag is not yet published.
3256        let mut table = self
3257            .scopes
3258            .write()
3259            .unwrap_or_else(|poisoned| poisoned.into_inner());
3260        let outcome = match end {
3261            Some(end) => table.apply(
3262                &owner,
3263                connection_id,
3264                is_current_launch,
3265                generation,
3266                scopes,
3267                end,
3268            ),
3269            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3270        };
3271        let drained = match &outcome {
3272            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3273            Err(_) => Vec::new(),
3274        };
3275        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3276        drop(table);
3277        if has_deadlines {
3278            self.scope_deadline_added.notify_one();
3279        }
3280        match outcome {
3281            Ok(applied) => {
3282                let counts = ScopeOutcomeCounts::of(&applied.results);
3283                info!(
3284                    owner = %owner,
3285                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3286                    generation,
3287                    records = applied.results.len(),
3288                    created = counts.created,
3289                    replaced = counts.replaced,
3290                    updated = counts.updated,
3291                    unchanged = counts.unchanged,
3292                    refused = counts.refused,
3293                    ended = applied.ended.len(),
3294                    tag_changes = applied.tag_changes.len(),
3295                    routes_closed = drained.len(),
3296                    "scope change accepted"
3297                );
3298                // An accepted scope change can still refuse individual records, and the
3299                // owner is the only party that sees the reply. Name them here so
3300                // an operator can tell a refused session from a missing one
3301                // without the owner's logs. Capped so a call that refuses
3302                // thousands cannot flood the log; the count above is complete.
3303                for refused in applied
3304                    .results
3305                    .iter()
3306                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3307                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3308                {
3309                    warn!(
3310                        owner = %owner,
3311                        generation,
3312                        scope_ref = %refused.scope_ref,
3313                        scope_epoch = refused.scope_epoch,
3314                        code = refused.code.as_deref().unwrap_or(""),
3315                        "scope record refused"
3316                    );
3317                }
3318                self.close_scope_drained_routes(drained);
3319                let response = if is_apply {
3320                    ModuleControlResponseToModule::ScopeApply {
3321                        generation,
3322                        results: applied.results,
3323                        end_results: applied.end_results,
3324                        ended: applied.ended,
3325                    }
3326                } else {
3327                    ModuleControlResponseToModule::ScopeSync {
3328                        generation,
3329                        results: applied.results,
3330                        ended: applied.ended,
3331                    }
3332                };
3333                Ok(vec![control_response_body_frame(
3334                    &frame,
3335                    &response,
3336                    "scope change response",
3337                )?])
3338            }
3339            Err(refusal) => {
3340                info!(
3341                    owner = %owner,
3342                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3343                    generation,
3344                    code = refusal.code,
3345                    "scope change refused"
3346                );
3347                Ok(vec![control_error_frame(
3348                    &frame,
3349                    refusal.code,
3350                    refusal.message,
3351                )?])
3352            }
3353        }
3354    }
3355
3356    /// Tell both ends of each route a scope change closed. The module gets a
3357    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3358    /// the client's route handle. The client also gets `route.closed` with the
3359    /// scope reason, one push per module and reason, so it can tell a revoked
3360    /// route from an ordinary close and not reopen it.
3361    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3362        if drained.is_empty() {
3363            return;
3364        }
3365        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3366            BTreeMap::new();
3367        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3368        for route in drained {
3369            warn!(
3370                module_id = %route.module_id,
3371                reason = ?route.reason,
3372                client_connection_id = route.client.connection_id.get(),
3373                route_channel = route.client.channel,
3374                "closing route because its scope changed"
3375            );
3376            pushes
3377                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3378                .or_insert_with(|| (route.reason, Vec::new()))
3379                .1
3380                .push(EndpointRoute {
3381                    goodbye_target: route.client.clone(),
3382                    principal: Principal::Unverified,
3383                    bound_at: Instant::now(),
3384                    draining: false,
3385                    drain_reason: None,
3386                });
3387            goodbyes.push(route.module);
3388            goodbyes.push(route.client);
3389        }
3390        for ((module_id, _), (reason, routes)) in pushes {
3391            send_route_control_pushes(
3392                &self.forwarding,
3393                routes,
3394                ClientControlPush::RouteClosed {
3395                    module_id,
3396                    channels: Vec::new(),
3397                    reason,
3398                    drained: false,
3399                    abandoned: 0,
3400                    excluded_subscriptions: 0,
3401                    terminal: Some(false),
3402                },
3403            );
3404        }
3405        self.emit_route_goodbyes(goodbyes);
3406    }
3407
3408    /// `scope.describe`: any registered module may read any scope, because a
3409    /// provider must read the scope a route it serves is stamped with.
3410    fn handle_scope_describe(
3411        &self,
3412        connection_id: ConnectionId,
3413        frame: Frame,
3414        owner: Principal,
3415        scope_ref: String,
3416    ) -> Result<Vec<Frame>, RouterError> {
3417        let registered = self
3418            .registry
3419            .get_module_by_connection(connection_id)
3420            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3421        let Some(registered) = registered else {
3422            return Ok(vec![control_error_frame(
3423                &frame,
3424                "not_registered",
3425                "scope.describe requires an active module registration owned by this connection",
3426            )?]);
3427        };
3428        let mut description = self
3429            .scopes
3430            .read()
3431            .unwrap_or_else(|poisoned| poisoned.into_inner())
3432            .describe(&owner, &scope_ref);
3433        // Describe readers also decode strict ScopeAttributes. Attribution is
3434        // safe to omit unless the requesting module declares it understands it.
3435        if !registered
3436            .manifest
3437            .capabilities
3438            .as_ref()
3439            .is_some_and(|caps| {
3440                caps.provides
3441                    .iter()
3442                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3443            })
3444        {
3445            if let Some(stamp) = description.stamp.as_mut() {
3446                stamp.attributes.on_behalf_of_agent = None;
3447            }
3448        }
3449        let owner_configured = match &owner {
3450            // Ask whether the owner is configured (`is_configured`), not
3451            // whether it is on the roster (`get(..).is_some()`): a supervised
3452            // module's process can register and describe a scope before the
3453            // supervisor has put it on the roster.
3454            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3455            _ => false,
3456        };
3457        let response = ModuleControlResponseToModule::ScopeDescribe {
3458            status: description.status,
3459            scope_epoch: description.scope_epoch,
3460            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3461            owner_synced: description.owner_synced,
3462            owner_configured,
3463            scope: description.stamp,
3464        };
3465        Ok(vec![control_response_body_frame(
3466            &frame,
3467            &response,
3468            "ModuleControlResponseToModule::ScopeDescribe",
3469        )?])
3470    }
3471
3472    fn handle_catalog_update(
3473        &self,
3474        connection_id: ConnectionId,
3475        frame: Frame,
3476        provides: Vec<ProviderRole>,
3477        capabilities: Option<CapabilityDeclarations>,
3478        ready: Option<bool>,
3479    ) -> Result<Vec<Frame>, RouterError> {
3480        self.refresh_capability_requirements();
3481        let Some(registration) = self
3482            .registry
3483            .get_module_by_connection(connection_id)
3484            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3485        else {
3486            return Ok(vec![control_error_frame(
3487                &frame,
3488                "not_registered",
3489                "catalog.update requires an active module registration owned by this connection",
3490            )?]);
3491        };
3492
3493        if let Some(message) =
3494            catalog_update_frozen_field_message(&registration.manifest, &provides)
3495        {
3496            return Ok(vec![control_error_frame(
3497                &frame,
3498                "catalog_update_frozen_field",
3499                message,
3500            )?]);
3501        }
3502
3503        let mut candidate = registration.manifest.clone();
3504        candidate.provides = provides.clone();
3505        candidate.capabilities = capabilities
3506            .clone()
3507            .or_else(|| registration.manifest.capabilities.clone());
3508        if let Some(rich_ui) = &candidate.rich_ui {
3509            if let Err(err) = rich_ui.validate() {
3510                return Ok(vec![control_error_frame(
3511                    &frame,
3512                    "invalid_manifest",
3513                    err.to_string(),
3514                )?]);
3515            }
3516        }
3517        if let Err(err) = candidate.validate_capability_grammar() {
3518            return Ok(vec![control_error_frame(
3519                &frame,
3520                "invalid_capability_grammar",
3521                err.to_string(),
3522            )?]);
3523        }
3524
3525        // Updates must honor the same reserved owner as initial registration;
3526        // otherwise an empty HELLO could acquire the claim after admission.
3527        let mut conflicts = self
3528            .capability_evaluator
3529            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3530        if let Some(conflict) = conflicts.first() {
3531            let message = format!(
3532                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3533                conflict.capability, conflict.claimants[0], candidate.module_id
3534            );
3535            for conflict in &mut conflicts {
3536                conflict.source = DuplicateClaimSource::CatalogUpdate;
3537            }
3538            log_duplicate_claim_events(conflicts);
3539            return Ok(vec![control_error_frame(
3540                &frame,
3541                "reserved_capability",
3542                message,
3543            )?]);
3544        }
3545
3546        let updated = self
3547            .registry
3548            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3549            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3550        if updated.is_none() {
3551            return Ok(vec![control_error_frame(
3552                &frame,
3553                "not_registered",
3554                "catalog.update requires an active module registration owned by this connection",
3555            )?]);
3556        }
3557        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3558            log_duplicate_claim_events(
3559                self.capability_evaluator
3560                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3561            );
3562        }
3563        if capability_census_trigger(
3564            registration.manifest.capabilities.as_ref(),
3565            updated
3566                .as_ref()
3567                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3568        ) {
3569            self.enforce_capability_denies();
3570        }
3571        self.refresh_capability_requirements();
3572
3573        let response = ModuleControlResponseToModule::CatalogUpdate {};
3574        control_response_body_frame(
3575            &frame,
3576            &response,
3577            "ModuleControlResponseToModule::CatalogUpdate",
3578        )
3579        .map(|frame| vec![frame])
3580    }
3581
3582    pub(crate) fn with_activity_ledger_config(
3583        mut self,
3584        run_dir: &Path,
3585        incarnation: String,
3586        config: crate::activity::ActivityConfig,
3587    ) -> Self {
3588        self.activity = Some(crate::activity::ActivityLedger::open_config(
3589            run_dir.join("activity.jsonl"),
3590            incarnation,
3591            config,
3592        ));
3593        self.supervisor.set_activity(self.activity.clone());
3594        self
3595    }
3596
3597    pub(crate) async fn configure_activity(
3598        &self,
3599        config: crate::activity::ActivityConfig,
3600    ) -> Result<(), &'static str> {
3601        match &self.activity {
3602            Some(ledger) => ledger.configure(config).await,
3603            None => Ok(()),
3604        }
3605    }
3606
3607    fn cancel_activity_subscription(
3608        &self,
3609        ctx: &RouteCtx,
3610        frame: &Frame,
3611    ) -> Result<bool, RouterError> {
3612        let task = self
3613            .activity_streams
3614            .lock()
3615            .unwrap_or_else(|p| p.into_inner())
3616            .remove(&(ctx.connection_id, frame.header.corr));
3617        let Some(task) = task else {
3618            return Ok(false);
3619        };
3620        task.abort();
3621        let end = Frame::build_with_version(
3622            response_version(frame),
3623            FrameType::StreamEnd,
3624            control_flags(),
3625            0,
3626            0,
3627            frame.header.corr,
3628            Vec::new(),
3629        )
3630        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3631        let sink = ctx.egress.clone();
3632        tokio::spawn(async move {
3633            let _ = sink.send(end).await;
3634        });
3635        Ok(true)
3636    }
3637
3638    async fn handle_activity_read(
3639        &self,
3640        ctx: &RouteCtx,
3641        frame: Frame,
3642    ) -> Result<Vec<Frame>, RouterError> {
3643        use subc_control::{
3644            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3645            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3646        };
3647        #[derive(Deserialize)]
3648        struct Identity {
3649            #[serde(default)]
3650            consumer_identity: Option<ConsumerIdentity>,
3651        }
3652        // A registered module connection already identifies its caller. Ignore
3653        // consumer_identity there so a module cannot borrow a mirror's identity.
3654        // Only separate consumer connections use the Direct/nonce trust boundary.
3655        let principal = match self
3656            .forwarding
3657            .module_id_for_connection(ctx.connection_id)
3658            .map_err(RouterError::Forwarding)?
3659        {
3660            Some(module_id) => Ok(Principal::Reserved { module_id }),
3661            None => {
3662                let identity: Identity = match serde_json::from_slice(&frame.body) {
3663                    Ok(identity) => identity,
3664                    Err(_) => {
3665                        return Ok(vec![control_error_frame(
3666                            &frame,
3667                            ACTIVITY_MALFORMED,
3668                            "malformed activity read",
3669                        )?])
3670                    }
3671                };
3672                self.route_open_principal(&frame, identity.consumer_identity)?
3673            }
3674        };
3675        let admitted = matches!(principal, Ok(Principal::Direct))
3676            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3677        if !admitted {
3678            return Ok(vec![control_error_frame(
3679                &frame,
3680                ACTIVITY_READ_FORBIDDEN,
3681                "activity reads require an operator or trusted mirror principal",
3682            )?]);
3683        }
3684        let Some(ledger) = &self.activity else {
3685            return Ok(vec![control_error_frame(
3686                &frame,
3687                ACTIVITY_UNAVAILABLE,
3688                "activity ledger unavailable",
3689            )?]);
3690        };
3691        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3692            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3693        {
3694            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3695                Ok(request) => ledger.query(request).await,
3696                Err(_) => Err(ACTIVITY_MALFORMED),
3697            };
3698            return match result {
3699                Ok(reply) => Ok(vec![control_response_body_frame(
3700                    &frame,
3701                    &reply,
3702                    ops::ACTIVITY_QUERY,
3703                )?]),
3704                Err(code) => Ok(vec![control_error_frame(
3705                    &frame,
3706                    code,
3707                    "activity query refused",
3708                )?]),
3709            };
3710        }
3711        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3712            Ok(request) => request,
3713            Err(_) => {
3714                return Ok(vec![control_error_frame(
3715                    &frame,
3716                    ACTIVITY_MALFORMED,
3717                    "malformed activity subscription",
3718                )?])
3719            }
3720        };
3721        let mut subscription = match ledger.subscribe(request.since).await {
3722            Ok(subscription) => subscription,
3723            Err(code) => {
3724                return Ok(vec![control_error_frame(
3725                    &frame,
3726                    code,
3727                    "activity subscription refused",
3728                )?])
3729            }
3730        };
3731        let key = (ctx.connection_id, frame.header.corr);
3732        let sink = ctx.egress.clone();
3733        let mut streams = self
3734            .activity_streams
3735            .lock()
3736            .unwrap_or_else(|p| p.into_inner());
3737        streams.retain(|_, task| !task.is_finished());
3738        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3739            return Ok(vec![control_error_frame(
3740                &frame,
3741                "duplicate_subscription",
3742                "activity subscription correlation id already in use",
3743            )?]);
3744        }
3745        let task = tokio::spawn(async move {
3746            loop {
3747                let entry = if let Some(entry) = subscription.replay.pop_front() {
3748                    Some(entry)
3749                } else {
3750                    subscription.receiver.recv().await
3751                };
3752                let Some(entry) = entry else {
3753                    break;
3754                };
3755                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3756                    return;
3757                };
3758                let Ok(next) = Frame::build_with_version(
3759                    response_version(&frame),
3760                    FrameType::StreamData,
3761                    control_flags(),
3762                    0,
3763                    0,
3764                    frame.header.corr,
3765                    body,
3766                ) else {
3767                    return;
3768                };
3769                if sink.send(next).await.is_err() {
3770                    return;
3771                }
3772            }
3773            // Dropping a full subscriber's sender drains its accepted entries
3774            // before the terminal error, preserving their order on the wire.
3775            if let Some(code) = subscription.terminal_code() {
3776                if let Ok(end) =
3777                    control_error_frame(&frame, code, "activity subscriber fell behind")
3778                {
3779                    let _ = sink.send(end).await;
3780                }
3781            }
3782        });
3783        streams.insert(key, task.abort_handle());
3784        Ok(Vec::new())
3785    }
3786
3787    async fn handle_activity_record(
3788        &self,
3789        ctx: &RouteCtx,
3790        frame: Frame,
3791    ) -> Result<Vec<Frame>, RouterError> {
3792        use subc_control::{
3793            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3794            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3795        };
3796        let result = async {
3797            let module = self
3798                .forwarding
3799                .module_id_for_connection(ctx.connection_id)
3800                .ok()
3801                .flatten()
3802                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3803            let request: ActivityRecordRequest =
3804                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3805            crate::activity::validate(&module, &request)?;
3806            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3807            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3808            let at_ms = std::time::SystemTime::now()
3809                .duration_since(std::time::UNIX_EPOCH)
3810                .unwrap_or_default()
3811                .as_millis() as u64;
3812            ledger.record(module, request, actor, at_ms).await
3813        }
3814        .await;
3815        match result {
3816            Ok(entry) => Ok(vec![control_response_body_frame(
3817                &frame,
3818                &ActivityRecordResponse { seq: entry.seq },
3819                "activity.record",
3820            )?]),
3821            Err(code) => Ok(vec![control_error_frame(
3822                &frame,
3823                code,
3824                "activity recording refused",
3825            )?]),
3826        }
3827    }
3828
3829    fn activity_actor(
3830        &self,
3831        connection: ConnectionId,
3832        module: &str,
3833        on: Option<&subc_control::ActivityOn>,
3834    ) -> Result<subc_control::ActivityActor, &'static str> {
3835        use crate::forwarding::{DataRoute, DataRouteState};
3836        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3837        let principal = |id: &str| Principal::Reserved {
3838            module_id: id.to_owned(),
3839        };
3840        let mut actor = ActivityActor {
3841            principal: principal(module),
3842            scope_owner: None,
3843            agent_id: None,
3844            run_id: None,
3845            flow_id: None,
3846            project_root: None,
3847            workdir: None,
3848            project_id: None,
3849        };
3850        let scope = match on {
3851            None => return Ok(actor),
3852            Some(ActivityOn::Scope(scope)) => {
3853                let selector = subc_protocol::scope::ScopeSelector::new(
3854                    principal(module),
3855                    scope.scope_ref.clone(),
3856                    Some(scope.epoch),
3857                );
3858                self.scopes
3859                    .read()
3860                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3861                    .attribute(&principal(module), module, &selector, true)
3862                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3863            }
3864            Some(ActivityOn::Route(route)) => {
3865                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3866                    .forwarding
3867                    .lookup_data_route(connection, route.channel, route.epoch)
3868                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3869                else {
3870                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3871                };
3872                if binding.module_id != module {
3873                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3874                }
3875                actor.principal = binding.principal.clone();
3876                actor.project_root = binding
3877                    .project_root
3878                    .as_ref()
3879                    .map(|root| root.as_path().to_path_buf());
3880                let Some(bound) = &binding.scope else {
3881                    return Ok(actor);
3882                };
3883                let owner = principal(&bound.owner);
3884                let selector = subc_protocol::scope::ScopeSelector::new(
3885                    owner.clone(),
3886                    bound.scope_ref.clone(),
3887                    Some(bound.tag.scope_epoch),
3888                );
3889                let admitted = self
3890                    .scopes
3891                    .read()
3892                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3893                    .attribute(&owner, module, &selector, true)
3894                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3895                if admitted.tag != bound.tag {
3896                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3897                }
3898                admitted
3899            }
3900        };
3901        actor.scope_owner = Some(scope.stamp.owner);
3902        actor.agent_id = scope.stamp.attributes.agent_id;
3903        actor.run_id = scope.stamp.attributes.run_id;
3904        actor.flow_id = scope.stamp.attributes.flow_id;
3905        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3906        actor.project_id = scope.stamp.project_id;
3907        Ok(actor)
3908    }
3909
3910    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3911        self.refresh_capability_requirements();
3912        // A bare connection count is ambiguous between many clients holding a
3913        // route each and one client accumulating hundreds, so publish the
3914        // concentration alongside it. Route state is best-effort here: a
3915        // diagnostic endpoint must still answer if the forwarding lock is
3916        // contended.
3917        let mut counters = self.counters.snapshot();
3918        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3919            obj.insert(
3920                "activity_unparseable_lines".into(),
3921                ledger.read_errors().into(),
3922            );
3923            obj.insert(
3924                "activity_lifecycle_append_failures".into(),
3925                ledger.lifecycle_failures().into(),
3926            );
3927            obj.insert(
3928                "activity_started_digest_skips".into(),
3929                ledger.lifecycle_digest_skips().into(),
3930            );
3931        }
3932        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3933            self.forwarding.client_route_concentration(),
3934            counters.as_object_mut(),
3935        ) {
3936            obj.insert(
3937                "client_connections_with_routes".into(),
3938                connections_with_routes.into(),
3939            );
3940            obj.insert("max_routes_on_one_connection".into(), max.into());
3941        }
3942        // A module that is being fast-refused and a module that is fine look
3943        // identical from a client that retries and succeeds, so name the open
3944        // breakers here. This rides the existing free-form counters object
3945        // rather than a new wire field, so no sibling that deserializes
3946        // `ServerDescribe` has to be rebuilt to keep reading it.
3947        if let (Some(open_breakers), Some(obj)) = (
3948            self.route_bind_breakers.open_snapshot(),
3949            counters.as_object_mut(),
3950        ) {
3951            obj.insert("route_bind_breakers_open".into(), open_breakers);
3952        }
3953        let response = ClientControlResponse::ServerDescribe {
3954            protocol_ver: PROTOCOL_VERSION,
3955            subc_ops: subc_ops(),
3956            capabilities: self.describe_capabilities(),
3957            connected_clients: self.connected_clients.count(),
3958            counters: Some(counters),
3959            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3960            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3961            capability_requirements: self.capability_requirement_statuses(),
3962            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3963            config_path: self
3964                .rescan
3965                .as_ref()
3966                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3967        };
3968        Ok(vec![control_response_body_frame(
3969            &frame,
3970            &response,
3971            "ClientControlResponse::ServerDescribe",
3972        )?])
3973    }
3974
3975    fn handle_catalog_list(
3976        &self,
3977        frame: Frame,
3978        module_id: Option<String>,
3979    ) -> Result<Vec<Frame>, RouterError> {
3980        let (generation, modules) = self
3981            .registry
3982            .list_modules_with_nonce_verification()
3983            .map_err(|err| {
3984                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3985            })?;
3986        let entries = modules
3987            .into_iter()
3988            .filter(|(registration, _)| {
3989                module_id
3990                    .as_deref()
3991                    .map(|wanted| registration.manifest.module_id == wanted)
3992                    .unwrap_or(true)
3993            })
3994            .map(|(registration, nonce_verified)| {
3995                let not_ready = self.not_ready_reason(&registration);
3996                let roles = registration.manifest.provides;
3997                CatalogEntry::new(
3998                    registration.manifest.module_id,
3999                    roles,
4000                    registration.control_ops,
4001                )
4002                .with_ready(not_ready.is_none())
4003                .with_not_ready(not_ready)
4004                .with_module_version(Some(registration.manifest.module_version))
4005                .with_capabilities(registration.manifest.capabilities)
4006                .with_self_signals(registration.manifest.self_signals)
4007                .with_events(registration.manifest.events)
4008                .with_operation_delivery(registration.manifest.operation_delivery)
4009                .with_operation_payload(registration.manifest.operation_payload)
4010                .with_rich_ui(registration.manifest.rich_ui)
4011                .with_role_declarations(registration.manifest.role_declarations)
4012                .with_nonce_verified(nonce_verified)
4013            })
4014            .collect();
4015        let response = ClientControlResponse::CatalogList {
4016            generation,
4017            modules: entries,
4018            subc_ops: subc_ops(),
4019        };
4020        Ok(vec![control_response_body_frame(
4021            &frame,
4022            &response,
4023            "ClientControlResponse::CatalogList",
4024        )?])
4025    }
4026
4027    /// The principal a route.open's `consumer_identity` attests: `Direct`
4028    /// when it presents none, `Reserved` when its launch nonce verifies, and
4029    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4030    /// can run before any refusal check without changing which check refuses.
4031    fn attest_route_open_consumer(
4032        &self,
4033        consumer_identity: Option<&ConsumerIdentity>,
4034    ) -> Option<Principal> {
4035        let Some(consumer_identity) = consumer_identity else {
4036            return Some(Principal::Direct);
4037        };
4038        self.supervisor
4039            .spawned_consumer_authorized(
4040                &consumer_identity.module_id,
4041                &consumer_identity.launch_nonce,
4042            )
4043            .then(|| Principal::Reserved {
4044                module_id: consumer_identity.module_id.clone(),
4045            })
4046    }
4047
4048    /// The caller fields for a refusal that only has the raw request frame,
4049    /// such as the connection-level admission refusal sent before the request
4050    /// reaches [`Self::handle_route_open`].
4051    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4052        let Ok(ClientControlRequest::RouteOpen {
4053            identity,
4054            consumer_identity,
4055            scope,
4056            ..
4057        }) = parse_client_control_request(&frame.body)
4058        else {
4059            return RouteOpenCaller::unknown();
4060        };
4061        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4062        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4063    }
4064
4065    fn route_open_principal(
4066        &self,
4067        frame: &Frame,
4068        consumer_identity: Option<ConsumerIdentity>,
4069    ) -> Result<Result<Principal, Frame>, RouterError> {
4070        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4071        self.route_open_principal_attested(frame, consumer_identity, attested)
4072    }
4073
4074    /// [`Self::route_open_principal`] with `attested` already taken by
4075    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4076    /// open checks the launch nonce once.
4077    fn route_open_principal_attested(
4078        &self,
4079        frame: &Frame,
4080        consumer_identity: Option<ConsumerIdentity>,
4081        attested: Option<Principal>,
4082    ) -> Result<Result<Principal, Frame>, RouterError> {
4083        if let Some(principal) = attested {
4084            return Ok(Ok(principal));
4085        }
4086        let Some(consumer_identity) = consumer_identity else {
4087            return Ok(Ok(Principal::Direct));
4088        };
4089
4090        Ok(Err(control_error_frame(
4091            frame,
4092            "bad_consumer_identity",
4093            format!(
4094                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4095                consumer_identity.module_id
4096            ),
4097        )?))
4098    }
4099
4100    /// Ordinary `route.open` refusals go through here; admission and breaker
4101    /// refusals log separately with their capacity or breaker state. The daemon can
4102    /// attest which code it sent: without the event, a client's "the daemon
4103    /// refused me" and the daemon's own view could only be reconciled by
4104    /// argument. Malformed input (`invalid_project_root`) does not come here;
4105    /// rejecting a request that was never a valid open is not a refusal of one.
4106    #[allow(clippy::too_many_arguments)]
4107    fn route_open_refusal_frame(
4108        &self,
4109        ctx: &RouteCtx,
4110        caller: &RouteOpenCaller,
4111        frame: &Frame,
4112        module_id: &str,
4113        reason: &'static str,
4114        code: &'static str,
4115        message: impl Into<String>,
4116    ) -> Result<Frame, RouterError> {
4117        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4118        control_error_frame(frame, code, message.into())
4119    }
4120
4121    /// Refuse a `route.open` because the target module's bind-relay breaker is
4122    /// open, without attempting the relay.
4123    ///
4124    /// The wire code is `module_timeout`, which is the truth (the module has
4125    /// not been answering binds) and which both SDKs already classify as
4126    /// retryable with capped backoff. Reusing it is what keeps this change out
4127    /// of both SDKs; the daemon-side distinction lives in the counter key
4128    /// instead.
4129    ///
4130    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4131    /// While a breaker is open this fires on every open to that module, and the
4132    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4133    /// produced 261 lines about a single module inside 3000 lines of daemon
4134    /// log. The rare transitions are logged at warn/info instead and the volume
4135    /// is carried by the counter, so the evidence survives without the flood.
4136    /// The debug line keeps a per-refusal record reachable for whoever turns
4137    /// the level up.
4138    #[allow(clippy::too_many_arguments)]
4139    fn route_open_breaker_refusal_frame(
4140        &self,
4141        ctx: &RouteCtx,
4142        caller: &RouteOpenCaller,
4143        frame: &Frame,
4144        module_id: &str,
4145        consecutive_timeouts: u32,
4146        retry_in: Duration,
4147        probe_in_flight: bool,
4148    ) -> Result<Frame, RouterError> {
4149        self.counters
4150            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4151        debug!(
4152            target: "control",
4153            code = "module_timeout",
4154            module_id = ?module_id,
4155            connection_id = ctx.connection_id.get(),
4156            principal = %caller.principal,
4157            scope = %caller.scope,
4158            harness = %caller.harness,
4159            consecutive_timeouts,
4160            retry_in_ms = retry_in.as_millis() as u64,
4161            probe_in_flight,
4162            "route.open refused by open bind-relay breaker"
4163        );
4164        // Say what a caller can act on. An open bind-relay breaker means the
4165        // module timed out accepting several new routes in a row. The module
4166        // is still running and its established routes keep working; only new
4167        // route.open requests are refused until the cooldown ends and one
4168        // test route (the probe) gets through. A message that only counts
4169        // failed relays reads as "the module is down" to a worker that sees it.
4170        let detail = if probe_in_flight {
4171            "one test route is already being tried; retry once it settles".to_string()
4172        } else {
4173            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4174        };
4175        control_error_frame(
4176            frame,
4177            "module_timeout",
4178            format!(
4179                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4180                 timed out in a row); {detail}; its established routes are unaffected"
4181            ),
4182        )
4183    }
4184
4185    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4186    /// requester's bytes (an unknown target is whatever the client sent) and
4187    /// is Debug-formatted so control characters land in the log escaped
4188    /// rather than as terminal sequences for whoever tails it.
4189    ///
4190    /// `reason` names the check that refused, because one wire code has
4191    /// several senders: after a module registers, `target_unavailable` can
4192    /// come from a missing role, an inactive registration, a supervisor that
4193    /// has not marked the process live, a missing forwarding connection, or a
4194    /// failed relay, and a log that records only the code cannot say which of
4195    /// them fired. It is a static, daemon-chosen label per branch, so it is
4196    /// safe to print plainly and stays a closed set.
4197    fn observe_route_open_refusal(
4198        &self,
4199        ctx: &RouteCtx,
4200        caller: &RouteOpenCaller,
4201        module_id: &str,
4202        reason: &'static str,
4203        code: &'static str,
4204    ) {
4205        self.counters.increment_route_open_refused(code);
4206        info!(
4207            target: "control",
4208            code,
4209            reason,
4210            module_id = ?module_id,
4211            connection_id = ctx.connection_id.get(),
4212            principal = %caller.principal,
4213            scope = %caller.scope,
4214            harness = %caller.harness,
4215            "route.open refused"
4216        );
4217        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4218            self.route_outages.record_not_serving(module_id, reason);
4219        }
4220    }
4221
4222    /// Record an ACCEPTED route.open.
4223    ///
4224    /// Refusals have been logged and counted since the attestation work; accepts
4225    /// were invisible, so the daemon knew every principal it stamped and wrote
4226    /// none of them down. The party that attests the identity was the only party
4227    /// not recording it, which left a credential vault unable to name the sender
4228    /// of a call that reached it (claustrum #43) and left the launch-nonce
4229    /// concurrency question unanswerable from the outside.
4230    ///
4231    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4232    /// `code`/`module_id`/`connection_id` returns both directions of the same
4233    /// decision rather than two shapes a reader has to join by hand.
4234    ///
4235    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4236    /// and the difference carries information rather than being an
4237    /// inconsistency. This line is only reachable after a successful bind to a
4238    /// REGISTERED module, so the value has already passed HELLO validation
4239    /// including the path-hazard refusal and cannot contain control bytes. A
4240    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4241    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4242    ///
4243    /// Bare is also what every other daemon line already emits (`module
4244    /// registered`, `configured module supervised`). Shipping `?module_id` here
4245    /// made this instrument the only one in the file whose ids did not answer
4246    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4247    /// line whose whole purpose is being grepped beside its sibling.
4248    ///
4249    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4250    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4251    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4252    /// one are recorded identically. A test written against that harness passes
4253    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4254    /// green assertion that cannot fail. The same limit applies to the escaping
4255    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4256    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4257    /// Fencing either needs the real formatter, not the capture layer.
4258    ///
4259    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4260    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4261    /// to record. The ephemeral port would decay within minutes and answer only a
4262    /// live question. The identity question is instead answered by counting
4263    /// distinct live connections presenting one module's `consumer_identity` --
4264    /// "is anyone else holding this secret" rather than "is this the right
4265    /// process".
4266    fn observe_route_open_accept(
4267        &self,
4268        ctx: &RouteCtx,
4269        module_id: &str,
4270        principal: &str,
4271        scope: &str,
4272    ) {
4273        self.route_outages.record_accepted(module_id);
4274        self.counters.increment_route_open_accepted(principal);
4275        info!(
4276            target: "control",
4277            principal,
4278            module_id,
4279            connection_id = ctx.connection_id.get(),
4280            scope = %scope,
4281            "route.open accepted"
4282        );
4283    }
4284
4285    fn supervised_absent_route_open_refusal_frame(
4286        &self,
4287        ctx: &RouteCtx,
4288        caller: &RouteOpenCaller,
4289        frame: &Frame,
4290        module_id: &str,
4291        code: &'static str,
4292        status: &crate::supervise::ModuleStatus,
4293    ) -> Result<Frame, RouterError> {
4294        self.counters.increment_route_open_refused(code);
4295        info!(
4296            target: "control",
4297            code,
4298            reason = "supervised_not_registered",
4299            module_id = ?module_id,
4300            connection_id = ctx.connection_id.get(),
4301            principal = %caller.principal,
4302            scope = %caller.scope,
4303            harness = %caller.harness,
4304            state = %status.state,
4305            enabled = status.enabled,
4306            live = status.live,
4307            "route.open refused"
4308        );
4309        // A supervised module whose process has not registered is not
4310        // serving, whatever the reason; the supervisor knows this id, so it is
4311        // safe to track.
4312        self.route_outages
4313            .record_not_serving(module_id, "supervised_not_registered");
4314        let message = if let Some(pid) = status.waiting_for_previous_pid {
4315            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4316                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4317            })
4318        } else {
4319            format!(
4320                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4321                status.state, status.enabled, status.live
4322            )
4323        };
4324        control_error_frame(frame, code, message)
4325    }
4326
4327    async fn handle_route_open(
4328        &self,
4329        ctx: &RouteCtx,
4330        frame: Frame,
4331        request: RouteOpenRequest,
4332        timings: &mut RouteOpenTimings,
4333    ) -> Result<Vec<Frame>, RouterError> {
4334        let RouteOpenRequest {
4335            target,
4336            mut identity,
4337            consumer_identity,
4338            consumer_capabilities,
4339            role_versions,
4340            admission_facts,
4341            scope,
4342        } = request;
4343        let target_module_id = target_module_id(&target).to_string();
4344        timings.module_id.clone_from(&target_module_id);
4345        if self
4346            .registry
4347            .get_module_by_connection(ctx.connection_id)
4348            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4349            .is_some()
4350        {
4351            return Ok(vec![control_error_frame(
4352                &frame,
4353                "invalid_request",
4354                "module connections cannot open client routes",
4355            )?]);
4356        }
4357        debug!(
4358            connection_id = ctx.connection_id.get(),
4359            corr = frame.header.corr,
4360            module_id = %target_module_id,
4361            "handling route.open"
4362        );
4363
4364        // The caller's attested principal, requested scope and harness, for
4365        // every `route.open refused` line below. The launch nonce is checked
4366        // here, once, and the answer is reused where the principal is
4367        // enforced further down, so the order of refusals is unchanged.
4368        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4369        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4370
4371        // A malformed declaration is refused first, before anything about the
4372        // target is looked up: the same body would be refused against any
4373        // module, so the caller learns nothing by retrying or waiting. An empty
4374        // map declares nothing and travels as no field at all, so a provider
4375        // only ever sees a missing field or a non-empty one.
4376        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4377        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4378            self.observe_route_open_refusal(
4379                ctx,
4380                &caller,
4381                &target_module_id,
4382                "invalid_role_versions",
4383                error_codes::INVALID_REQUEST,
4384            );
4385            return Ok(vec![control_error_body_frame(
4386                &frame,
4387                ErrorBody {
4388                    code: error_codes::INVALID_REQUEST.to_string(),
4389                    message: error.to_string(),
4390                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4391                },
4392            )?]);
4393        }
4394
4395        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4396            if status.waiting_for_previous_pid.is_some() {
4397                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4398                    ctx,
4399                    &caller,
4400                    &frame,
4401                    &target_module_id,
4402                    "module_warming",
4403                    &status,
4404                )?]);
4405            }
4406        }
4407
4408        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4409        // opposite. Below, a caller learns whether a module is unregistered,
4410        // supervised-but-down (with state/enabled/live), or registered without the
4411        // requested role. Elsewhere that is an enumeration leak: a probe learning
4412        // the shape of a fleet it cannot otherwise see.
4413        //
4414        // It is not one here, and the reason is the ACCESS MODEL rather than
4415        // anything about these errors. Reaching route.open requires the
4416        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4417        // connection file, so any caller who completes it already runs as this
4418        // user -- and can read subc.jsonc for the module list and `ck module
4419        // status` for live state. The reply discloses nothing the caller cannot
4420        // read more easily from disk, while the precision is load-bearing:
4421        // `unknown_module` is retryable and a missing role is not.
4422        //
4423        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4424        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4425        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4426        let Some(registration) = self
4427            .registry
4428            .get_module(&target_module_id)
4429            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4430        else {
4431            if let Some((status, warming)) =
4432                self.supervisor_status(&target_module_id, frame.header.corr)?
4433            {
4434                // BEFORE the two availability codes below, because for a module
4435                // that speaks no subc wire both of them are false comfort: they
4436                // say "not right now" and are retried, and this module will
4437                // never register no matter how long the caller waits. The
4438                // absence here is the declaration being honoured, not a module
4439                // that is late.
4440                if status.protocol == ModuleProtocol::None {
4441                    return Ok(vec![self.route_open_refusal_frame(
4442                        ctx,
4443                        &caller,
4444                        &frame,
4445                        &target_module_id,
4446                        "protocol_none",
4447                        error_codes::MODULE_NO_PROTOCOL,
4448                        format!(
4449                            "module_id '{target_module_id}' is declared protocol: none; \
4450                             it speaks no subc wire and serves no routes"
4451                        ),
4452                    )?]);
4453                }
4454                let code = if warming {
4455                    "module_warming"
4456                } else {
4457                    "target_unavailable"
4458                };
4459                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4460                    ctx,
4461                    &caller,
4462                    &frame,
4463                    &target_module_id,
4464                    code,
4465                    &status,
4466                )?]);
4467            }
4468            if let Some(removed_ago_ms) =
4469                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4470            {
4471                return Ok(vec![self.route_open_refusal_frame(
4472                    ctx,
4473                    &caller,
4474                    &frame,
4475                    &target_module_id,
4476                    "removed",
4477                    error_codes::MODULE_REMOVED,
4478                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4479                )?]);
4480            }
4481            return Ok(vec![self.route_open_refusal_frame(
4482                ctx,
4483                &caller,
4484                &frame,
4485                &target_module_id,
4486                "not_registered",
4487                error_codes::UNKNOWN_MODULE,
4488                format!("module_id '{target_module_id}' is not registered"),
4489            )?]);
4490        };
4491
4492        // Best-effort only: registry readiness and forwarding reservation use
4493        // different locks, so a module can flip readiness between this read and
4494        // the relay. Modules must still tolerate an `on_bind` while not ready.
4495        if !registration.ready {
4496            self.counters
4497                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4498            info!(
4499                target: "control",
4500                code = error_codes::MODULE_WARMING,
4501                module_id = ?target_module_id,
4502                connection_id = ctx.connection_id.get(),
4503                principal = %caller.principal,
4504                scope = %caller.scope,
4505                harness = %caller.harness,
4506                reason = "declared_not_ready",
4507                "route.open refused"
4508            );
4509            // The module is registered but says it cannot take work, which is
4510            // an outage from the caller's side even though its process is up.
4511            self.route_outages
4512                .record_not_serving(&target_module_id, "declared_not_ready");
4513            return Ok(vec![control_error_body_frame(
4514                &frame,
4515                ErrorBody {
4516                    code: error_codes::MODULE_WARMING.to_string(),
4517                    message: format!(
4518                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4519                    ),
4520                    detail: Some(serde_json::json!({
4521                        "reason": "declared_not_ready"
4522                    })),
4523                },
4524            )?]);
4525        }
4526
4527        // Effective readiness, second half: a module that declares a capability
4528        // `need: required` is not routable while that capability has no
4529        // registered provider. It is enforced HERE, as a retryable routing
4530        // refusal, and deliberately not as spawn ordering or a boot block. The
4531        // module is still started and registered and can make its own calls;
4532        // spawn ordering is a promise that cannot be kept once a provider
4533        // crashes at runtime, and refusing to boot would stop the whole
4534        // machine, including the tools needed to fix its configuration.
4535        //
4536        // "Provided" is the evaluator's verdict, which counts a provider as
4537        // soon as it has REGISTERED, not once it is ready. Two modules that
4538        // require each other's capabilities are therefore both routable once
4539        // both register; counting readiness instead would deadlock them.
4540        //
4541        // Only new opens are refused. Routes already bound when a provider
4542        // goes away stay bound: nothing here tears them down, and the module
4543        // answers them as it can. Like the readiness read above this is
4544        // best-effort against a provider registering or leaving concurrently.
4545        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4546            self.counters
4547                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4548            info!(
4549                target: "control",
4550                code = error_codes::MODULE_WARMING,
4551                module_id = ?target_module_id,
4552                connection_id = ctx.connection_id.get(),
4553                principal = %caller.principal,
4554                scope = %caller.scope,
4555                harness = %caller.harness,
4556                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4557                capability = %capability,
4558                "route.open refused"
4559            );
4560            return Ok(vec![control_error_body_frame(
4561                &frame,
4562                ErrorBody {
4563                    code: error_codes::MODULE_WARMING.to_string(),
4564                    message: format!(
4565                        "module_id '{target_module_id}' requires capability '{capability}', \
4566                         which no registered module provides; retry"
4567                    ),
4568                    detail: Some(serde_json::json!({
4569                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4570                        "capability": capability,
4571                    })),
4572                },
4573            )?]);
4574        }
4575
4576        if !target_has_required_role(&target, &registration.manifest.provides) {
4577            return Ok(vec![self.route_open_refusal_frame(
4578                ctx,
4579                &caller,
4580                &frame,
4581                &target_module_id,
4582                "role_not_provided",
4583                "target_unavailable",
4584                format!("module_id '{target_module_id}' does not provide the requested target"),
4585            )?]);
4586        }
4587
4588        if registration.state != ChannelState::Active {
4589            return Ok(vec![self.route_open_refusal_frame(
4590                ctx,
4591                &caller,
4592                &frame,
4593                &target_module_id,
4594                "registration_not_active",
4595                "target_unavailable",
4596                format!("module_id '{target_module_id}' is not active"),
4597            )?]);
4598        }
4599
4600        if self
4601            .forwarding
4602            .module_is_draining(&target_module_id)
4603            .map_err(RouterError::Forwarding)?
4604        {
4605            return Ok(vec![self.route_open_refusal_frame(
4606                ctx,
4607                &caller,
4608                &frame,
4609                &target_module_id,
4610                "reloading",
4611                "module_reloading",
4612                format!("module_id '{target_module_id}' is reloading"),
4613            )?]);
4614        }
4615
4616        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4617            process_liveness.process_live(&target_module_id) == Some(false)
4618        }) {
4619            // A module the supervisor is restarting or reloading can still hold
4620            // a registration: the old process before its connection closes, or
4621            // a new one that registered while the supervisor was draining. The
4622            // forwarding table does not see that as draining, but the consumer
4623            // should still be told to retry soon, exactly as for the drain
4624            // above, rather than that the target is unavailable.
4625            if process_liveness.process_replacing(&target_module_id) {
4626                return Ok(vec![self.route_open_refusal_frame(
4627                    ctx,
4628                    &caller,
4629                    &frame,
4630                    &target_module_id,
4631                    "reloading",
4632                    "module_reloading",
4633                    format!("module_id '{target_module_id}' is reloading"),
4634                )?]);
4635            }
4636            return Ok(vec![self.route_open_refusal_frame(
4637                ctx,
4638                &caller,
4639                &frame,
4640                &target_module_id,
4641                "supervisor_not_live",
4642                "target_unavailable",
4643                format!("module_id '{target_module_id}' is not live"),
4644            )?]);
4645        }
4646
4647        if !self
4648            .forwarding
4649            .has_live_module_connection(&target_module_id)
4650            .map_err(RouterError::Forwarding)?
4651        {
4652            return Ok(vec![self.route_open_refusal_frame(
4653                ctx,
4654                &caller,
4655                &frame,
4656                &target_module_id,
4657                "no_forwarding_connection",
4658                "target_unavailable",
4659                format!("module_id '{target_module_id}' has no live forwarding connection"),
4660            )?]);
4661        }
4662
4663        if let Some(error) =
4664            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4665        {
4666            self.observe_route_open_refusal(
4667                ctx,
4668                &caller,
4669                &target_module_id,
4670                "op_not_allowed",
4671                "op_not_allowed",
4672            );
4673            return Ok(vec![error]);
4674        }
4675
4676        let principal =
4677            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4678                Ok(principal) => principal,
4679                Err(error) => {
4680                    self.observe_route_open_refusal(
4681                        ctx,
4682                        &caller,
4683                        &target_module_id,
4684                        "bad_consumer_identity",
4685                        "bad_consumer_identity",
4686                    );
4687                    return Ok(vec![error]);
4688                }
4689            };
4690        timings.principal = principal_label(&principal);
4691
4692        // This is attested, control-plane policy for supervised module origins.
4693        // Keep it before route reservation and out of the opaque forwarding hot
4694        // path: data frames must never acquire a per-frame capability check.
4695        if let Principal::Reserved {
4696            module_id: opening_module_id,
4697        } = &principal
4698        {
4699            if let Some(opening_registration) = self
4700                .registry
4701                .get_module(opening_module_id)
4702                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4703            {
4704                if let Some(capability) =
4705                    denied_capability(&opening_registration.manifest, &registration.manifest)
4706                {
4707                    warn!(
4708                        opening_module_id,
4709                        target_module_id,
4710                        capability,
4711                        "refusing route.open because an attested capability deny edge matches"
4712                    );
4713                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4714                        &frame,
4715                        &target_module_id,
4716                        "capability_deny_edge",
4717                        "capability_forbidden",
4718                        format!(
4719                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4720                        ),
4721                    )?]);
4722                }
4723            }
4724        }
4725
4726        if admission_facts.is_some() {
4727            let carrier_matches = matches!(
4728                &principal,
4729                Principal::Reserved { module_id }
4730                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4731            );
4732            if !carrier_matches {
4733                return Ok(vec![self.route_open_refusal_frame(
4734                    ctx,
4735                    &caller,
4736                    &frame,
4737                    &target_module_id,
4738                    "admission_facts_carrier_not_permitted",
4739                    "admission_facts_not_permitted",
4740                    "admission facts may only be carried by the configured reserved module",
4741                )?]);
4742            }
4743
4744            let target_allowed = self
4745                .admission_facts_targets
4746                .as_ref()
4747                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4748            if !target_allowed {
4749                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4750                    &frame,
4751                    &target_module_id,
4752                    "admission_facts_target_not_listed",
4753                    "admission_facts_target_not_allowed",
4754                    format!(
4755                        "admission facts are not permitted for target module_id '{target_module_id}'"
4756                    ),
4757                )?]);
4758            }
4759
4760            // Keep the value opaque to subc. The downstream admission validator owns
4761            // schema and semantic checks; this daemon only enforces carrier authority
4762            // and the configured destination allowlist.
4763        }
4764
4765        // Scope admission, on the attested principal above and never on the
4766        // request body. The tag read here travels with the pending bind and is
4767        // compared with the published one at commit, so a sync between here
4768        // and the module's ack refuses the open instead of binding a stamp
4769        // that is no longer true.
4770        let (bound_scope, mut scope_stamp) = match &scope {
4771            None => (None, None),
4772            Some(selector) => {
4773                let owner_configured = match &selector.owner {
4774                    // A reserved owner counts as configured from before its
4775                    // process is spawned (see `SupervisorHandle::is_configured`).
4776                    // So an owner that has not synced its scopes yet is refused
4777                    // as retryable (`scope_not_synced`), not as one that will
4778                    // never sync.
4779                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4780                    _ => false,
4781                };
4782                let mut scopes = self
4783                    .scopes
4784                    .write()
4785                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4786                let admitted = scopes
4787                    .admit(&principal, &target_module_id, selector, owner_configured)
4788                    .and_then(|admission| {
4789                        crate::scopes::check_target_flow_support(
4790                            &admission.stamp,
4791                            &target_module_id,
4792                            registration.manifest.capabilities.as_ref(),
4793                        )?;
4794                        crate::scopes::check_target_agent_run_support(
4795                            &admission.stamp,
4796                            &target_module_id,
4797                            registration.manifest.capabilities.as_ref(),
4798                        )?;
4799                        let bound = scopes.reserve_bind(
4800                            &admission,
4801                            &principal,
4802                            ctx.connection_id,
4803                            selector.hold_presence,
4804                        );
4805                        Ok((admission, bound))
4806                    });
4807                match admitted {
4808                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4809                    Err(refusal) => {
4810                        return Ok(vec![self.route_open_refusal_frame(
4811                            ctx,
4812                            &caller,
4813                            &frame,
4814                            &target_module_id,
4815                            refusal.code,
4816                            refusal.code,
4817                            refusal.message,
4818                        )?]);
4819                    }
4820                }
4821            }
4822        };
4823        // From here on a refusal names the scope by the stamp admission
4824        // issued, exactly as the `route.open accepted` line does.
4825        if let Some(stamp) = &scope_stamp {
4826            caller.scope = route_open_scope_label(stamp);
4827        }
4828
4829        // Bind admits a root that no longer exists on disk, because refusing here
4830        // closes the only exit from a paused run: cancel needs a bound route, and a
4831        // renamed or reclaimed directory makes that route unopenable forever. The
4832        // run itself is intact and still addressable by its recorded identity.
4833        //
4834        // This does NOT relax the rule the strict constructor protects. That rule is
4835        // that no root is ever aliased into NEW durable state -- a missing component
4836        // can reappear as a symlink elsewhere, which would move the identity and
4837        // split a session's history across two of them. The engine now refuses the
4838        // two operations that create such state (send and import) at admission,
4839        // which is a narrower way to hold the same invariant: reads and terminations
4840        // are admitted, writes are not. That refusal had to ship before this line
4841        // changed, or there is an interval where a send commits under a provisional
4842        // identity -- the exact failure the original policy existed to prevent.
4843        //
4844        // Resolution follows realpath rather than lexical cleanup: the longest
4845        // existing ancestor is canonicalized and the missing tail re-appended, so a
4846        // live root is unchanged and a vanished leaf keeps the identity it was
4847        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4848        // same caller the moment the directory vanished, which strands the run more
4849        // quietly than refusing it.
4850        timings.enter(RouteOpenPhase::ProjectRoot);
4851        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4852            Ok(project_root) => project_root,
4853            Err(err) => {
4854                return Ok(vec![control_error_frame(
4855                    &frame,
4856                    "invalid_project_root",
4857                    err.to_string(),
4858                )?])
4859            }
4860        };
4861        timings.enter(RouteOpenPhase::Admission);
4862        identity.project_root = project_root.as_path().to_path_buf();
4863
4864        // Last gate before any relay work, and deliberately after the cheap
4865        // registry and availability checks above: those name a more precise
4866        // condition (unknown, removed, reloading) and a caller is better served
4867        // by the precise code than by this one.
4868        //
4869        // Everything below this point costs an egress permit, a reserved handle
4870        // pair and, if the module does not answer, the whole relay budget. The
4871        // reader no longer waits for that budget, so cap each target explicitly;
4872        // serial dispatch used to provide the accidental cap of one relay per
4873        // connection. Admission is a mutex-protected count and never waits.
4874        let _concurrency_guard = match self
4875            .route_bind_concurrency
4876            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4877        {
4878            Ok(guard) => guard,
4879            Err(in_flight) => {
4880                return Ok(vec![self.route_open_target_capacity_refusal(
4881                    ctx,
4882                    &caller,
4883                    &frame,
4884                    &target_module_id,
4885                    in_flight,
4886                )?]);
4887            }
4888        };
4889
4890        // A module that has already burned the whole budget `threshold` times
4891        // in a row does not get to charge it again until a probe says it recovered.
4892        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4893            RouteBindAdmission::Admitted { guard, probe } => {
4894                if probe {
4895                    info!(
4896                        module_id = %target_module_id,
4897                        connection_id = ctx.connection_id.get(),
4898                        "route.bind breaker half-open: admitting one probe"
4899                    );
4900                }
4901                guard
4902            }
4903            RouteBindAdmission::Refused {
4904                consecutive_timeouts,
4905                retry_in,
4906                probe_in_flight,
4907            } => {
4908                return Ok(vec![self.route_open_breaker_refusal_frame(
4909                    ctx,
4910                    &caller,
4911                    &frame,
4912                    &target_module_id,
4913                    consecutive_timeouts,
4914                    retry_in,
4915                    probe_in_flight,
4916                )?]);
4917            }
4918        };
4919
4920        // Resolve the per-module budget here so the wait matches the operator's
4921        // intent for this specific target. A per-module override in
4922        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4923        // daemons) wins over the daemon-wide default.
4924        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4925        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4926        timings.enter(RouteOpenPhase::EgressReservation);
4927        let pending = match self
4928            .forwarding
4929            .begin_route_bind_relay_for(
4930                ctx.connection_id,
4931                ctx.egress.clone(),
4932                response_version(&frame),
4933                frame.header.corr,
4934                &target_module_id,
4935                principal.clone(),
4936                bound_scope,
4937                Some(project_root),
4938                relay_deadline,
4939            )
4940            .await
4941        {
4942            Ok(pending) => pending,
4943            Err(err) => {
4944                return Ok(vec![self.route_open_refusal_frame(
4945                    ctx,
4946                    &caller,
4947                    &frame,
4948                    &target_module_id,
4949                    "relay_reservation_failed",
4950                    forwarding_error_code(&err),
4951                    err.to_string(),
4952                )?])
4953            }
4954        };
4955        timings.enter(RouteOpenPhase::Admission);
4956        let crate::forwarding::PendingRouteBindRelay {
4957            endpoint,
4958            module_sink,
4959            negotiated_ver,
4960            client_channel,
4961            client_epoch,
4962            module_channel,
4963            module_epoch,
4964            corr: relay_corr,
4965            receiver,
4966        } = pending;
4967        let mut reservation =
4968            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4969
4970        // Reserving egress can wait while a module reconnects or a swap cuts
4971        // over. Check the relay's captured connection, not the earlier by-id
4972        // lookup: the original module's flow/run capabilities cannot authorize
4973        // its replacement. The captured sink stays bound to that connection.
4974        let relay_registration = self
4975            .registry
4976            .get_module_by_connection(endpoint.connection_id)
4977            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4978        let relay_capabilities = relay_registration
4979            .as_ref()
4980            .and_then(|registration| registration.manifest.capabilities.as_ref());
4981        if let Some(selector) = &scope {
4982            // Egress reservation may wait across a scope update. Recheck the
4983            // proof against the current record before sending the captured bind.
4984            if let Err(refusal) = self
4985                .scopes
4986                .read()
4987                .unwrap_or_else(|poisoned| poisoned.into_inner())
4988                .check_bind_proof(&principal, selector)
4989            {
4990                reservation.release_and_disarm();
4991                return Ok(vec![self.route_open_refusal_frame(
4992                    ctx,
4993                    &caller,
4994                    &frame,
4995                    &target_module_id,
4996                    refusal.code,
4997                    refusal.code,
4998                    refusal.message,
4999                )?]);
5000            }
5001        }
5002        if let Some(stamp) = scope_stamp
5003            .as_ref()
5004            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5005        {
5006            if let Err(refusal) = crate::scopes::check_target_flow_support(
5007                stamp,
5008                &target_module_id,
5009                relay_capabilities,
5010            )
5011            .and_then(|()| {
5012                crate::scopes::check_target_agent_run_support(
5013                    stamp,
5014                    &target_module_id,
5015                    relay_capabilities,
5016                )
5017            }) {
5018                reservation.release_and_disarm();
5019                return Ok(vec![self.route_open_refusal_frame(
5020                    ctx,
5021                    &caller,
5022                    &frame,
5023                    &target_module_id,
5024                    refusal.code,
5025                    refusal.code,
5026                    refusal.message,
5027                )?]);
5028            }
5029        }
5030
5031        // Older readers may reject unknown fields. Select optional fields from
5032        // the captured endpoint's declaration, including on unscoped binds.
5033        if !relay_capabilities.is_some_and(|caps| {
5034            caps.provides
5035                .iter()
5036                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5037        }) {
5038            if let Some(stamp) = scope_stamp.as_mut() {
5039                stamp.workdir = None;
5040                stamp.project_id = None;
5041            }
5042        }
5043        if !relay_capabilities.is_some_and(|caps| {
5044            caps.provides
5045                .iter()
5046                .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5047        }) {
5048            if let Some(stamp) = scope_stamp.as_mut() {
5049                stamp.attributes.on_behalf_of_agent = None;
5050            }
5051        }
5052        let call_mode = if relay_capabilities
5053            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5054        {
5055            Some(derive_call_mode(scope_stamp.as_ref()))
5056        } else {
5057            if let Some(stamp) = scope_stamp.as_mut() {
5058                stamp.expires_at_ms = None;
5059            }
5060            None
5061        };
5062
5063        debug!(
5064            connection_id = ctx.connection_id.get(),
5065            client_channel,
5066            client_epoch,
5067            module_channel,
5068            module_epoch,
5069            "reserved route handle pair"
5070        );
5071        // Rendered BEFORE the move into the relay, because the accept arm below
5072        // is where it is logged and the principal is gone by then.
5073        let route_principal_label = principal_label(&principal);
5074        let relay = ModuleControlRequest::RouteBind {
5075            route_channel: module_channel,
5076            epoch: module_epoch,
5077            target,
5078            identity,
5079            principal: Some(principal),
5080            consumer_capabilities,
5081            role_versions,
5082            admission_facts,
5083            scope: scope_stamp,
5084            call_mode,
5085        };
5086        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5087            RouterError::backend(
5088                0,
5089                frame.header.corr,
5090                format!("failed to encode route.bind request: {err}"),
5091            )
5092        })?;
5093        let relay_frame = Frame::build_with_version(
5094            negotiated_ver,
5095            FrameType::Request,
5096            control_flags(),
5097            0,
5098            0,
5099            relay_corr,
5100            relay_body,
5101        )
5102        .map_err(RouterError::FrameBuild)?;
5103
5104        timings.enter(RouteOpenPhase::BindSend);
5105        let sent = module_sink.send(relay_frame).await;
5106        timings.enter(RouteOpenPhase::Admission);
5107        if let Err(err) = sent {
5108            reservation.release_and_disarm();
5109            return Ok(vec![self.route_open_refusal_frame(
5110                ctx,
5111                &caller,
5112                &frame,
5113                &target_module_id,
5114                "relay_send_failed",
5115                "target_unavailable",
5116                err.to_string(),
5117            )?]);
5118        }
5119
5120        #[cfg(test)]
5121        if let Some(hold) = &self.route_bind_relay_sent_hold {
5122            // First wait: tell the test the bind is queued. Second wait: stay
5123            // here until the test has driven whatever it wants to happen
5124            // between queueing the bind and marking it enqueued.
5125            hold.wait().await;
5126            hold.wait().await;
5127        }
5128
5129        // A missing relay is not by itself an abandoned bind: the module may
5130        // already have acked it and the route may be committed. Only a relay
5131        // dropped without any module reply needs this GOODBYE; one a reply
5132        // settled has already had its GOODBYE decided by the reply path.
5133        if self
5134            .forwarding
5135            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5136            .map_err(RouterError::Forwarding)?
5137            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5138        {
5139            self.send_abandoned_route_bind_goodbye(
5140                &module_sink,
5141                negotiated_ver,
5142                module_channel,
5143                module_epoch,
5144            );
5145        }
5146
5147        timings.enter(RouteOpenPhase::BindAckWait);
5148        let outcome = timeout_at(relay_deadline, receiver).await;
5149        timings.enter(RouteOpenPhase::Admission);
5150        match outcome {
5151            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5152                reservation.disarm();
5153                if breaker.record_accepted() {
5154                    info!(
5155                        module_id = %target_module_id,
5156                        "route.bind breaker closed: the probe was accepted"
5157                    );
5158                }
5159                self.observe_route_open_accept(
5160                    ctx,
5161                    &target_module_id,
5162                    &route_principal_label,
5163                    &caller.scope,
5164                );
5165                Ok(Vec::new())
5166            }
5167            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5168                reservation.release_and_disarm();
5169                // A module that says no in microseconds is healthy. Rejection
5170                // is a different condition with its own refusal and must not
5171                // move the breaker.
5172                breaker.record_inconclusive();
5173                // The daemon's own commit re-check refused the bind because the
5174                // scope ended or changed after admission. The module accepted;
5175                // counting it as a module rejection would blame the module.
5176                let scope_code = match body.code.as_str() {
5177                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5178                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5179                    _ => None,
5180                };
5181                if let Some(code) = scope_code {
5182                    self.observe_route_open_refusal(
5183                        ctx,
5184                        &caller,
5185                        &target_module_id,
5186                        "scope_changed_before_commit",
5187                        code,
5188                    );
5189                    return Ok(vec![control_error_body_frame(&frame, body)?]);
5190                }
5191                self.counters
5192                    .increment_route_open_refused("module_rejected");
5193                info!(
5194                    target: "control",
5195                    code = "module_rejected",
5196                    module_code = ?body.code,
5197                    module_id = ?target_module_id,
5198                    connection_id = ctx.connection_id.get(),
5199                    principal = %caller.principal,
5200                    scope = %caller.scope,
5201                    harness = %caller.harness,
5202                    "route.open refused"
5203                );
5204                Ok(vec![control_error_body_frame(&frame, body)?])
5205            }
5206            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5207                reservation.release_and_disarm();
5208                breaker.record_inconclusive();
5209                // Fires when the module's connection closes while a relayed
5210                // bind is pending -- typically a caller racing a module restart
5211                // whose bind was relayed BEFORE the drain mark went up. Logged
5212                // because the caller sees only its own error and the fleet has
5213                // already spent one diagnosis round unable to tell this arm
5214                // from a relay timeout without daemon-side evidence.
5215                tracing::warn!(
5216                    module_id = %target_module_id,
5217                    "route.bind relay abandoned: {message}"
5218                );
5219                Ok(vec![self.route_open_refusal_frame(
5220                    ctx,
5221                    &caller,
5222                    &frame,
5223                    &target_module_id,
5224                    "relay_abandoned",
5225                    "target_unavailable",
5226                    message,
5227                )?])
5228            }
5229            Ok(Err(_)) => {
5230                reservation.release_and_disarm();
5231                breaker.record_inconclusive();
5232                Ok(vec![self.route_open_refusal_frame(
5233                    ctx,
5234                    &caller,
5235                    &frame,
5236                    &target_module_id,
5237                    "relay_waiter_canceled",
5238                    "target_unavailable",
5239                    "route.bind relay waiter was canceled before the module responded",
5240                )?])
5241            }
5242            Err(_) => {
5243                reservation.release_and_disarm();
5244                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5245                // answer at all is the one condition a fast refusal can
5246                // usefully stand in for; every other arm already answered.
5247                if let Some(opened) = breaker.record_timeout(
5248                    self.route_bind_breaker_threshold,
5249                    self.route_bind_breaker_cooldown,
5250                ) {
5251                    warn!(
5252                        module_id = %target_module_id,
5253                        consecutive_timeouts = opened.consecutive_timeouts,
5254                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5255                        reopened_after_probe = opened.reopened_after_probe,
5256                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5257                    );
5258                }
5259                // The generous budget just burned to no answer: the module is
5260                // registered and its connection is up, but its bind handler sat
5261                // on the ack for the full budget (warm-on-bind, cold configure,
5262                // or a wedged handler). Every earlier unavailability shape
5263                // fast-refuses BEFORE the relay, so this arm firing means the
5264                // slowness is module-side -- log it so the per-module timeline
5265                // is reconstructable without client audit rows.
5266                tracing::warn!(
5267                    module_id = %target_module_id,
5268                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5269                    "route.bind relay timed out: module did not ack within budget"
5270                );
5271                Ok(vec![self.route_open_refusal_frame(
5272                    ctx,
5273                    &caller,
5274                    &frame,
5275                    &target_module_id,
5276                    "relay_timed_out",
5277                    "module_timeout",
5278                    format!(
5279                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5280                        route_bind_relay_timeout
5281                    ),
5282                )?])
5283            }
5284        }
5285    }
5286
5287    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5288        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5289            snapshot: self.supervisor.spawn_snapshot(),
5290        };
5291        Ok(vec![control_response_body_frame(
5292            &frame,
5293            &response,
5294            "ClientControlResponse::SupervisorSpawnSnapshot",
5295        )?])
5296    }
5297
5298    fn handle_supervisor_spawn_subscribe(
5299        &self,
5300        ctx: &RouteCtx,
5301        frame: Frame,
5302        since: Option<SpawnCursor>,
5303    ) -> Result<Vec<Frame>, RouterError> {
5304        match self.supervisor.subscribe_spawns(
5305            ctx.connection_id,
5306            frame.header.corr,
5307            response_version(&frame),
5308            since,
5309            ctx.egress.clone(),
5310        ) {
5311            Ok(()) => Ok(Vec::new()),
5312            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5313                Ok(vec![control_error_body_frame(
5314                    &frame,
5315                    ErrorBody {
5316                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5317                        message: "spawn cursor belongs to a different daemon incarnation"
5318                            .to_string(),
5319                        detail: Some(serde_json::json!({
5320                            "current_daemon_incarnation": current
5321                        })),
5322                    },
5323                )?])
5324            }
5325            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5326                &frame,
5327                ErrorBody {
5328                    code: "spawn_cursor_too_old".to_string(),
5329                    message: "spawn cursor predates the retained event ring".to_string(),
5330                    detail: Some(serde_json::json!({
5331                        "oldest_retained_cursor": oldest
5332                    })),
5333                },
5334            )?]),
5335            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5336                0,
5337                frame.header.corr,
5338                format!("failed to open supervisor spawn subscription: {error}"),
5339            )),
5340        }
5341    }
5342
5343    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5344        let generation = self
5345            .registry
5346            .generation()
5347            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5348        let mut modules = Vec::new();
5349        for module in self.supervisor.list() {
5350            let status = module.status_for_control("list").map_err(|err| {
5351                RouterError::backend(
5352                    0,
5353                    frame.header.corr,
5354                    format!("failed to read supervisor status: {err}"),
5355                )
5356            })?;
5357            let (configured, _) = module.configuration().map_err(|err| {
5358                RouterError::backend(
5359                    0,
5360                    frame.header.corr,
5361                    format!("failed to read module configuration: {err}"),
5362                )
5363            })?;
5364            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5365                RouterError::backend(
5366                    0,
5367                    frame.header.corr,
5368                    format!("failed to read retry time: {err}"),
5369                )
5370            })?;
5371            // Status and configuration snapshots release their locks before the image probe awaits.
5372            let image = module.running_image_agreement().await;
5373            // Read per request so the figure is current when the operator asks;
5374            // the daemon samples nothing in between.
5375            let resources = Some(module.child_resource_usage());
5376            let mut verdict =
5377                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5378            verdict.spec = Some(launch_spec_agreement(
5379                &configured,
5380                status.process_alive,
5381                status.launch_spec_digest.as_ref(),
5382            ));
5383            let pending_reload = Some(verdict);
5384            let configured_enabled = self
5385                .capability_evaluator
5386                .configured_enabled(&status.module_id);
5387            // Keep the retired policy field on the wire for one release so
5388            // existing status consumers still receive the platform policy.
5389            modules.push(
5390                SupervisorEntry::new(
5391                    status.module_id,
5392                    status.state.to_string(),
5393                    status.enabled,
5394                    status.live,
5395                    status.health.status,
5396                )
5397                .with_configured_enabled(configured_enabled)
5398                .with_next_retry_at_ms(next_retry_at_ms)
5399                .with_launch_nonce_env(Some(!cfg!(unix)))
5400                .with_launch_nonce_source(status.launch_nonce_source)
5401                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5402                .with_protocol(status.protocol)
5403                .with_pending_reload(pending_reload)
5404                .with_last_probe_ms(status.health.last_probe_ms)
5405                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5406                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5407                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5408                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5409                .with_restart_count(Some(status.restart_count))
5410                .with_max_restarts(Some(status.max_restarts))
5411                .with_lifetime_restarts(Some(status.lifetime_restarts))
5412                .with_spawn_generation(Some(status.spawn_generation))
5413                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5414                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5415                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5416                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5417                .with_resources(resources),
5418            );
5419        }
5420        let response = ClientControlResponse::SupervisorList {
5421            generation,
5422            modules,
5423        };
5424        Ok(vec![control_response_body_frame(
5425            &frame,
5426            &response,
5427            "ClientControlResponse::SupervisorList",
5428        )?])
5429    }
5430
5431    fn handle_supervisor_stderr_tail(
5432        &self,
5433        frame: Frame,
5434        module_id: String,
5435        max_lines: Option<u32>,
5436        max_bytes: Option<u32>,
5437    ) -> Result<Vec<Frame>, RouterError> {
5438        let Some(module) = self.supervisor.get(&module_id) else {
5439            return Ok(vec![control_error_frame(
5440                &frame,
5441                "unknown_module",
5442                format!("module_id '{module_id}' is not supervised"),
5443            )?]);
5444        };
5445
5446        let snapshot = module.stderr_tail(
5447            max_lines.map(|value| value as usize),
5448            max_bytes.map(|value| value as usize),
5449        );
5450
5451        let response = ClientControlResponse::SupervisorStderrTail {
5452            module_id,
5453            tail: StderrTail {
5454                capture: match snapshot.capture {
5455                    CaptureState::Captured => StderrCaptureState::Captured,
5456                    CaptureState::Incomplete { reason } => {
5457                        StderrCaptureState::Incomplete { reason }
5458                    }
5459                    CaptureState::NotCaptured { reason } => {
5460                        StderrCaptureState::NotCaptured { reason }
5461                    }
5462                },
5463                entries: snapshot
5464                    .entries
5465                    .into_iter()
5466                    .map(|entry| match entry {
5467                        TailEntry::Line {
5468                            text,
5469                            truncated,
5470                            at_ms,
5471                        } => StderrTailEntry::Line {
5472                            text,
5473                            truncated,
5474                            at_ms,
5475                        },
5476                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5477                    })
5478                    .collect(),
5479                dropped_lines: snapshot.dropped_lines,
5480            },
5481        };
5482        Ok(vec![control_response_body_frame(
5483            &frame,
5484            &response,
5485            "ClientControlResponse::SupervisorStderrTail",
5486        )?])
5487    }
5488
5489    async fn handle_supervisor_terminals(
5490        &self,
5491        frame: Frame,
5492        module_id: String,
5493    ) -> Result<Vec<Frame>, RouterError> {
5494        let Some(module) = self.supervisor.get(&module_id) else {
5495            return Ok(vec![control_error_frame(
5496                &frame,
5497                "unknown_module",
5498                format!("module_id '{module_id}' is not supervised"),
5499            )?]);
5500        };
5501
5502        // The journal read runs on a blocking thread: it can be megabytes of
5503        // file I/O and must not occupy a runtime worker.
5504        let terminals = module
5505            .read_durable_terminal_history()
5506            .await
5507            .map_err(|error| {
5508                RouterError::backend(
5509                    0,
5510                    frame.header.corr,
5511                    format!("failed to read terminal history: {error}"),
5512                )
5513            })?;
5514        let response = ClientControlResponse::SupervisorTerminals {
5515            module_id,
5516            terminals,
5517        };
5518        Ok(vec![control_response_body_frame(
5519            &frame,
5520            &response,
5521            "ClientControlResponse::SupervisorTerminals",
5522        )?])
5523    }
5524
5525    fn handle_supervisor_routes(
5526        &self,
5527        frame: Frame,
5528        module_id: Option<String>,
5529    ) -> Result<Vec<Frame>, RouterError> {
5530        let modules = self
5531            .forwarding
5532            .route_census(module_id.as_deref())
5533            .map_err(RouterError::Forwarding)?
5534            .into_iter()
5535            .map(|(module_id, routes)| SupervisorRouteModule {
5536                module_id,
5537                routes: routes
5538                    .into_iter()
5539                    .map(|route| SupervisorRoute {
5540                        consumer: match route.principal {
5541                            Principal::Reserved { module_id } => {
5542                                SupervisorRouteConsumer::Reserved { module_id }
5543                            }
5544                            Principal::Direct | Principal::Unverified => {
5545                                SupervisorRouteConsumer::Direct {
5546                                    connection_id: route.goodbye_target.connection_id.get(),
5547                                }
5548                            }
5549                        },
5550                        age_ms: Instant::now()
5551                            .saturating_duration_since(route.bound_at)
5552                            .as_millis()
5553                            .try_into()
5554                            .unwrap_or(u64::MAX),
5555                        draining: route.draining,
5556                        drain_reason: route.drain_reason,
5557                    })
5558                    .collect(),
5559            })
5560            .collect();
5561        let response = ClientControlResponse::SupervisorRoutes { modules };
5562        Ok(vec![control_response_body_frame(
5563            &frame,
5564            &response,
5565            "ClientControlResponse::SupervisorRoutes",
5566        )?])
5567    }
5568
5569    async fn handle_supervisor_provenance(
5570        &self,
5571        frame: Frame,
5572        module_id: Option<String>,
5573    ) -> Result<Vec<Frame>, RouterError> {
5574        let mut selected = if let Some(module_id) = module_id {
5575            let Some(module) = self.supervisor.get(&module_id) else {
5576                return Ok(vec![control_error_frame(
5577                    &frame,
5578                    "unknown_module",
5579                    format!("module_id '{module_id}' is not supervised"),
5580                )?]);
5581            };
5582            vec![module]
5583        } else {
5584            self.supervisor.list()
5585        };
5586
5587        let mut modules = Vec::with_capacity(selected.len());
5588        for module in selected.drain(..) {
5589            let (status, observed_image) = module
5590                .status_and_running_image_agreement()
5591                .await
5592                .map_err(|err| {
5593                    RouterError::backend(
5594                        0,
5595                        frame.header.corr,
5596                        format!("failed to read supervisor status: {err}"),
5597                    )
5598                })?;
5599            let module_declared = self
5600                .registry
5601                .get_module(&status.module_id)
5602                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5603                .and_then(|registration| registration.manifest.provenance)
5604                .map(|build| ModuleDeclaredProvenance::Reported { build })
5605                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5606            #[cfg(test)]
5607            let running_image = match &self.provenance_probe_override {
5608                Some(result) => result.clone(),
5609                None => observed_image,
5610            };
5611            #[cfg(not(test))]
5612            let running_image = observed_image;
5613            modules.push(SupervisorModuleProvenance {
5614                module_id: status.module_id,
5615                module_declared,
5616                daemon_observed: SupervisorObservedProcess {
5617                    launch_nonce_source: status.launch_nonce_source,
5618                    pid: status.pid,
5619                    spawned_at_ms: status.spawned_at_ms,
5620                    spawned_from: status.spawned_from,
5621                    running_image,
5622                },
5623            });
5624        }
5625        let daemon = SupervisorDaemonProvenance {
5626            daemon_build: self.daemon_provenance.build.clone(),
5627            daemon_observed: DaemonObservedProcess {
5628                pid: self.daemon_provenance.pid,
5629                started_at_ms: self
5630                    .daemon_provenance
5631                    .start_clock
5632                    .map(|clock| clock.started_at_ms())
5633                    .or(self.daemon_provenance.started_at_ms),
5634                running_image: self
5635                    .daemon_provenance
5636                    .probe
5637                    .observe(
5638                        self.daemon_provenance.pid,
5639                        self.daemon_provenance.executable_path.as_deref(),
5640                        self.daemon_provenance.executable_identity.clone(),
5641                        self.daemon_provenance.process_start_time,
5642                    )
5643                    .await,
5644            },
5645        };
5646        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5647        Ok(vec![control_response_body_frame(
5648            &frame,
5649            &response,
5650            "ClientControlResponse::SupervisorProvenance",
5651        )?])
5652    }
5653
5654    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5655        self.refresh_capability_requirements();
5656        let generation = self
5657            .registry
5658            .generation()
5659            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5660        let modules = self
5661            .supervisor
5662            .list()
5663            .into_iter()
5664            .map(|module| {
5665                let status = module.status_for_control("health").map_err(|err| {
5666                    RouterError::backend(
5667                        0,
5668                        frame.header.corr,
5669                        format!("failed to read supervisor health: {err}"),
5670                    )
5671                })?;
5672                let module_id = status.module_id;
5673                let capability_detail = self
5674                    .capability_evaluator
5675                    .required_problem_detail(&module_id);
5676                Ok(SupervisorHealthEntry {
5677                    module_id,
5678                    status: status.health.status,
5679                    detail: append_capability_problem_detail(
5680                        status.health.detail,
5681                        capability_detail,
5682                    ),
5683                    metrics: status.health.metrics,
5684                    consecutive_failures: status.health.consecutive_failures,
5685                    late_answer_count: status.health.late_answer_count,
5686                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5687                    last_action: status.health.last_action,
5688                    last_action_ms: status.health.last_action_ms,
5689                    last_probe_ms: status.health.last_probe_ms,
5690                })
5691            })
5692            .collect::<Result<Vec<_>, RouterError>>()?;
5693        let response = ClientControlResponse::SupervisorHealth {
5694            generation,
5695            modules,
5696        };
5697        Ok(vec![control_response_body_frame(
5698            &frame,
5699            &response,
5700            "ClientControlResponse::SupervisorHealth",
5701        )?])
5702    }
5703
5704    async fn handle_supervisor_restart(
5705        &self,
5706        frame: Frame,
5707        module_id: String,
5708        drain_timeout_ms: Option<u64>,
5709        requester: Principal,
5710    ) -> Result<Vec<Frame>, RouterError> {
5711        let operation_lock = self.supervisor.operation_lock();
5712        let _operation_guard = operation_lock.lock().await;
5713        let Some(module) = self.supervisor.get(&module_id) else {
5714            return Ok(vec![control_error_frame(
5715                &frame,
5716                "unknown_module",
5717                format!("module_id '{module_id}' is not supervised"),
5718            )?]);
5719        };
5720
5721        self.route_outages.mark_operator_action(&module_id);
5722        if let Err(err) = module
5723            .restart_requested(drain_timeout_ms, Some(requester))
5724            .await
5725        {
5726            self.route_outages
5727                .operator_action_ended_unrefused(&module_id);
5728            let (code, message) = match err {
5729                crate::supervise::SuperviseError::Disabled { .. } => {
5730                    ("module_disabled", err.to_string())
5731                }
5732                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5733                    ("swap_in_progress", err.to_string())
5734                }
5735                _ => (
5736                    "target_unavailable",
5737                    format!("failed to restart module_id '{module_id}': {err}"),
5738                ),
5739            };
5740            return Ok(vec![control_error_frame(&frame, code, message)?]);
5741        }
5742
5743        let response = ClientControlResponse::SupervisorAck {
5744            module_id,
5745            applied: true,
5746        };
5747        Ok(vec![control_response_body_frame(
5748            &frame,
5749            &response,
5750            "ClientControlResponse::SupervisorAck",
5751        )?])
5752    }
5753
5754    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5755    /// when the old process has finished draining: a caller whose own lane
5756    /// rides the old process must get its reply before that drain waits on it.
5757    async fn handle_supervisor_swap(
5758        &self,
5759        frame: Frame,
5760        module_id: String,
5761        ready_timeout_ms: Option<u64>,
5762        requester: Principal,
5763    ) -> Result<Vec<Frame>, RouterError> {
5764        // The daemon-wide operation lock is held only to resolve the handle,
5765        // not across the swap. The swap can take its whole readiness budget,
5766        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5767        // holding it here would park an operator's stop behind the swap it is
5768        // meant to abort. A rescan or stop that reaches the module during the
5769        // swap is served by the swap itself (see `supervise_swap`).
5770        let module = {
5771            let operation_lock = self.supervisor.operation_lock();
5772            let _operation_guard = operation_lock.lock().await;
5773            self.supervisor.get(&module_id)
5774        };
5775        let Some(module) = module else {
5776            return Ok(vec![control_error_frame(
5777                &frame,
5778                "unknown_module",
5779                format!("module_id '{module_id}' is not supervised"),
5780            )?]);
5781        };
5782
5783        self.route_outages.mark_operator_action(&module_id);
5784        if let Err(err) = module
5785            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5786            .await
5787        {
5788            self.route_outages
5789                .operator_action_ended_unrefused(&module_id);
5790            use crate::supervise::SuperviseError;
5791            let message = err.to_string();
5792            let error = match err {
5793                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5794                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5795                    code: "swap_refused".to_string(),
5796                    message,
5797                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5798                },
5799                SuperviseError::SwapFailed {
5800                    arm,
5801                    candidate_exit,
5802                    ..
5803                } => ErrorBody {
5804                    code: "swap_failed".to_string(),
5805                    message,
5806                    detail: Some(serde_json::json!({
5807                        "arm": arm.as_str(),
5808                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5809                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5810                    })),
5811                },
5812                _ => ErrorBody::new(
5813                    "target_unavailable",
5814                    format!("failed to swap module_id '{module_id}': {message}"),
5815                ),
5816            };
5817            return Ok(vec![control_error_body_frame(&frame, error)?]);
5818        }
5819        // A completed swap kept the incumbent serving until cutover, so it
5820        // usually opened no outage; a mark left behind would make the next,
5821        // unrelated outage read as requested.
5822        self.route_outages
5823            .operator_action_ended_unrefused(&module_id);
5824
5825        let response = ClientControlResponse::SupervisorAck {
5826            module_id,
5827            applied: true,
5828        };
5829        Ok(vec![control_response_body_frame(
5830            &frame,
5831            &response,
5832            "ClientControlResponse::SupervisorAck",
5833        )?])
5834    }
5835
5836    async fn handle_supervisor_reload(
5837        &self,
5838        frame: Frame,
5839        module_id: String,
5840        requester: Principal,
5841    ) -> Result<Vec<Frame>, RouterError> {
5842        let operation_lock = self.supervisor.operation_lock();
5843        let _operation_guard = operation_lock.lock().await;
5844        let Some(module) = self.supervisor.get(&module_id) else {
5845            return Ok(vec![control_error_frame(
5846                &frame,
5847                "unknown_module",
5848                format!("module_id '{module_id}' is not supervised"),
5849            )?]);
5850        };
5851
5852        self.route_outages.mark_operator_action(&module_id);
5853        if let Err(err) = module.reload_requested(Some(requester)).await {
5854            self.route_outages
5855                .operator_action_ended_unrefused(&module_id);
5856            let (code, message) = match err {
5857                crate::supervise::SuperviseError::Disabled { .. } => {
5858                    ("module_disabled", err.to_string())
5859                }
5860                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5861                    ("swap_in_progress", err.to_string())
5862                }
5863                _ => (
5864                    "reload_failed",
5865                    format!("failed to reload module_id '{module_id}': {err}"),
5866                ),
5867            };
5868            return Ok(vec![control_error_frame(&frame, code, message)?]);
5869        }
5870
5871        let response = ClientControlResponse::SupervisorAck {
5872            module_id,
5873            applied: true,
5874        };
5875        Ok(vec![control_response_body_frame(
5876            &frame,
5877            &response,
5878            "ClientControlResponse::SupervisorAck",
5879        )?])
5880    }
5881
5882    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5883    /// the configuration comes from and whether to apply it; every mode then
5884    /// runs the same comparison, so a preview describes the apply it previews.
5885    async fn handle_supervisor_rescan(
5886        &self,
5887        frame: Frame,
5888        mode: RescanMode,
5889        requester: Principal,
5890    ) -> Result<Vec<Frame>, RouterError> {
5891        let preview = mode.previews();
5892        let Some(context) = self.rescan.clone() else {
5893            return Ok(vec![control_error_frame(
5894                &frame,
5895                "rescan_unavailable",
5896                "the daemon was not started with a reloadable config path".to_string(),
5897            )?]);
5898        };
5899
5900        let operation_lock = self.supervisor.operation_lock();
5901        let _operation_guard = operation_lock.lock().await;
5902        // Loaded under the operation lock, so no other rescan can change the
5903        // staged configuration between a checked apply's comparison and the
5904        // apply itself.
5905        let loaded = match self.load_rescan_config(&mode, &context) {
5906            Ok(config) => config,
5907            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5908        };
5909        // `load` reports a missing file as Ok(None), which is correct at boot
5910        // (no config, nothing to supervise) and catastrophic here: rescan treats
5911        // "not in the config" as "remove it", so an absent file would read as an
5912        // empty module list and retire the entire running fleet. An editor
5913        // writing via write-new-then-rename, or a half-finished edit, is enough
5914        // to open that window. Refuse instead: a config that cannot be read
5915        // carries no instruction to remove anything.
5916        let Some((config, activity_config)) = loaded else {
5917            return Ok(vec![control_error_frame(
5918                &frame,
5919                "invalid_daemon_config",
5920                format!(
5921                    "daemon config not found at {}; refusing to rescan (an absent config would \
5922                     retire every supervised module)",
5923                    context.config_path.display()
5924                ),
5925            )?]);
5926        };
5927        let (
5928            configured_port,
5929            storage_config,
5930            admission_facts_carrier_module_id,
5931            admission_facts_targets,
5932            scope_authority_owners,
5933            privileged_operator_prompt,
5934            modules,
5935            reserved_capabilities,
5936        ) = (
5937            config.port,
5938            config.storage,
5939            config.admission_facts_carrier_module_id,
5940            config.admission_facts_targets,
5941            config.scope_authority_owners,
5942            config.privileged_operator_prompt,
5943            config.modules,
5944            config.reserved_capabilities,
5945        );
5946
5947        // Collect the sections rescan cannot apply, so the REPLY carries them.
5948        //
5949        // The warning below has always been correct and has always gone only to
5950        // the journal -- addressed to whoever reads logs, while the person who
5951        // just edited the config is looking at the CLI. Naming each section
5952        // individually rather than setting a flag: "something outside modules
5953        // changed" sends the operator back to diffing their own file, which is
5954        // the work this is meant to save.
5955        let mut restart_required = Vec::new();
5956        for section in RestartRequiredSection::ALL {
5957            let changed = match section {
5958                RestartRequiredSection::Port => configured_port != context.configured_port,
5959                RestartRequiredSection::Storage => storage_config != context.storage_config,
5960                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5961                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5962                }
5963                RestartRequiredSection::AdmissionFactsTargets => {
5964                    admission_facts_targets != context.admission_facts_targets
5965                }
5966                RestartRequiredSection::ScopeAuthorityOwners => {
5967                    scope_authority_owners != context.scope_authority_owners
5968                }
5969                RestartRequiredSection::PrivilegedOperatorPrompt => {
5970                    privileged_operator_prompt != context.privileged_operator_prompt
5971                }
5972            };
5973            if changed {
5974                restart_required.push(section.label().to_string());
5975            }
5976        }
5977        if !restart_required.is_empty() {
5978            warn!(
5979                config_path = %context.config_path.display(),
5980                sections = %restart_required.join(", "),
5981                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5982            );
5983        }
5984
5985        for configured in &modules {
5986            if let Err(err) = validate_spec(&configured.module_spec()) {
5987                return Ok(vec![control_error_frame(
5988                    &frame,
5989                    "invalid_daemon_config",
5990                    format!("supervisor rescan rejected daemon config: {err}"),
5991                )?]);
5992            }
5993        }
5994
5995        let configured_capabilities = modules
5996            .iter()
5997            .map(|module| (module.module_id.clone(), module.enabled))
5998            .collect::<Vec<_>>();
5999        let preview_capability_warnings = if preview {
6000            let (_, registrations) = self.runtime_capability_snapshot()?;
6001            let current_modules = self
6002                .supervisor
6003                .list()
6004                .into_iter()
6005                .map(|module| module.module_id().to_string())
6006                .collect::<BTreeSet<_>>();
6007            let resulting_modules = configured_capabilities.clone();
6008            let removed = current_modules
6009                .into_iter()
6010                .filter(|module_id| {
6011                    !resulting_modules
6012                        .iter()
6013                        .any(|(configured_id, _)| configured_id == module_id)
6014                })
6015                .collect::<Vec<_>>();
6016            // A candidate preview also names providers the candidate disables.
6017            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6018            let disabled = match &mode {
6019                RescanMode::Candidate(_) => resulting_modules
6020                    .iter()
6021                    .filter(|(_, enabled)| !enabled)
6022                    .map(|(module_id, _)| module_id.clone())
6023                    .collect::<Vec<_>>(),
6024                _ => Vec::new(),
6025            };
6026            self.capability_evaluator.preview_warnings(
6027                resulting_modules,
6028                &removed,
6029                &disabled,
6030                &registrations,
6031            )
6032        } else {
6033            Vec::new()
6034        };
6035        let reconciled = match &mode {
6036            RescanMode::Candidate(candidate) => self
6037                .preview_candidate(&context, modules, candidate)
6038                .map(|result| (result, Vec::new())),
6039            _ => {
6040                self.reconcile_supervised_modules(
6041                    &context.supervisor,
6042                    modules,
6043                    preview,
6044                    requester.clone(),
6045                )
6046                .await
6047            }
6048        };
6049        let (result, changed) = match reconciled {
6050            Ok(result) => result,
6051            Err(message) => {
6052                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6053            }
6054        };
6055        if !preview {
6056            if let Some(ledger) = &self.activity {
6057                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6058                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6059            }
6060            if let Err(code) = self.configure_activity(activity_config).await {
6061                warn!(
6062                    code,
6063                    "activity retention could not be applied during rescan"
6064                );
6065            }
6066            self.capability_evaluator
6067                .configure(configured_capabilities, reserved_capabilities);
6068            self.capability_evaluator.wake_deadline_loop();
6069            self.refresh_capability_requirements();
6070        }
6071        let mut result = result;
6072        result.restart_required = restart_required;
6073        result.capability_warnings = preview_capability_warnings;
6074        let response = ClientControlResponse::SupervisorRescan { result };
6075        Ok(vec![control_response_body_frame(
6076            &frame,
6077            &response,
6078            "ClientControlResponse::SupervisorRescan",
6079        )?])
6080    }
6081
6082    async fn handle_supervisor_release_reserved(
6083        &self,
6084        frame: Frame,
6085        module_id: String,
6086    ) -> Result<Vec<Frame>, RouterError> {
6087        let Some(context) = self.rescan.clone() else {
6088            return Ok(vec![control_error_frame(
6089                &frame,
6090                "release_unavailable",
6091                "reserved-id release requires a daemon started with a reloadable config path",
6092            )?]);
6093        };
6094        let operation_lock = self.supervisor.operation_lock();
6095        let _operation_guard = operation_lock.lock().await;
6096        let loaded = match crate::daemon_config::load(&context.config_path) {
6097            Ok(Some(config)) => config,
6098            Ok(None) => {
6099                return Ok(vec![control_error_frame(
6100                    &frame,
6101                    "invalid_daemon_config",
6102                    format!(
6103                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6104                        context.config_path.display()
6105                    ),
6106                )?])
6107            }
6108            Err(err) => {
6109                return Ok(vec![control_error_frame(
6110                    &frame,
6111                    "invalid_daemon_config",
6112                    format!("unable to verify reserved-id release against daemon config: {err}"),
6113                )?])
6114            }
6115        };
6116        if loaded
6117            .modules
6118            .iter()
6119            .any(|configured| configured.module_id == module_id)
6120        {
6121            return Ok(vec![control_error_frame(
6122                &frame,
6123                "reserved_module_configured",
6124                format!(
6125                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6126                ),
6127            )?]);
6128        }
6129        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6130            return Ok(vec![control_error_frame(
6131                &frame,
6132                "reserved_gate_not_retained",
6133                format!(
6134                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6135                ),
6136            )?]);
6137        }
6138
6139        let response = ClientControlResponse::SupervisorAck {
6140            module_id,
6141            applied: true,
6142        };
6143        Ok(vec![control_response_body_frame(
6144            &frame,
6145            &response,
6146            "ClientControlResponse::SupervisorAck",
6147        )?])
6148    }
6149
6150    /// Compare the configured module set with the supervised one, changing
6151    /// nothing. The staged spec each supervised module holds is what the
6152    /// configured entry is compared with.
6153    fn plan_rescan(
6154        &self,
6155        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6156    ) -> Result<RescanPlan, String> {
6157        let mut current = BTreeMap::new();
6158        for module in self.supervisor.list() {
6159            let (spec, health) = module.configuration().map_err(|err| {
6160                format!(
6161                    "failed to read configuration for module_id '{}': {err}",
6162                    module.module_id()
6163                )
6164            })?;
6165            let enabled = module
6166                .status()
6167                .map_err(|err| {
6168                    format!(
6169                        "failed to read status for module_id '{}': {err}",
6170                        module.module_id()
6171                    )
6172                })?
6173                .enabled;
6174            current.insert(
6175                module.module_id().to_string(),
6176                (module, spec, health, enabled),
6177            );
6178        }
6179        let configured = configured_modules
6180            .into_iter()
6181            .map(|module| (module.module_id.clone(), module))
6182            .collect::<BTreeMap<_, _>>();
6183
6184        let added = configured
6185            .keys()
6186            .filter(|module_id| !current.contains_key(*module_id))
6187            .cloned()
6188            .collect::<Vec<_>>();
6189        let removed = current
6190            .keys()
6191            .filter(|module_id| !configured.contains_key(*module_id))
6192            .cloned()
6193            .collect::<Vec<_>>();
6194        let mut changed_pending_reload = Vec::new();
6195        let mut configuration_changes = BTreeSet::new();
6196        let mut enabled_changes = BTreeSet::new();
6197        let mut unchanged = 0_u32;
6198
6199        for (module_id, configured_module) in &configured {
6200            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6201            else {
6202                continue;
6203            };
6204            // Compare the whole launch spec so a future launch field cannot
6205            // accidentally become a live-only policy change. Health is stored
6206            // separately and applies live without replacing the process.
6207            let launch_changed = *current_spec != configured_module.module_spec();
6208            let configuration_changed =
6209                launch_changed || *current_health != configured_module.health;
6210            let enabled_changed = *current_enabled != configured_module.enabled;
6211            if configuration_changed {
6212                configuration_changes.insert(module_id.clone());
6213            }
6214            if launch_changed {
6215                changed_pending_reload.push(module_id.clone());
6216            }
6217            if enabled_changed {
6218                enabled_changes.insert(module_id.clone());
6219            }
6220            if !configuration_changed && !enabled_changed {
6221                unchanged = unchanged.saturating_add(1);
6222            }
6223        }
6224        Ok(RescanPlan {
6225            current,
6226            configured,
6227            added,
6228            removed,
6229            changed_pending_reload,
6230            configuration_changes,
6231            enabled_changes,
6232            unchanged,
6233        })
6234    }
6235
6236    /// Reconcile the running module set against the configured one.
6237    ///
6238    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6239    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6240    /// deliberately shares this function with the executing path rather than
6241    /// computing the same diff somewhere else -- two implementations of one
6242    /// decision agree until they do not, and the whole value of a preview is that
6243    /// it describes the operation that will actually run.
6244    async fn reconcile_supervised_modules(
6245        &self,
6246        supervisor: &Supervisor,
6247        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6248        preview: bool,
6249        requester: Principal,
6250    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6251        let plan = self.plan_rescan(configured_modules)?;
6252
6253        // Everything above this point is pure computation over two snapshots.
6254        // Everything below MUTATES. The preview returns here so the boundary is a
6255        // single early return rather than a condition repeated at each mutation
6256        // site, where one missed guard would apply part of a change the caller was
6257        // told would not happen.
6258        if preview {
6259            return Ok((plan.preview_result(), Vec::new()));
6260        }
6261        let changed = plan
6262            .configuration_changes
6263            .union(&plan.enabled_changes)
6264            .cloned()
6265            .collect::<Vec<_>>();
6266        let RescanPlan {
6267            current,
6268            configured,
6269            added,
6270            removed,
6271            changed_pending_reload,
6272            configuration_changes,
6273            enabled_changes,
6274            unchanged,
6275        } = plan;
6276        // The staged generation moves before the first change rather than
6277        // after the last, so a rescan that fails partway still invalidates
6278        // every earlier `supervisor.rescan_checked` preview: part of its diff
6279        // may have run.
6280        if !(added.is_empty()
6281            && removed.is_empty()
6282            && configuration_changes.is_empty()
6283            && enabled_changes.is_empty())
6284        {
6285            if let Some(context) = &self.rescan {
6286                context
6287                    .generation
6288                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6289            }
6290        }
6291
6292        for module_id in &removed {
6293            let module = &current
6294                .get(module_id)
6295                .expect("removed module came from current supervisor state")
6296                .0;
6297            module
6298                .retire_requested(Some(requester.clone()))
6299                .await
6300                .map_err(|err| {
6301                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6302                })?;
6303            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6304            //
6305            // `handle_route_open` resolves an absent module in three steps:
6306            // registry, then supervisor status, then tombstone. Retiring first
6307            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6308            // went with the teardown above, the supervisor entry went with
6309            // `retire`, and the tombstone does not exist yet -- so a route.open
6310            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6311            // it") for a module that was deliberately removed and whose caller
6312            // should get `module_removed` (TERMINAL, carrying a removal age).
6313            //
6314            // Writing the tombstone first closes it: during the window the
6315            // supervisor entry still answers, so the caller gets
6316            // `target_unavailable` -- retryable, and TRUE, because the module
6317            // is mid-teardown. After both statements it is `module_removed`.
6318            // No instant remains where a removed module reads as one that
6319            // never existed.
6320            //
6321            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6322            // the absence of a test beside a fix invites deletion: these are
6323            // two sync statements with no await between them, so reaching the
6324            // window needs a second worker thread to land exactly between them
6325            // and there is no hook to force it. MEASURED: the 25 daemon_config
6326            // tests pass identically with the old order and the new one, so
6327            // the existing suite cannot see this and a green run is not
6328            // evidence either way. What the suite does hold is the
6329            // post-condition -- a removed module answers `module_removed` --
6330            // which this preserves.
6331            //
6332            // Found by an Athena panel reading the shipped tree against a
6333            // design note (2026-09-19), as the one concrete instance of that
6334            // note's class that survived contact with source. Direction is
6335            // benign: retryable where terminal was intended, never the reverse.
6336            self.supervisor.record_rescan_removal(module_id);
6337            self.supervisor.retire(module_id);
6338            self.route_outages.forget(module_id);
6339        }
6340
6341        for module_id in configured.keys() {
6342            let Some((module, _, _, _)) = current.get(module_id) else {
6343                continue;
6344            };
6345            let configured_module = configured
6346                .get(module_id)
6347                .expect("configured module id came from configured map");
6348            if configuration_changes.contains(module_id) {
6349                module
6350                    .update_configuration(
6351                        configured_module.module_spec(),
6352                        configured_module.health.clone(),
6353                        configured_module.drain_timeout_ms,
6354                    )
6355                    .await
6356                    .map_err(|err| {
6357                        format!(
6358                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6359                        )
6360                    })?;
6361            }
6362            if enabled_changes.contains(module_id) {
6363                // A rescan that starts or stops a module applies an operator's
6364                // edit to the config, so the resulting outage was asked for.
6365                self.route_outages.mark_operator_action(module_id);
6366                module
6367                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6368                    .await
6369                    .map_err(|err| {
6370                        self.route_outages.operator_action_ended_unrefused(module_id);
6371                        format!(
6372                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6373                            configured_module.enabled
6374                        )
6375                    })?;
6376            }
6377        }
6378
6379        for module_id in &added {
6380            let configured_module = configured
6381                .get(module_id)
6382                .expect("added module id came from configured map");
6383            supervisor
6384                .supervise_configured_requested(
6385                    configured_module.module_spec(),
6386                    configured_module.enabled,
6387                    configured_module.health.clone(),
6388                    configured_module.drain_timeout_ms,
6389                    configured_module.restart,
6390                    configured_module.register_timeout,
6391                    Some(requester.clone()),
6392                )
6393                .map_err(|err| {
6394                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6395                })?;
6396        }
6397
6398        Ok((
6399            SupervisorRescanResult {
6400                added,
6401                removed,
6402                changed_pending_reload,
6403                enabled_changes: enabled_changes.iter().cloned().collect(),
6404                unchanged,
6405                preview: false,
6406                // Filled by the caller, which is the only layer that can see the
6407                // previous config to diff against.
6408                restart_required: Vec::new(),
6409                capability_warnings: Vec::new(),
6410                module_actions: Vec::new(),
6411                earlier_pending: Vec::new(),
6412                expect: None,
6413            },
6414            changed,
6415        ))
6416    }
6417
6418    async fn handle_supervisor_set_enabled(
6419        &self,
6420        frame: Frame,
6421        module_id: String,
6422        enabled: bool,
6423        requester: Principal,
6424    ) -> Result<Vec<Frame>, RouterError> {
6425        let operation_lock = self.supervisor.operation_lock();
6426        let _operation_guard = operation_lock.lock().await;
6427        let Some(module) = self.supervisor.get(&module_id) else {
6428            return Ok(vec![control_error_frame(
6429                &frame,
6430                "unknown_module",
6431                format!("module_id '{module_id}' is not supervised"),
6432            )?]);
6433        };
6434
6435        // Enabling counts as well as disabling: a module an operator starts
6436        // is refused until it registers, and that wait was asked for.
6437        self.route_outages.mark_operator_action(&module_id);
6438        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6439            Ok(applied) => applied,
6440            Err(err) => {
6441                self.route_outages
6442                    .operator_action_ended_unrefused(&module_id);
6443                return Ok(vec![control_error_frame(
6444                    &frame,
6445                    "target_unavailable",
6446                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6447                )?]);
6448            }
6449        };
6450        if !applied {
6451            // Already in the requested state: nothing was made unavailable,
6452            // so the mark must not outlive this request.
6453            self.route_outages
6454                .operator_action_ended_unrefused(&module_id);
6455        }
6456
6457        self.capability_evaluator.wake_deadline_loop();
6458        self.refresh_capability_requirements();
6459        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6460        Ok(vec![control_response_body_frame(
6461            &frame,
6462            &response,
6463            "ClientControlResponse::SupervisorAck",
6464        )?])
6465    }
6466
6467    async fn handle_supervisor_health_probe(
6468        &self,
6469        frame: Frame,
6470        module_id: String,
6471    ) -> Result<Vec<Frame>, RouterError> {
6472        self.refresh_capability_requirements();
6473        let Some(registration) = self
6474            .registry
6475            .get_module(&module_id)
6476            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6477        else {
6478            return Ok(vec![control_error_frame(
6479                &frame,
6480                "unknown_module",
6481                format!("module_id '{module_id}' is not registered"),
6482            )?]);
6483        };
6484
6485        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6486        // named for it. Making `module_registration_grants_op` return false
6487        // unconditionally reddens five tests, and every one is named for something
6488        // else -- capability relay, probe/bind demultiplexing, supervision-only
6489        // probing. They exercise a successful advertisement check on the way to their
6490        // own subject.
6491        //
6492        // Real protection, fragile in a specific way: narrowing any of those tests to
6493        // focus on its stated subject would silently remove coverage nobody knows
6494        // they are carrying. Recorded here rather than as a sixth test, because the
6495        // useful fact is WHICH tests hold the guard up -- a new test would add
6496        // coverage without telling the next person what the existing ones quietly do.
6497        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6498        {
6499            return Ok(vec![control_error_frame(
6500                &frame,
6501                "health_not_advertised",
6502                format!("module_id '{module_id}' did not advertise health.check"),
6503            )?]);
6504        }
6505
6506        let deadline = Instant::now() + self.health_probe_timeout;
6507        let pending = match self.forwarding.begin_module_control_rpc_for(
6508            &module_id,
6509            MODULE_CONTROL_OP_HEALTH_CHECK,
6510            deadline,
6511        ) {
6512            Ok(pending) => pending,
6513            Err(err) => {
6514                return Ok(vec![control_error_frame(
6515                    &frame,
6516                    forwarding_error_code(&err),
6517                    err.to_string(),
6518                )?])
6519            }
6520        };
6521
6522        let PendingModuleControlRpc {
6523            endpoint,
6524            module_sink,
6525            negotiated_ver,
6526            corr: probe_corr,
6527            receiver,
6528        } = pending;
6529        let mut guard =
6530            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6531        let probe_body =
6532            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6533                RouterError::backend(
6534                    0,
6535                    frame.header.corr,
6536                    format!("failed to encode health.check request: {err}"),
6537                )
6538            })?;
6539        let probe_frame = Frame::build_with_version(
6540            negotiated_ver,
6541            FrameType::Request,
6542            control_flags(),
6543            0,
6544            0,
6545            probe_corr,
6546            probe_body,
6547        )
6548        .map_err(RouterError::FrameBuild)?;
6549
6550        // Queue admission and the reply share one deadline, so a module that
6551        // stops reading cannot park the requesting connection's reader forever.
6552        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6553            Ok(Ok(())) => {}
6554            Ok(Err(err)) => {
6555                return Ok(vec![control_error_frame(
6556                    &frame,
6557                    "target_unavailable",
6558                    err.to_string(),
6559                )?]);
6560            }
6561            Err(_) => {
6562                return Ok(vec![control_error_frame(
6563                    &frame,
6564                    "module_timeout",
6565                    format!(
6566                        "module_id '{module_id}' did not answer health.check within {:?}",
6567                        self.health_probe_timeout
6568                    ),
6569                )?]);
6570            }
6571        }
6572
6573        match timeout_at(deadline, receiver).await {
6574            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6575                guard.disarm();
6576                let Some(report) = response.health_report() else {
6577                    return Ok(vec![control_error_frame(
6578                        &frame,
6579                        "invalid_control_body",
6580                        "health.check RPC returned a non-health response",
6581                    )?]);
6582                };
6583                // Metrics go out whole here. The supervisor's cached snapshot
6584                // caps this blob (see truncate_health_metrics), and this path
6585                // exists precisely to answer without that cap -- so applying it
6586                // here would leave no way to see what the cached view drops.
6587                let HealthReport {
6588                    status,
6589                    detail,
6590                    metrics,
6591                } = report;
6592                let capability_detail = self
6593                    .capability_evaluator
6594                    .required_problem_detail(&module_id);
6595                let response = ClientControlResponse::SupervisorHealthProbe {
6596                    module_id,
6597                    status,
6598                    detail: append_capability_problem_detail(detail, capability_detail),
6599                    metrics,
6600                };
6601                Ok(vec![control_response_body_frame(
6602                    &frame,
6603                    &response,
6604                    "ClientControlResponse::SupervisorHealthProbe",
6605                )?])
6606            }
6607            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6608                guard.disarm();
6609                Ok(vec![control_error_body_frame(&frame, body)?])
6610            }
6611            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6612                guard.disarm();
6613                Ok(vec![control_error_frame(
6614                    &frame,
6615                    "target_unavailable",
6616                    message,
6617                )?])
6618            }
6619            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6620                guard.disarm();
6621                Ok(vec![control_error_frame(
6622                    &frame,
6623                    "invalid_control_body",
6624                    message,
6625                )?])
6626            }
6627            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6628                guard.disarm();
6629                Ok(vec![control_error_frame(
6630                    &frame,
6631                    "invalid_control_body",
6632                    format!("expected module-control op '{expected}', got '{actual}'"),
6633                )?])
6634            }
6635            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6636                guard.disarm();
6637                Ok(vec![control_error_frame(
6638                    &frame,
6639                    "module_timeout",
6640                    format!(
6641                        "module_id '{module_id}' answered health.check after {:?}",
6642                        self.health_probe_timeout
6643                    ),
6644                )?])
6645            }
6646            Ok(Err(_)) => Ok(vec![control_error_frame(
6647                &frame,
6648                "target_unavailable",
6649                "health.check waiter was canceled before the module responded",
6650            )?]),
6651            Err(_) => Ok(vec![control_error_frame(
6652                &frame,
6653                "module_timeout",
6654                format!(
6655                    "module_id '{module_id}' did not answer health.check within {:?}",
6656                    self.health_probe_timeout
6657                ),
6658            )?]),
6659        }
6660    }
6661
6662    fn supervisor_status(
6663        &self,
6664        module_id: &str,
6665        corr: u64,
6666    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6667        self.supervisor
6668            .get(module_id)
6669            .map(|module| {
6670                let warming = module.is_warming_for_control("status").map_err(|err| {
6671                    RouterError::backend(
6672                        0,
6673                        corr,
6674                        format!(
6675                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6676                        ),
6677                    )
6678                })?;
6679                module.status_for_control("status").map_err(|err| {
6680                    RouterError::backend(
6681                        0,
6682                        corr,
6683                        format!(
6684                            "failed to read supervisor status for module_id '{module_id}': {err}"
6685                        ),
6686                    )
6687                }).map(|status| (status, warming))
6688            })
6689            .transpose()
6690    }
6691
6692    fn guard_module_control_op(
6693        &self,
6694        frame: &Frame,
6695        module_id: &str,
6696        op: &str,
6697    ) -> Result<Option<Frame>, RouterError> {
6698        if self.module_grants_op(module_id, op, frame.header.corr)? {
6699            return Ok(None);
6700        }
6701
6702        Ok(Some(control_error_frame(
6703            frame,
6704            "op_not_allowed",
6705            format!("module_id '{module_id}' did not grant control op '{op}'"),
6706        )?))
6707    }
6708
6709    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6710        let Some(registration) = self
6711            .registry
6712            .get_module(module_id)
6713            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6714        else {
6715            return Ok(false);
6716        };
6717        Ok(module_registration_grants_op(&registration.control_ops, op))
6718    }
6719
6720    fn handle_status_update(
6721        &self,
6722        endpoint: ModuleEndpointId,
6723        frame: Frame,
6724    ) -> Result<Vec<Frame>, RouterError> {
6725        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6726            Ok(update) => update,
6727            Err(err) => {
6728                // Forward-compat: a newer module may push a channel-0 op this subc
6729                // version doesn't know. The control contract says unknown push ops
6730                // are IGNORED, never answered with an error. Only a malformed body
6731                // for an op we DO know is a real error worth surfacing.
6732                if is_known_module_push_op(&frame.body) {
6733                    return Ok(vec![control_error_frame(
6734                        &frame,
6735                        "invalid_control_body",
6736                        format!("malformed module control push body: {err}"),
6737                    )?]);
6738                }
6739                return Ok(Vec::new());
6740            }
6741        };
6742
6743        match update {
6744            ModuleControlPush::RouteStatus {
6745                route_channel,
6746                route_epoch,
6747                status,
6748            } => {
6749                self.forwarding
6750                    .cache_status(endpoint, route_channel, route_epoch, status)
6751                    .map_err(RouterError::Forwarding)?;
6752            }
6753        }
6754        Ok(Vec::new())
6755    }
6756
6757    fn handle_route_poll(
6758        &self,
6759        ctx: &RouteCtx,
6760        frame: Frame,
6761        route_channel: u16,
6762        route_epoch: u32,
6763        kind: PollKind,
6764    ) -> Result<Vec<Frame>, RouterError> {
6765        let snapshot = self
6766            .forwarding
6767            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6768            .map_err(RouterError::Forwarding)?;
6769        let response = match (kind, snapshot) {
6770            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6771                ClientControlResponse::RoutePoll {
6772                    route_channel,
6773                    route_epoch,
6774                    status,
6775                    live: None,
6776                }
6777            }
6778            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6779                route_channel,
6780                route_epoch,
6781                status: None,
6782                live: None,
6783            },
6784            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6785                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6786                // is what makes reporting `true` correct rather than a
6787                // confident guess. `process_live` returns None only when the
6788                // module id has no supervisor snapshot at all -- an
6789                // externally-started module the daemon did not spawn -- and
6790                // for those the supervisor has no opinion to offer, ever. It
6791                // is never None for a supervised module in an unknown state:
6792                // a supervised module always has a snapshot, and the answer
6793                // comes from `state == Running && process_alive`.
6794                //
6795                // The route is Bound, so the module completed a HELLO on a
6796                // live connection; "the process this route points at is
6797                // running" is therefore attested by the binding rather than
6798                // assumed. Reporting `false` for an unsupervised module would
6799                // be the actual lie -- it would tell a client its healthy
6800                // route is dead because the daemon does not manage the
6801                // process.
6802                //
6803                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6804                // module whose liveness is genuinely unknown, e.g. a snapshot
6805                // that has not been populated yet -- THIS DEFAULT BECOMES
6806                // WRONG and must split: unsupervised stays true, unknown
6807                // becomes null so the client can tell the two apart. The
6808                // response field is already `Option<bool>`, so the wire can
6809                // carry that distinction today.
6810                let live = self
6811                    .process_liveness
6812                    .as_ref()
6813                    .and_then(|source| source.process_live(&module_id))
6814                    .unwrap_or(true);
6815                ClientControlResponse::RoutePoll {
6816                    route_channel,
6817                    route_epoch,
6818                    status: None,
6819                    live: Some(live),
6820                }
6821            }
6822            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6823                route_channel,
6824                route_epoch,
6825                status: None,
6826                live: Some(false),
6827            },
6828        };
6829
6830        Ok(vec![control_response_body_frame(
6831            &frame,
6832            &response,
6833            "ClientControlResponse::RoutePoll",
6834        )?])
6835    }
6836
6837    pub(crate) fn observe_module_control_completion(
6838        &self,
6839        completion: ModuleControlRpcCompletion,
6840    ) -> bool {
6841        match completion {
6842            ModuleControlRpcCompletion::Unknown => false,
6843            ModuleControlRpcCompletion::Settled => true,
6844            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6845                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6846                info!(
6847                    module_id = %module_id,
6848                    latency_ms,
6849                    "late health.check answer proves the module is alive"
6850                );
6851                match self
6852                    .supervisor
6853                    .record_late_health_answer(&module_id, latency_ms)
6854                {
6855                    Ok(true) => {}
6856                    Ok(false) => debug!(
6857                        module_id = %module_id,
6858                        latency_ms,
6859                        "late health.check answer has no active supervisor snapshot"
6860                    ),
6861                    Err(err) => warn!(
6862                        module_id = %module_id,
6863                        latency_ms,
6864                        error = %err,
6865                        "failed to record late health.check answer"
6866                    ),
6867                }
6868                true
6869            }
6870        }
6871    }
6872
6873    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6874    /// the module connection whose frame is being handled, or to the client that
6875    /// relay was opened for.
6876    ///
6877    /// This runs on the MODULE connection's frame handler, where returning `Err`
6878    /// ends that connection -- and a module connection carries every client's
6879    /// routes to that module, so ending it costs the whole fleet its tools.
6880    /// `ConnectionClosing` carries the id of the connection that is closing, and
6881    /// when that id is a CLIENT's, the condition is entirely about that one
6882    /// client's route.open. A client-scoped condition has no authority over a
6883    /// shared module connection, so it is logged and the single relay is dropped:
6884    /// the client is going away, and `complete_pending_relay` already removed the
6885    /// relay before failing, so there is nothing left to settle. Anything that
6886    /// relay still reserved is released by that client's own connection teardown,
6887    /// which is already under way -- that is what "closing" means.
6888    ///
6889    /// Every other failure is a statement about THIS connection and stays fatal:
6890    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6891    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6892    /// frames correctly.
6893    fn refuse_to_end_module_connection_for_a_client(
6894        &self,
6895        module_connection_id: ConnectionId,
6896        corr: u64,
6897        err: ForwardingError,
6898    ) -> Result<(), RouterError> {
6899        if let ForwardingError::ConnectionClosing { connection_id } = err {
6900            if connection_id != module_connection_id {
6901                warn!(
6902                    module_connection_id = module_connection_id.get(),
6903                    client_connection_id = connection_id.get(),
6904                    corr,
6905                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6906                );
6907                return Ok(());
6908            }
6909        }
6910        Err(RouterError::Forwarding(err))
6911    }
6912
6913    fn handle_module_relay_response(
6914        &self,
6915        connection_id: ConnectionId,
6916        frame: Frame,
6917    ) -> Result<Vec<Frame>, RouterError> {
6918        let mut secondary_error = None;
6919        let outcome = match frame.header.ty {
6920            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6921                Ok(probe) if probe.op == "route.bind" => {
6922                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6923                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6924                            RouteBindRelayOutcome::Accepted
6925                        }
6926                        Ok(other) => {
6927                            let message =
6928                                format!("route.bind response carried unexpected body: {other:?}");
6929                            secondary_error = Some(control_error_frame(
6930                                &frame,
6931                                "invalid_control_body",
6932                                message.clone(),
6933                            )?);
6934                            RouteBindRelayOutcome::ModuleGone(message)
6935                        }
6936                        Err(err) => {
6937                            let message = format!("malformed route.bind response body: {err}");
6938                            secondary_error = Some(control_error_frame(
6939                                &frame,
6940                                "invalid_control_body",
6941                                message.clone(),
6942                            )?);
6943                            RouteBindRelayOutcome::ModuleGone(message)
6944                        }
6945                    }
6946                }
6947                Ok(probe) => {
6948                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6949                    {
6950                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6951                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6952                            "malformed {} response body: {err}",
6953                            probe.op
6954                        )),
6955                    };
6956                    let completion = self
6957                        .forwarding
6958                        .complete_module_control_rpc(
6959                            connection_id,
6960                            frame.header.corr,
6961                            Some(&probe.op),
6962                            outcome,
6963                        )
6964                        .map_err(RouterError::Forwarding)?;
6965                    if !self.observe_module_control_completion(completion) {
6966                        debug!(
6967                            connection_id = connection_id.get(),
6968                            corr = frame.header.corr,
6969                            op = %probe.op,
6970                            "dropping late or unknown module-control RPC response"
6971                        );
6972                    }
6973                    return Ok(Vec::new());
6974                }
6975                Err(err) => {
6976                    if let Some(expected_op) = self
6977                        .forwarding
6978                        .pending_module_control_op(connection_id, frame.header.corr)
6979                        .map_err(RouterError::Forwarding)?
6980                    {
6981                        let completion = self
6982                            .forwarding
6983                            .complete_module_control_rpc(
6984                                connection_id,
6985                                frame.header.corr,
6986                                None,
6987                                ModuleControlRpcOutcome::MalformedResponse(format!(
6988                                    "malformed {expected_op} response body: {err}"
6989                                )),
6990                            )
6991                            .map_err(RouterError::Forwarding)?;
6992                        if !self.observe_module_control_completion(completion) {
6993                            debug!(
6994                                connection_id = connection_id.get(),
6995                                corr = frame.header.corr,
6996                                "dropping late malformed module-control RPC response"
6997                            );
6998                        }
6999                        return Ok(Vec::new());
7000                    }
7001                    let message = format!("malformed route.bind response body: {err}");
7002                    secondary_error = Some(control_error_frame(
7003                        &frame,
7004                        "invalid_control_body",
7005                        message.clone(),
7006                    )?);
7007                    RouteBindRelayOutcome::ModuleGone(message)
7008                }
7009            },
7010            FrameType::Error => {
7011                if self
7012                    .forwarding
7013                    .pending_module_control_op(connection_id, frame.header.corr)
7014                    .map_err(RouterError::Forwarding)?
7015                    .is_some()
7016                {
7017                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7018                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7019                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7020                            "malformed module-control ERROR body: {err}"
7021                        )),
7022                    };
7023                    let completion = self
7024                        .forwarding
7025                        .complete_module_control_rpc(
7026                            connection_id,
7027                            frame.header.corr,
7028                            None,
7029                            outcome,
7030                        )
7031                        .map_err(RouterError::Forwarding)?;
7032                    if !self.observe_module_control_completion(completion) {
7033                        debug!(
7034                            connection_id = connection_id.get(),
7035                            corr = frame.header.corr,
7036                            "dropping late or unknown module-control RPC error"
7037                        );
7038                    }
7039                    return Ok(Vec::new());
7040                }
7041                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7042                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7043                    Err(err) => {
7044                        let message = format!("malformed route.bind ERROR body: {err}");
7045                        secondary_error = Some(control_error_frame(
7046                            &frame,
7047                            "invalid_control_body",
7048                            message.clone(),
7049                        )?);
7050                        RouteBindRelayOutcome::ModuleGone(message)
7051                    }
7052                }
7053            }
7054            ty => {
7055                return Ok(vec![control_error_frame(
7056                    &frame,
7057                    "unsupported_control_frame",
7058                    format!("unsupported module channel-0 frame {ty:?}"),
7059                )?])
7060            }
7061        };
7062
7063        let settled =
7064            self.forwarding
7065                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7066        let completion = match settled {
7067            Ok(completion) => completion,
7068            Err(err) => {
7069                self.refuse_to_end_module_connection_for_a_client(
7070                    connection_id,
7071                    frame.header.corr,
7072                    err,
7073                )?;
7074                return Ok(secondary_error.into_iter().collect());
7075            }
7076        };
7077        if let Some(target) = completion.abandoned.as_ref() {
7078            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7079        }
7080        if !completion.settled {
7081            debug!(
7082                connection_id = connection_id.get(),
7083                corr = frame.header.corr,
7084                frame_type = ?frame.header.ty,
7085                "dropping late or unknown route.bind relay response"
7086            );
7087        }
7088        Ok(secondary_error.into_iter().collect())
7089    }
7090
7091    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7092        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7093        // GOODBYE ends the connection's logical session even when its socket
7094        // stays open. Use disconnect teardown so verdicts, client notices and
7095        // scope authority are released at the same lifecycle boundary.
7096        self.cleanup_connection_with_end_reason(
7097            connection_id,
7098            RegistrationEndReason::ExplicitGoodbye,
7099        )
7100        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7101        Ok(Vec::new())
7102    }
7103}
7104
7105impl Default for ControlHandler {
7106    fn default() -> Self {
7107        Self::new(Arc::new(Registry::default()))
7108    }
7109}
7110
7111impl crate::supervise::SwapPromotionObserver for ControlHandler {
7112    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7113        self.apply_registration_capabilities(registration);
7114    }
7115}
7116
7117fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7118    CapabilityRequirementStatus {
7119        consumer: status.consumer,
7120        capability: status.capability,
7121        need: match status.need {
7122            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7123            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7124        },
7125        verdict: status.verdict.as_str().to_string(),
7126        episode_seq: status.episode_seq,
7127        config_satisfiable: status.config_satisfiable,
7128        runtime_available: status.runtime_available,
7129        detail: status.detail,
7130    }
7131}
7132
7133fn append_capability_problem_detail(
7134    detail: Option<String>,
7135    capability_detail: Option<String>,
7136) -> Option<String> {
7137    match (detail, capability_detail) {
7138        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7139        (Some(detail), None) => Some(detail),
7140        (None, Some(capability_detail)) => Some(capability_detail),
7141        (None, None) => None,
7142    }
7143}
7144
7145fn subc_ops() -> Vec<String> {
7146    SUBC_CONTROL_OPS
7147        .iter()
7148        .map(|op| (*op).to_string())
7149        .collect()
7150}
7151
7152fn module_subc_ops() -> Vec<String> {
7153    SUBC_CONTROL_OPS
7154        .iter()
7155        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7156        .map(|op| (*op).to_string())
7157        .collect()
7158}
7159
7160#[cfg(test)]
7161fn module_baseline_control_ops() -> Vec<String> {
7162    MODULE_BASELINE_CONTROL_OPS
7163        .iter()
7164        .map(|op| (*op).to_string())
7165        .collect()
7166}
7167
7168fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7169    let mut seen = HashSet::new();
7170    let mut effective = Vec::new();
7171    for op in MODULE_BASELINE_CONTROL_OPS {
7172        if seen.insert((*op).to_string()) {
7173            effective.push((*op).to_string());
7174        }
7175    }
7176    for op in declared.unwrap_or_default() {
7177        if seen.insert(op.clone()) {
7178            effective.push(op);
7179        }
7180    }
7181    effective
7182}
7183
7184fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7185    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7186}
7187
7188fn target_module_id(target: &RouteTarget) -> &str {
7189    match target {
7190        RouteTarget::ToolProvider { module_id }
7191        | RouteTarget::ManagementSurface { module_id }
7192        | RouteTarget::InternalService { module_id, .. } => module_id,
7193    }
7194}
7195
7196fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7197    roles.iter().any(|role| match (target, role) {
7198        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7199        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7200        (
7201            RouteTarget::InternalService { service_id, .. },
7202            ProviderRole::InternalService {
7203                service_id: provided,
7204                ..
7205            },
7206        ) => service_id == provided,
7207        _ => false,
7208    })
7209}
7210
7211fn is_routable_role(role: &ProviderRole) -> bool {
7212    matches!(
7213        role,
7214        ProviderRole::ToolProvider { .. }
7215            | ProviderRole::ManagementSurface { .. }
7216            | ProviderRole::InternalService { .. }
7217    )
7218}
7219
7220#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7221enum ControlRequestBodyError {
7222    UnknownOp,
7223    InvalidBody,
7224}
7225
7226#[derive(Debug, Deserialize)]
7227struct ControlOpProbe {
7228    op: String,
7229}
7230
7231/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7232/// this set is treated as a forward-compat unknown and ignored rather than errored.
7233const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7234
7235fn is_known_module_push_op(body: &[u8]) -> bool {
7236    serde_json::from_slice::<ControlOpProbe>(body)
7237        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7238        .unwrap_or(false)
7239}
7240
7241fn is_known_module_request_op(body: &[u8]) -> bool {
7242    serde_json::from_slice::<ControlOpProbe>(body)
7243        .map(|probe| is_module_to_subc_op(&probe.op))
7244        .unwrap_or(false)
7245}
7246
7247fn is_module_to_subc_op(op: &str) -> bool {
7248    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7249}
7250
7251fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7252    debug!(
7253        op = %op,
7254        connection_id = connection_id.get(),
7255        corr,
7256        "control dispatch"
7257    );
7258}
7259
7260fn log_slow_control_dispatch(
7261    dispatch_started_at: Option<StdInstant>,
7262    op: &'static str,
7263    connection_id: ConnectionId,
7264    corr: u64,
7265    route_open_timings: Option<&RouteOpenTimings>,
7266) {
7267    let Some(dispatch_started_at) = dispatch_started_at else {
7268        return;
7269    };
7270    // Use the same monotonic clock as the phase timers and relay deadline.
7271    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7272    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7273        if let Some(timings) = route_open_timings {
7274            warn!(
7275                op = %op,
7276                connection_id = connection_id.get(),
7277                corr,
7278                elapsed_ms = elapsed.as_millis() as u64,
7279                module_id = ?timings.module_id,
7280                principal = %timings.principal,
7281                admission_ms = timings.admission.as_millis() as u64,
7282                project_root_ms = timings.project_root.as_millis() as u64,
7283                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7284                bind_send_ms = timings.bind_send.as_millis() as u64,
7285                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7286                "slow control dispatch"
7287            );
7288            return;
7289        }
7290        warn!(
7291            op = %op,
7292            connection_id = connection_id.get(),
7293            corr,
7294            elapsed_ms = elapsed.as_millis() as u64,
7295            "slow control dispatch"
7296        );
7297    }
7298}
7299
7300/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7301#[cfg(test)]
7302type TestCallerLookup =
7303    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7304
7305/// Every new client operation needs an explicit privilege decision here.
7306fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7307    match request {
7308        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7309            Some(PrivilegedOperation::Restart { module_id })
7310        }
7311        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7312            Some(PrivilegedOperation::Swap { module_id })
7313        }
7314        ClientControlRequest::SupervisorReload { module_id } => {
7315            Some(PrivilegedOperation::Reload { module_id })
7316        }
7317        ClientControlRequest::SupervisorRescan { preview } => {
7318            if *preview {
7319                None
7320            } else {
7321                Some(PrivilegedOperation::Rescan)
7322            }
7323        }
7324        // Only the well-formed apply asks for approval. A preview changes
7325        // nothing, and any other combination is refused before it touches
7326        // anything, so prompting the operator for it would ask about a request
7327        // that cannot run.
7328        ClientControlRequest::SupervisorRescanChecked {
7329            preview,
7330            candidate,
7331            expect,
7332        } => (!*preview && candidate.is_none() && expect.is_some())
7333            .then_some(PrivilegedOperation::Rescan),
7334        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7335            Some(PrivilegedOperation::SetEnabled {
7336                module_id,
7337                enabled: *enabled,
7338            })
7339        }
7340        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7341            Some(PrivilegedOperation::ReleaseReserved { module_id })
7342        }
7343        ClientControlRequest::ServerDescribe {}
7344        | ClientControlRequest::CatalogList { .. }
7345        | ClientControlRequest::RouteOpen { .. }
7346        | ClientControlRequest::RoutePoll { .. }
7347        | ClientControlRequest::SupervisorList {}
7348        | ClientControlRequest::SupervisorHealth {}
7349        | ClientControlRequest::SupervisorHealthProbe { .. }
7350        | ClientControlRequest::SupervisorRoutes { .. }
7351        | ClientControlRequest::SupervisorProvenance { .. }
7352        | ClientControlRequest::SupervisorStderrTail { .. }
7353        | ClientControlRequest::SupervisorTerminals { .. }
7354        | ClientControlRequest::SupervisorSpawnSnapshot {}
7355        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7356    }
7357}
7358
7359fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7360    match request {
7361        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7362        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7363        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7364        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7365        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7366        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7367        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7368        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7369        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7370        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7371        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7372        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7373        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7374        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7375        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7376        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7377        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7378        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7379        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7380        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7381    }
7382}
7383
7384/// The caller of one `route.open`, as its `route.open refused` line names it.
7385///
7386/// A connection id alone cannot name a caller after the fact: ids restart at
7387/// every daemon restart, so the same number belongs to different modules
7388/// before and after one. These three fields are what the refused line needs
7389/// to be attributable on its own, and they are rendered with the same helpers
7390/// as the `route.open accepted` line so both directions grep alike.
7391struct RouteOpenCaller {
7392    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
7393    /// when the request presented a consumer identity whose launch nonce did
7394    /// not verify. The claimed module id is never printed as a principal.
7395    principal: String,
7396    /// `none` for an unscoped open; the admitted stamp's label once scope
7397    /// admission issued one; before that, the requested selector's label.
7398    scope: String,
7399    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
7400    /// because it is a string the caller chose.
7401    harness: String,
7402}
7403
7404impl RouteOpenCaller {
7405    fn new(
7406        principal: Option<&Principal>,
7407        scope: Option<&ScopeSelector>,
7408        identity: &BindIdentity,
7409    ) -> Self {
7410        Self {
7411            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7412            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7413            harness: bounded_log_text(&identity.harness),
7414        }
7415    }
7416
7417    /// For a refusal that has no parsed request at hand: names whatever the
7418    /// frame still says, and `unknown` where it says nothing parseable.
7419    fn unknown() -> Self {
7420        Self {
7421            principal: "unknown".to_string(),
7422            scope: "unknown".to_string(),
7423            harness: "unknown".to_string(),
7424        }
7425    }
7426}
7427
7428/// Longest client-chosen text (a scope ref, a harness) a route.open log line
7429/// prints before cutting it.
7430const MAX_LOGGED_TEXT_CHARS: usize = 64;
7431
7432/// `value` Debug-quoted, so control characters land escaped and cannot forge
7433/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
7434/// `…` inside the quotes so one request cannot flood the line.
7435fn bounded_log_text(value: &str) -> String {
7436    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7437    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7438        bounded.push('…');
7439    }
7440    format!("{bounded:?}")
7441}
7442
7443/// The `scope=` field of the `route.open accepted` line, from the stamp that
7444/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7445/// so a ref holding control characters cannot forge a log line, and cut to 64
7446/// characters with a trailing `…` to bound the line. The bind proof and the
7447/// scope's attributes never appear.
7448fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7449    use subc_protocol::scope::ScopeKind;
7450    let kind = match stamp.kind {
7451        ScopeKind::Head => "head",
7452        ScopeKind::Worker => "worker",
7453        ScopeKind::Ephemeral => "ephemeral",
7454    };
7455    format!(
7456        "{kind}/{}/{}@{}",
7457        principal_label(&stamp.owner),
7458        bounded_log_text(&stamp.scope_ref),
7459        stamp.scope_epoch
7460    )
7461}
7462
7463/// The `scope=` field of a `route.open refused` line for an open refused
7464/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
7465/// The kind is not known yet, so the literal `requested` stands in its place.
7466/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
7467/// owner it has not been checked against the configured modules: a quoted
7468/// owner means the value was never validated. The bind proof never appears.
7469fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7470    let epoch = selector
7471        .scope_epoch
7472        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7473    format!(
7474        "requested/{}/{}@{epoch}",
7475        bounded_log_text(&principal_label(&selector.owner)),
7476        bounded_log_text(&selector.scope_ref),
7477    )
7478}
7479
7480fn principal_label(principal: &Principal) -> String {
7481    match principal {
7482        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7483        Principal::Direct => "direct".to_string(),
7484        other => format!("{other:?}"),
7485    }
7486}
7487
7488fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7489    match request {
7490        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7491        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7492        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7493        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7494        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7495    }
7496}
7497
7498fn parse_client_control_request(
7499    body: &[u8],
7500) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7501    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7502        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7503            Ok(probe)
7504                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7505                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7506            {
7507                ControlRequestBodyError::InvalidBody
7508            }
7509            Ok(_) => ControlRequestBodyError::UnknownOp,
7510            Err(_) => ControlRequestBodyError::InvalidBody,
7511        };
7512        (err, classification)
7513    })
7514}
7515
7516fn parse_module_control_request_from_module(
7517    body: &[u8],
7518) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7519    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7520        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7521            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7522            Ok(_) => ControlRequestBodyError::UnknownOp,
7523            Err(_) => ControlRequestBodyError::InvalidBody,
7524        };
7525        (err, classification)
7526    })
7527}
7528
7529#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7530enum ProviderRoleKind {
7531    ToolProvider,
7532    PipelineStage,
7533    ManagementSurface,
7534    InternalService,
7535}
7536
7537fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7538    match role {
7539        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7540        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7541        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7542        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7543    }
7544}
7545
7546fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7547    roles.iter().map(provider_role_kind).collect()
7548}
7549
7550/// Most refused scope records named individually in the log per sync; the
7551/// `refused` count on the accepted line is always complete.
7552const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7553
7554/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7555#[derive(Debug, Default, PartialEq, Eq)]
7556struct ScopeOutcomeCounts {
7557    created: usize,
7558    replaced: usize,
7559    updated: usize,
7560    unchanged: usize,
7561    refused: usize,
7562}
7563
7564impl ScopeOutcomeCounts {
7565    fn of(results: &[ScopeRecordResult]) -> Self {
7566        let mut counts = Self::default();
7567        for result in results {
7568            let slot = match result.outcome {
7569                ScopeRecordOutcome::Created => &mut counts.created,
7570                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7571                ScopeRecordOutcome::Updated => &mut counts.updated,
7572                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7573                ScopeRecordOutcome::Refused => &mut counts.refused,
7574            };
7575            *slot += 1;
7576        }
7577        counts
7578    }
7579}
7580
7581#[cfg(test)]
7582mod scope_outcome_count_tests {
7583    use super::*;
7584
7585    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7586        ScopeRecordResult {
7587            scope_ref: "r".to_string(),
7588            scope_epoch: 1,
7589            outcome,
7590            code: None,
7591            message: None,
7592            version: None,
7593            parent_state: None,
7594        }
7595    }
7596
7597    /// Each outcome lands in its own count, so a refused record can never be
7598    /// hidden inside the total the log already printed.
7599    #[test]
7600    fn every_outcome_is_counted_in_its_own_field() {
7601        let results = [
7602            result(ScopeRecordOutcome::Created),
7603            result(ScopeRecordOutcome::Created),
7604            result(ScopeRecordOutcome::Replaced),
7605            result(ScopeRecordOutcome::Updated),
7606            result(ScopeRecordOutcome::Unchanged),
7607            result(ScopeRecordOutcome::Refused),
7608            result(ScopeRecordOutcome::Refused),
7609            result(ScopeRecordOutcome::Refused),
7610        ];
7611        assert_eq!(
7612            ScopeOutcomeCounts::of(&results),
7613            ScopeOutcomeCounts {
7614                created: 2,
7615                replaced: 1,
7616                updated: 1,
7617                unchanged: 1,
7618                refused: 3,
7619            }
7620        );
7621    }
7622}
7623
7624/// Return whether a catalog change can create a newly violating live route.
7625/// Removing an attested claim is intentionally excluded: it makes fewer routes
7626/// forbidden and therefore must leave the existing route census untouched.
7627fn capability_census_trigger(
7628    old: Option<&CapabilityDeclarations>,
7629    new: Option<&CapabilityDeclarations>,
7630) -> bool {
7631    let old_provides = old
7632        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7633        .unwrap_or_default();
7634    let old_denies = old
7635        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7636        .unwrap_or_default();
7637    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7638        provides: Vec::new(),
7639        requires: Vec::new(),
7640        must_never_reach: Vec::new(),
7641    });
7642
7643    new.provides
7644        .iter()
7645        .any(|capability| !old_provides.contains(capability))
7646        || new
7647            .must_never_reach
7648            .iter()
7649            .any(|capability| !old_denies.contains(capability))
7650}
7651
7652/// Find the first capability an attested opener denies that an attested target
7653/// claims. Both manifests are live registry records, never cached or client data.
7654fn denied_capability<'a>(
7655    opening_manifest: &'a ModuleManifest,
7656    target_manifest: &ModuleManifest,
7657) -> Option<&'a str> {
7658    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7659    let target_capabilities = target_manifest.capabilities.as_ref()?;
7660    opening_capabilities
7661        .must_never_reach
7662        .iter()
7663        .find(|denied| {
7664            target_capabilities
7665                .provides
7666                .iter()
7667                .any(|provided| provided == *denied)
7668        })
7669        .map(String::as_str)
7670}
7671
7672fn catalog_update_frozen_field_message(
7673    registered: &ModuleManifest,
7674    provides: &[ProviderRole],
7675) -> Option<String> {
7676    let old_has_provides = !registered.provides.is_empty();
7677    let new_has_provides = !provides.is_empty();
7678    if old_has_provides != new_has_provides {
7679        return Some(format!(
7680            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7681            registered.module_id
7682        ));
7683    }
7684
7685    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7686        return Some(format!(
7687            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7688            registered.module_id
7689        ));
7690    }
7691
7692    let registered_concurrency = manifest_concurrency(registered);
7693    let mut candidate = registered.clone();
7694    candidate.provides = provides.to_vec();
7695    let candidate_concurrency = manifest_concurrency(&candidate);
7696    if candidate_concurrency != registered_concurrency {
7697        return Some(format!(
7698            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7699            registered.module_id, registered_concurrency, candidate_concurrency
7700        ));
7701    }
7702
7703    // control_ops live beside the manifest in the HELLO body, not inside
7704    // ModuleManifest, so a provides-only catalog.update cannot change them.
7705    None
7706}
7707
7708fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7709    manifest.provides.iter().any(is_routable_role)
7710}
7711
7712/// Returns the routable-provider concurrency subc should enforce for this manifest.
7713///
7714/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7715/// InternalService has no role-specific concurrency field, so it retains the
7716/// existing ModuleManaged default for backward compatibility.
7717fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7718    manifest
7719        .provides
7720        .iter()
7721        .find_map(|provider| match provider {
7722            ProviderRole::ToolProvider { concurrency, .. }
7723            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7724            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7725        })
7726        .unwrap_or(Concurrency::ModuleManaged)
7727}
7728
7729/// True when the manifest carries a ManagementSurface role whose concurrency
7730/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7731/// bytes because the typed manifest deliberately erases that distinction: the
7732/// default exists for wire compatibility, and this probe exists so the default
7733/// stays observable. Any parse irregularity returns false -- the caller only
7734/// logs, and a malformed body already failed registration upstream.
7735fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7736    let has_management_surface = manifest
7737        .provides
7738        .iter()
7739        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7740    if !has_management_surface {
7741        return false;
7742    }
7743    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7744        return false;
7745    };
7746    let Some(provides) = raw
7747        .get("manifest")
7748        .and_then(|manifest| manifest.get("provides"))
7749        .and_then(serde_json::Value::as_array)
7750    else {
7751        return false;
7752    };
7753    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7754    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7755    // against the management_surface_manifest_without_concurrency golden, not
7756    // recalled (the externally-tagged guess was this function's first bug).
7757    provides.iter().any(|role| {
7758        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7759            && role.get("concurrency").is_none()
7760    })
7761}
7762
7763fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7764    if peer_version != PROTOCOL_VERSION {
7765        return Err(format!(
7766            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7767        ));
7768    }
7769    Ok(PROTOCOL_VERSION)
7770}
7771
7772fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7773    Frame::build_with_version(
7774        response_version(frame),
7775        FrameType::Pong,
7776        frame.header.flags,
7777        0,
7778        0,
7779        frame.header.corr,
7780        Vec::new(),
7781    )
7782    .map_err(RouterError::FrameBuild)
7783}
7784
7785pub(crate) fn control_error_frame(
7786    frame: &Frame,
7787    code: &'static str,
7788    message: impl Into<String>,
7789) -> Result<Frame, RouterError> {
7790    control_error_body_frame(
7791        frame,
7792        ErrorBody {
7793            code: code.to_string(),
7794            message: message.into(),
7795            detail: None,
7796        },
7797    )
7798}
7799
7800fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7801    let body = serde_json::to_vec(&error).map_err(|err| {
7802        RouterError::backend(
7803            0,
7804            frame.header.corr,
7805            format!("failed to encode control ERROR: {err}"),
7806        )
7807    })?;
7808
7809    Frame::build_with_version(
7810        response_version(frame),
7811        FrameType::Error,
7812        control_flags(),
7813        0,
7814        0,
7815        frame.header.corr,
7816        body,
7817    )
7818    .map_err(RouterError::FrameBuild)
7819}
7820
7821fn control_response_body_frame<T: Serialize>(
7822    frame: &Frame,
7823    reply: &T,
7824    label: &'static str,
7825) -> Result<Frame, RouterError> {
7826    let body = serde_json::to_vec(reply).map_err(|err| {
7827        RouterError::backend(
7828            0,
7829            frame.header.corr,
7830            format!("failed to encode {label}: {err}"),
7831        )
7832    })?;
7833
7834    Frame::build_with_version(
7835        response_version(frame),
7836        FrameType::Response,
7837        control_flags(),
7838        0,
7839        0,
7840        frame.header.corr,
7841        body,
7842    )
7843    .map_err(RouterError::FrameBuild)
7844}
7845
7846/// Map a forwarding failure to the wire code a client sees.
7847///
7848/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7849/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7850/// chosen here decides whether a caller retries or gives up.
7851///
7852/// That makes attribution the load-bearing property, not merely having a code. A
7853/// permanent fault published as a retryable one produces a fleet-wide retry storm
7854/// against something that can never recover; a transient fault published as
7855/// permanent gives up on work that would have succeeded. Both look correct in a
7856/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7857/// the mapping per variant rather than merely asserting that some code exists.
7858///
7859/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7860/// two codes on the same side of the boundary passes it. Measured rather than
7861/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7862/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7863/// a test named for something else that happens to assert the string.
7864///
7865/// That accidental coverage is deliberately left alone rather than promoted to a
7866/// named test, because it guards a property this function does not promise.
7867/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7868/// specific code within a class, so identity is free to change and only the
7869/// partition is a contract. Splitting it out would assert a guarantee nothing
7870/// depends on — and a suite that promises more than the code does is the harder
7871/// thing to correct later, because the next reader cannot tell which assertions
7872/// are load-bearing.
7873///
7874/// Pin identity here the moment a consumer branches on a specific code.
7875fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7876    match err {
7877        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7878        ForwardingError::NoModuleConnection => "target_unavailable",
7879        ForwardingError::ModuleReloading { .. } => "module_reloading",
7880        ForwardingError::ClientRouteChannelExhausted { .. }
7881        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7882        ForwardingError::StaleModuleEndpoint
7883        | ForwardingError::UnknownReservation { .. }
7884        | ForwardingError::ConnectionClosing { .. }
7885        | ForwardingError::ClientEgressClosed { .. }
7886        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7887        // Only a swap candidate's registration can produce this, and it means
7888        // exactly what a second active HELLO for a live id means.
7889        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7890        ForwardingError::RelayCorrelationExhausted
7891        | ForwardingError::RouteOpenBuild(_)
7892        | ForwardingError::Poisoned => "forwarding_error",
7893    }
7894}
7895
7896fn response_version(frame: &Frame) -> u8 {
7897    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7898        frame.header.ver
7899    } else {
7900        PROTOCOL_VERSION
7901    }
7902}
7903
7904fn control_flags() -> Flags {
7905    Flags::new(false, Priority::Passive, false)
7906}
7907
7908/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7909/// channel. The target is the module (a client never saw the route), so this
7910/// takes the module path: delivered late rather than dropped when the module's
7911/// queue is momentarily full, and never closing its connection.
7912fn send_goodbye_target_best_effort(
7913    counters: &DaemonCounters,
7914    target: &GoodbyeTarget,
7915    context: &'static str,
7916) {
7917    let Ok(frame) = Frame::build_with_version(
7918        target.negotiated_ver,
7919        FrameType::Goodbye,
7920        control_flags(),
7921        target.channel,
7922        target.epoch,
7923        0,
7924        Vec::new(),
7925    ) else {
7926        return;
7927    };
7928    crate::forwarding::send_module_route_goodbye(
7929        counters,
7930        &target.sink,
7931        frame,
7932        target.module_id.as_deref(),
7933        context,
7934    );
7935}
7936
7937pub(crate) fn send_route_control_pushes(
7938    forwarding: &ForwardingTable,
7939    routes: Vec<EndpointRoute>,
7940    push: ClientControlPush,
7941) {
7942    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7943    for route in routes {
7944        let target = route.goodbye_target;
7945        if let Some((existing, channels)) = targets
7946            .iter_mut()
7947            .find(|(existing, _)| existing.connection_id == target.connection_id)
7948        {
7949            debug_assert_eq!(
7950                existing.negotiated_ver, target.negotiated_ver,
7951                "one connection cannot negotiate multiple frame versions"
7952            );
7953            if !channels.contains(&target.channel) {
7954                channels.push(target.channel);
7955            }
7956            continue;
7957        }
7958        let channel = target.channel;
7959        targets.push((target, vec![channel]));
7960    }
7961    for (target, mut channels) in targets {
7962        channels.sort_unstable();
7963        let mut push = push.clone();
7964        match &mut push {
7965            ClientControlPush::RouteClosing {
7966                channels: covered, ..
7967            }
7968            | ClientControlPush::RouteClosed {
7969                channels: covered, ..
7970            } => *covered = channels,
7971        }
7972        let body = match serde_json::to_vec(&push) {
7973            Ok(body) => body,
7974            Err(err) => {
7975                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7976                continue;
7977            }
7978        };
7979        let frame = match Frame::build_with_version(
7980            target.negotiated_ver,
7981            FrameType::Push,
7982            control_flags(),
7983            0,
7984            0,
7985            0,
7986            body.clone(),
7987        ) {
7988            Ok(frame) => frame,
7989            Err(err) => {
7990                warn!(
7991                    route_channel = target.channel,
7992                    error = %err,
7993                    "failed to build route lifecycle control PUSH frame"
7994                );
7995                continue;
7996            }
7997        };
7998        if let Err(err) = target.sink.try_send(frame) {
7999            if target.close_on_delivery_failure() {
8000                warn!(
8001                    target_connection_id = target.connection_id.get(),
8002                    route_channel = target.channel,
8003                    error = %err,
8004                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8005                );
8006                let _ = forwarding.escalate_client_delivery_failure(
8007                    target.connection_id,
8008                    target.channel,
8009                    target.epoch,
8010                    CloseReason::new(
8011                        "route_lifecycle_push_delivery_failed",
8012                        format!(
8013                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8014                            target.channel
8015                        ),
8016                    ),
8017                    crate::forwarding::UndeliveredFrame {
8018                        module_id: target.module_id.as_deref(),
8019                        sink: &target.sink,
8020                    },
8021                );
8022            }
8023        }
8024    }
8025}
8026
8027#[cfg(test)]
8028mod tests {
8029    mod activity_read {
8030        include!("activity_read_tests.rs");
8031    }
8032
8033    mod lifecycle_activity {
8034        include!("control_lifecycle_tests.rs");
8035    }
8036    #[cfg(unix)]
8037    #[tokio::test]
8038    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8039        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8040        let path = dir.join("subc.jsonc");
8041        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8042            "program":"/bin/sleep","args":["60"],"protocol":"none",
8043            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8044        }}}).to_string()).unwrap();
8045        let mut configured = crate::daemon_config::load(&path)
8046            .unwrap()
8047            .unwrap()
8048            .modules
8049            .pop()
8050            .unwrap();
8051        let registry = std::sync::Arc::new(crate::Registry::default());
8052        let handle = crate::SupervisorHandle::new();
8053        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8054            .with_handle(handle.clone());
8055        let module = supervisor
8056            .supervise_configured_with_health(
8057                configured.module_spec(),
8058                true,
8059                configured.health.clone(),
8060                None,
8061                configured.restart,
8062            )
8063            .unwrap();
8064        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8065        let before = module.status().unwrap().pid;
8066        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8067        configured.health.cadence = std::time::Duration::from_secs(3600);
8068        let health_only = handler
8069            .reconcile_supervised_modules(
8070                &supervisor,
8071                vec![configured.clone()],
8072                false,
8073                Principal::Direct,
8074            )
8075            .await
8076            .unwrap();
8077        assert!(
8078            health_only.0.changed_pending_reload.is_empty(),
8079            "health policy is already applied live"
8080        );
8081        assert_eq!(module.status().unwrap().pid, before);
8082        assert_eq!(
8083            module.configuration().unwrap().1.http,
8084            configured.health.http
8085        );
8086        configured.args = vec!["61".into()];
8087        let launch = handler
8088            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8089            .await
8090            .unwrap();
8091        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8092        assert_eq!(
8093            module.status().unwrap().pid,
8094            before,
8095            "a launch edit is stored until reload"
8096        );
8097        module.drain().await.unwrap();
8098    }
8099    use cortexkit_test_support::ScratchDir;
8100    use std::{
8101        collections::BTreeMap,
8102        fmt,
8103        path::PathBuf,
8104        sync::{Arc, Mutex},
8105        time::Duration,
8106    };
8107
8108    use serde_json::{json, Value};
8109    use subc_protocol::{
8110        manifest::{
8111            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8112            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8113        },
8114        session::HealthStatus,
8115        FrameType,
8116    };
8117
8118    use super::*;
8119    use crate::{
8120        forwarding::{DataRoute, DataRouteState},
8121        registry::ChannelState,
8122        router::FrameSink,
8123        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8124        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8125        RouteCtx, Router,
8126    };
8127    use tokio::{
8128        sync::mpsc,
8129        time::{sleep, Instant},
8130    };
8131    use tracing::{
8132        field::{Field, Visit},
8133        Event, Subscriber,
8134    };
8135    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8136
8137    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8138    ///
8139    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8140    /// is only populated for `tests/*.rs` integration test binaries -- this file
8141    /// compiles as part of the library target, which gets neither. This test's
8142    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8143    /// and the sibling binary lives two directories up at
8144    /// `<target-dir>/<profile>/fake-aft-stub`.
8145    ///
8146    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8147    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8148    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8149    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8150    /// `NotFound`, which reads as a broken test rather than an unbuilt
8151    /// dependency -- so state the cause and the remedy instead. Deliberately a
8152    /// panic and not a silent skip: a test that quietly passes when it could not
8153    /// run is worse than one that fails, because it reports health it never
8154    /// verified.
8155    fn fake_aft_stub_path() -> PathBuf {
8156        let mut path = std::env::current_exe().expect("current_exe available in tests");
8157        path.pop(); // .../deps/
8158        path.pop(); // .../<profile>/
8159        path.push(if cfg!(windows) {
8160            "fake-aft-stub.exe"
8161        } else {
8162            "fake-aft-stub"
8163        });
8164        assert!(
8165            path.exists(),
8166            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8167             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8168            path.display()
8169        );
8170        path
8171    }
8172
8173    /// Whether clients retry `code` in place: the predicate itself, never a copy
8174    /// of its set. A copied list breaks silently when a code is added to or
8175    /// removed from the real one, and a stale copy here would let exactly the
8176    /// failure this test exists to catch pass.
8177    fn client_retries(code: &str) -> bool {
8178        subc_protocol::error_codes::is_retryable_route_open(code)
8179    }
8180
8181    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8182    /// of failure is worse than publishing none. A permanent fault dressed as
8183    /// retryable makes every client in the fleet retry forever against something
8184    /// that cannot recover; a transient fault dressed as permanent abandons work
8185    /// that would have succeeded.
8186    ///
8187    /// Asserting "a code exists" cannot catch either, because the string is free
8188    /// to say anything. This enumerates every variant and pins which side of the
8189    /// retry boundary it lands on, so a new variant must be classified here
8190    /// deliberately rather than inheriting whichever arm it was appended to.
8191    #[test]
8192    fn retryability_of_forwarding_codes_matches_the_failure() {
8193        // Transient by nature: the target is booting, reloading, or its endpoint
8194        // was swapped mid-flight. Retrying is how these resolve.
8195        let transient = [
8196            ForwardingError::NoModuleConnection,
8197            ForwardingError::ModuleReloading {
8198                module_id: "m".into(),
8199            },
8200            ForwardingError::StaleModuleEndpoint,
8201            ForwardingError::UnknownReservation {
8202                client_channel: 1,
8203                module_channel: 1,
8204            },
8205            ForwardingError::ConnectionClosing {
8206                connection_id: ConnectionId::new(1),
8207            },
8208            ForwardingError::ClientEgressClosed {
8209                connection_id: ConnectionId::new(1),
8210            },
8211            ForwardingError::ModuleEgressUnavailable {
8212                connection_id: ConnectionId::new(1),
8213            },
8214        ];
8215        for err in transient {
8216            let code = forwarding_error_code(&err);
8217            assert!(
8218                client_retries(code),
8219                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8220            );
8221        }
8222
8223        // Not fixed by retrying. Channel and correlation exhaustion need the
8224        // caller to close routes, and a poisoned lock is a daemon that cannot
8225        // recover at all — the worst thing to advertise as retryable, since every
8226        // client would storm a daemon that will never answer.
8227        let permanent = [
8228            ForwardingError::ConnectionRoleConflict {
8229                connection_id: ConnectionId::new(1),
8230            },
8231            ForwardingError::ClientRouteChannelExhausted {
8232                connection_id: ConnectionId::new(1),
8233            },
8234            ForwardingError::ModuleRouteChannelExhausted {
8235                endpoint: ModuleEndpointId {
8236                    connection_id: ConnectionId::new(1),
8237                    generation: 1,
8238                },
8239            },
8240            ForwardingError::RelayCorrelationExhausted,
8241            ForwardingError::RouteOpenBuild("x".into()),
8242            ForwardingError::Poisoned,
8243        ];
8244        for err in permanent {
8245            let code = forwarding_error_code(&err);
8246            assert!(
8247                !client_retries(code),
8248                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8249            );
8250        }
8251    }
8252
8253    /// The principal is the daemon's answer to "who is calling", and modules
8254    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8255    /// plexus gates connector invocation. So a stamp is an authorization input in
8256    /// another process, not a label — and both possible answers SUCCEED, which is
8257    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8258    /// first-party capability to something that never proved it; a supervised one
8259    /// stamped `Direct` silently strips a module of capability it is entitled to.
8260    ///
8261    /// Neither shows up in a test that only checks the bind succeeded. Before this
8262    /// test the only coverage was accidental —
8263    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8264    /// stamped principal on its way past, so narrowing that wire-shape test to its
8265    /// stated subject would have deleted the last assertion on this value. It
8266    /// still asserts the stamp, which is now redundancy rather than the only
8267    /// guard: both fail under the same mutation, and this one names the reason.
8268    /// SCOPE: this handler's supervisor has spawned nothing, so
8269    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8270    /// is unreachable here. Both assertions below are refusals, and a mutant that
8271    /// refuses everything would satisfy them.
8272    ///
8273    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8274    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8275    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8276    /// at source rather than assumed, since a citation is a claim about another
8277    /// file and ages like one. Recorded because a harness that structurally
8278    /// cannot reach an arm reports "none" for that arm identically to one that
8279    /// covers it and found nothing.
8280    #[tokio::test]
8281    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8282        let handler = ControlHandler::default();
8283        let frame =
8284            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8285
8286        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8287        // any process holding the connection file. Nothing was proved, so nothing
8288        // may be granted beyond the unattested floor.
8289        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8290        assert_eq!(
8291            stamped,
8292            Principal::Direct,
8293            "a caller that proved nothing must not be stamped as a supervised module"
8294        );
8295
8296        // A claimed module_id with a nonce no supervised child was given is a
8297        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8298        // quietly demoted to Direct, or an impersonation attempt looks identical
8299        // to an ordinary unattested connection.
8300        let forged = handler
8301            .route_open_principal(
8302                &frame,
8303                Some(ConsumerIdentity {
8304                    module_id: "aft".to_string(),
8305                    launch_nonce: "not-a-real-nonce".to_string(),
8306                }),
8307            )
8308            .unwrap();
8309        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8310        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8311    }
8312
8313    /// The test above hands `route_open_principal` an identity it built itself,
8314    /// which proves the stamping rule and nothing about where the identity comes
8315    /// from. The real producer is a wire body, and the two are joined by a serde
8316    /// field name that nothing else asserts.
8317    ///
8318    /// That join fails quietly in one specific way: an unrecognised key is simply
8319    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8320    /// yields `None` and every supervised module silently drops to `Direct`.
8321    /// Capability-wise that is the safe direction, but it surfaces far from its
8322    /// cause — as a module mysteriously refused bash — and it would pass every
8323    /// test that builds its own input.
8324    ///
8325    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8326    /// would break every client the moment the daemon gains a field, trading a
8327    /// quiet demotion for a hard refusal on additive change. Asserting the join
8328    /// instead means a rename breaks a test here rather than the fleet.
8329    #[test]
8330    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8331        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8332        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8333        let ClientControlRequest::RouteOpen {
8334            consumer_identity, ..
8335        } = parsed
8336        else {
8337            panic!("route.open body must parse as RouteOpen");
8338        };
8339        assert_eq!(
8340            consumer_identity,
8341            Some(ConsumerIdentity {
8342                module_id: "aft".to_string(),
8343                launch_nonce: "n".to_string(),
8344            }),
8345            "the wire field name must reach the value route_open_principal reads"
8346        );
8347    }
8348
8349    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8350        ModuleManifest::builder(module_id, "0.1.0")
8351            .protocol_ver(protocol_ver)
8352            .provides(vec![ProviderRole::ToolProvider {
8353                tools: vec![Tool {
8354                    name: "read".to_string(),
8355                    description: None,
8356                    execution_mode: ExecutionMode::Pure,
8357                    schema: json!({"type": "object"}),
8358                }],
8359                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8360                concurrency: Concurrency::ModuleManaged,
8361                emits_push: true,
8362                sub_supervises: true,
8363            }])
8364            .build()
8365    }
8366
8367    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8368        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8369    }
8370
8371    fn hello_frame_with_control_ops(
8372        module_id: &str,
8373        protocol_ver: u8,
8374        corr: u64,
8375        control_ops: Option<Vec<String>>,
8376    ) -> Frame {
8377        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8378    }
8379
8380    fn hello_frame_with_nonce(
8381        module_id: &str,
8382        protocol_ver: u8,
8383        corr: u64,
8384        launch_nonce: Option<&str>,
8385    ) -> Frame {
8386        hello_frame_full(
8387            module_id,
8388            protocol_ver,
8389            corr,
8390            None,
8391            launch_nonce.map(ToOwned::to_owned),
8392        )
8393    }
8394
8395    fn hello_frame_full(
8396        module_id: &str,
8397        protocol_ver: u8,
8398        corr: u64,
8399        control_ops: Option<Vec<String>>,
8400        launch_nonce: Option<String>,
8401    ) -> Frame {
8402        let body = serde_json::to_vec(&ModuleHelloBody {
8403            manifest: manifest(module_id, protocol_ver),
8404            protocol_ver,
8405            control_ops,
8406            launch_nonce,
8407        })
8408        .unwrap();
8409        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8410    }
8411
8412    fn hello_frame_with_rich_ui(
8413        module_id: &str,
8414        protocol_ver: u8,
8415        corr: u64,
8416        launch_nonce: Option<&str>,
8417        rich_ui: Value,
8418    ) -> Frame {
8419        let mut body = serde_json::to_value(ModuleHelloBody {
8420            manifest: manifest(module_id, protocol_ver),
8421            protocol_ver,
8422            control_ops: None,
8423            launch_nonce: launch_nonce.map(str::to_string),
8424        })
8425        .unwrap();
8426        body["manifest"]["rich_ui"] = rich_ui;
8427        Frame::build(
8428            FrameType::Hello,
8429            control_flags(),
8430            0,
8431            0,
8432            corr,
8433            serde_json::to_vec(&body).unwrap(),
8434        )
8435        .unwrap()
8436    }
8437
8438    fn non_routable_hello_frame_with_control_ops(
8439        module_id: &str,
8440        corr: u64,
8441        control_ops: Option<Vec<String>>,
8442    ) -> Frame {
8443        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8444        manifest.provides.clear();
8445        let body = serde_json::to_vec(&ModuleHelloBody {
8446            manifest,
8447            protocol_ver: PROTOCOL_VERSION,
8448            control_ops,
8449            launch_nonce: None,
8450        })
8451        .unwrap();
8452        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8453    }
8454
8455    fn capability_grammar_hello_frame(
8456        capabilities: Value,
8457        runtime_computed: Option<Value>,
8458        corr: u64,
8459    ) -> Frame {
8460        let mut body = serde_json::to_value(ModuleHelloBody {
8461            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8462            protocol_ver: PROTOCOL_VERSION,
8463            control_ops: None,
8464            launch_nonce: None,
8465        })
8466        .expect("HELLO body serializes");
8467        body["manifest"]["capabilities"] = capabilities;
8468        if let Some(runtime_computed) = runtime_computed {
8469            body["runtime_computed"] = runtime_computed;
8470        }
8471        Frame::build(
8472            FrameType::Hello,
8473            control_flags(),
8474            0,
8475            0,
8476            corr,
8477            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8478        )
8479        .expect("HELLO frame builds")
8480    }
8481
8482    fn channel_request(channel: u16, corr: u64) -> Frame {
8483        Frame::build(
8484            FrameType::Request,
8485            Flags::new(true, Priority::Interactive, false),
8486            channel,
8487            0,
8488            corr,
8489            b"opaque".to_vec(),
8490        )
8491        .unwrap()
8492    }
8493
8494    fn route_ctx(
8495        connection_id: ConnectionId,
8496    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8497        let (tx, rx) = mpsc::channel(8);
8498        (
8499            RouteCtx {
8500                connection_id,
8501                egress: FrameSink::new(tx),
8502            },
8503            rx,
8504        )
8505    }
8506
8507    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8508        serde_json::from_slice(&frame.body).unwrap()
8509    }
8510
8511    /// Register a module over a connection that has a sink and return the
8512    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8513    /// module's own sink rather than returning it as a reply, so the ack is
8514    /// taken off `rx` here and whatever the test reads next is what followed it.
8515    async fn hello_via_sink(
8516        handler: &ControlHandler,
8517        ctx: &RouteCtx,
8518        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8519        hello: Frame,
8520    ) -> Frame {
8521        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8522        assert!(
8523            replies.is_empty(),
8524            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8525        );
8526        let ack = rx
8527            .try_recv()
8528            .expect("HELLO_ACK is queued on the module sink")
8529            .frame;
8530        assert_eq!(ack.header.ty, FrameType::HelloAck);
8531        ack
8532    }
8533
8534    fn parse_error(frame: &Frame) -> Value {
8535        serde_json::from_slice(&frame.body).unwrap()
8536    }
8537
8538    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8539        serde_json::from_slice(&frame.body).unwrap()
8540    }
8541
8542    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8543        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8544            route_channel,
8545            route_epoch: 0,
8546            kind,
8547        })
8548        .unwrap();
8549        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8550    }
8551
8552    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8553        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8554            module_id: module_id.to_string(),
8555        })
8556        .unwrap();
8557        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8558    }
8559
8560    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8561        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8562    }
8563
8564    fn route_open_frame_with_consumer_capabilities(
8565        corr: u64,
8566        module_id: &str,
8567        project_root: ScratchDir,
8568        consumer_capabilities: Option<Vec<String>>,
8569    ) -> Frame {
8570        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8571            target: RouteTarget::ToolProvider {
8572                module_id: module_id.to_string(),
8573            },
8574            identity: BindIdentity::new(
8575                project_root.path().to_path_buf(),
8576                "unit".to_string(),
8577                "session".to_string(),
8578            ),
8579            consumer_identity: None,
8580            consumer_capabilities,
8581            role_versions: None,
8582            admission_facts: None,
8583            scope: None,
8584        })
8585        .unwrap();
8586        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8587    }
8588
8589    fn route_open_frame_with_role_versions(
8590        corr: u64,
8591        module_id: &str,
8592        project_root: ScratchDir,
8593        role_versions: Option<BTreeMap<String, String>>,
8594    ) -> Frame {
8595        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8596            target: RouteTarget::ToolProvider {
8597                module_id: module_id.to_string(),
8598            },
8599            identity: BindIdentity::new(
8600                project_root.path().to_path_buf(),
8601                "unit".to_string(),
8602                format!("session-{corr}"),
8603            ),
8604            consumer_identity: None,
8605            consumer_capabilities: None,
8606            role_versions,
8607            admission_facts: None,
8608            scope: None,
8609        })
8610        .unwrap();
8611        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8612    }
8613
8614    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8615        entries
8616            .iter()
8617            .map(|(role, version)| (role.to_string(), version.to_string()))
8618            .collect()
8619    }
8620
8621    fn route_open_frame_with_admission_facts(
8622        corr: u64,
8623        module_id: &str,
8624        project_root: ScratchDir,
8625        consumer_identity: Option<subc_control::ConsumerIdentity>,
8626        facts: Option<Value>,
8627    ) -> Frame {
8628        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8629            target: RouteTarget::ToolProvider {
8630                module_id: module_id.to_string(),
8631            },
8632            identity: BindIdentity::new(
8633                project_root.path().to_path_buf(),
8634                "unit".to_string(),
8635                format!("session-{corr}"),
8636            ),
8637            consumer_identity,
8638            consumer_capabilities: None,
8639            role_versions: None,
8640            admission_facts: facts,
8641            scope: None,
8642        })
8643        .unwrap();
8644        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8645    }
8646
8647    #[derive(Clone, Default)]
8648    struct EventCapture {
8649        events: Arc<Mutex<Vec<CapturedEvent>>>,
8650    }
8651
8652    #[derive(Clone, Debug)]
8653    struct CapturedEvent {
8654        target: String,
8655        level: tracing::Level,
8656        fields: BTreeMap<String, String>,
8657    }
8658
8659    impl EventCapture {
8660        fn events(&self) -> Vec<CapturedEvent> {
8661            self.events.lock().unwrap().clone()
8662        }
8663    }
8664
8665    impl<S> Layer<S> for EventCapture
8666    where
8667        S: Subscriber,
8668    {
8669        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8670            let mut visitor = EventFieldVisitor::default();
8671            event.record(&mut visitor);
8672            self.events.lock().unwrap().push(CapturedEvent {
8673                target: event.metadata().target().to_string(),
8674                level: *event.metadata().level(),
8675                fields: visitor.fields,
8676            });
8677        }
8678    }
8679
8680    #[derive(Default)]
8681    struct EventFieldVisitor {
8682        fields: BTreeMap<String, String>,
8683    }
8684
8685    impl Visit for EventFieldVisitor {
8686        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8687            self.fields
8688                .insert(field.name().to_string(), format!("{value:?}"));
8689        }
8690    }
8691
8692    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8693        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8694            status,
8695            detail: Some("warming".to_string()),
8696            metrics: Some(json!({"queue_depth": 3})),
8697        })
8698        .unwrap();
8699        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8700    }
8701
8702    fn route_bind_ack(corr: u64) -> Frame {
8703        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8704        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8705    }
8706
8707    fn unique_project_root(label: &str) -> ScratchDir {
8708        ScratchDir::new(label)
8709    }
8710
8711    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8712        match parse_route_poll(frame) {
8713            ClientControlResponse::RoutePoll {
8714                status: None,
8715                live: Some(live),
8716                ..
8717            } => assert_eq!(live, expected_live),
8718            other => panic!("unexpected route.poll response: {other:?}"),
8719        }
8720    }
8721
8722    fn bind_liveness_route(
8723        registry: &Registry,
8724        forwarding: &ForwardingTable,
8725        module_id: &str,
8726    ) -> (RouteCtx, u16, u32) {
8727        let module_connection = ConnectionId::new(101);
8728        let client_connection = ConnectionId::new(202);
8729        let registration = registry
8730            .register_with_control_ops(
8731                manifest(module_id, PROTOCOL_VERSION),
8732                PROTOCOL_VERSION,
8733                module_connection,
8734                module_baseline_control_ops(),
8735            )
8736            .unwrap();
8737        let (module_tx, _module_rx) = mpsc::channel(8);
8738        let endpoint = forwarding
8739            .register_module_connection(
8740                module_connection,
8741                module_id.to_string(),
8742                PROTOCOL_VERSION,
8743                manifest_concurrency(&registration.manifest),
8744                FrameSink::new(module_tx),
8745            )
8746            .unwrap();
8747        let (client_ctx, _client_rx) = route_ctx(client_connection);
8748        let pending = forwarding
8749            .begin_route_bind_relay_for_test(
8750                client_connection,
8751                client_ctx.egress.clone(),
8752                1,
8753                module_id,
8754            )
8755            .unwrap();
8756        assert_eq!(pending.endpoint, endpoint);
8757        let route_channel = pending.client_channel;
8758        let route_epoch = pending.client_epoch;
8759        forwarding
8760            .complete_pending_relay(
8761                module_connection,
8762                pending.corr,
8763                RouteBindRelayOutcome::Accepted,
8764            )
8765            .unwrap();
8766        (client_ctx, route_channel, route_epoch)
8767    }
8768
8769    struct FakeProcessLiveness {
8770        live: Option<bool>,
8771    }
8772
8773    impl ModuleProcessLiveness for FakeProcessLiveness {
8774        fn process_live(&self, _module_id: &str) -> Option<bool> {
8775            self.live
8776        }
8777    }
8778
8779    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8780    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8781    {
8782        let registry = Arc::new(Registry::default());
8783        let supervisor_handle = SupervisorHandle::new();
8784        let supervisor = Supervisor::new_for_test(
8785            Arc::clone(&registry),
8786            RestartPolicy::new(1, Duration::from_millis(10)),
8787        )
8788        .with_handle(supervisor_handle.clone());
8789        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8790        let module = supervisor
8791            .spawn(ModuleSpec {
8792                module_id: "stderr-tail-wire".to_string(),
8793                program: fake_aft_stub_path(),
8794                args: Vec::new(),
8795                env: vec![
8796                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8797                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8798                ],
8799                reserved: false,
8800                reserved_prefixes: Vec::new(),
8801                protocol: ModuleProtocol::Subc,
8802                overlap: Default::default(),
8803            })
8804            .unwrap();
8805
8806        let deadline = Instant::now() + Duration::from_secs(5);
8807        loop {
8808            let tail = module.stderr_tail(None, None);
8809            if tail
8810                .entries
8811                .iter()
8812                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8813                && tail.entries.iter().any(|entry| {
8814                    matches!(
8815                        entry,
8816                        TailEntry::Line {
8817                            truncated: true,
8818                            ..
8819                        }
8820                    )
8821                })
8822            {
8823                break;
8824            }
8825            assert!(
8826                Instant::now() < deadline,
8827                "module did not produce a truncated line and restart boundary: {tail:?}"
8828            );
8829            sleep(Duration::from_millis(10)).await;
8830        }
8831
8832        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8833        let request = ClientControlRequest::SupervisorStderrTail {
8834            module_id: "stderr-tail-wire".to_string(),
8835            max_lines: None,
8836            max_bytes: None,
8837        };
8838        let frame = Frame::build(
8839            FrameType::Request,
8840            control_flags(),
8841            0,
8842            0,
8843            1,
8844            serde_json::to_vec(&request).unwrap(),
8845        )
8846        .unwrap();
8847        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8848        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8849        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8850            serde_json::from_slice(&responses[0].body).unwrap()
8851        else {
8852            panic!("expected supervisor.stderr_tail response");
8853        };
8854
8855        assert!(
8856            tail.entries
8857                .iter()
8858                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8859            "the control response lost the restart boundary"
8860        );
8861        let Some(StderrTailEntry::Line {
8862            text,
8863            truncated,
8864            at_ms,
8865        }) = tail.entries.iter().find(|entry| {
8866            matches!(
8867                entry,
8868                StderrTailEntry::Line {
8869                    truncated: true,
8870                    ..
8871                }
8872            )
8873        })
8874        else {
8875            panic!("the control response lost the truncated line");
8876        };
8877        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8878        assert!(*truncated);
8879        assert!(
8880            at_ms.is_some(),
8881            "the control response lost the line's capture time"
8882        );
8883    }
8884
8885    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8886    /// read done on the worker thread would stall every other task until it
8887    /// finished; the read must run off the worker so this test's own task keeps
8888    /// running while the read is paused.
8889    #[tokio::test(flavor = "current_thread")]
8890    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8891        let dir = ScratchDir::new("terminals-off-worker");
8892        let journal_path = dir.join("terminals.jsonl");
8893        let registry = Arc::new(Registry::default());
8894        let supervisor_handle = SupervisorHandle::new();
8895        let supervisor =
8896            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8897                .with_handle(supervisor_handle.clone())
8898                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8899        let module = supervisor
8900            .spawn(ModuleSpec {
8901                module_id: "terminal-off-worker".to_string(),
8902                program: fake_aft_stub_path(),
8903                args: Vec::new(),
8904                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8905                reserved: false,
8906                reserved_prefixes: Vec::new(),
8907                protocol: ModuleProtocol::Subc,
8908                overlap: Default::default(),
8909            })
8910            .unwrap();
8911        let deadline = Instant::now() + Duration::from_secs(5);
8912        while module.terminal_history().entries.len() != 2 {
8913            assert!(Instant::now() < deadline, "module did not record two exits");
8914            sleep(Duration::from_millis(10)).await;
8915        }
8916
8917        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8918        let handler =
8919            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8920        let frame = Frame::build(
8921            FrameType::Request,
8922            control_flags(),
8923            0,
8924            0,
8925            1,
8926            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8927                module_id: "terminal-off-worker".to_string(),
8928            })
8929            .unwrap(),
8930        )
8931        .unwrap();
8932        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8933        let spawned_at = std::time::Instant::now();
8934        let read = tokio::spawn({
8935            let handler = Arc::clone(&handler);
8936            async move { handler.handle_control_frame(&ctx, frame).await }
8937        });
8938        // Waiting for the pause from a blocking thread keeps this task pending,
8939        // so the runtime's single worker is free to run the read task.
8940        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8941            .await
8942            .unwrap()
8943            .expect("the history read reached its pause");
8944        let elapsed = spawned_at.elapsed();
8945        assert!(
8946            elapsed < Duration::from_secs(2) && !read.is_finished(),
8947            "this task could not run while the history read was paused \
8948             (resumed after {elapsed:?}, read finished: {})",
8949            read.is_finished()
8950        );
8951
8952        drop(release);
8953        let responses = read.await.unwrap().unwrap();
8954        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8955        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8956            panic!("expected supervisor.terminals response");
8957        };
8958        assert_eq!(terminals.entries.len(), 2);
8959        assert_eq!(terminals.journal_skipped_lines, 0);
8960        assert_eq!(terminals.journal_read_errors, 0);
8961    }
8962
8963    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8964    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8965        let registry = Arc::new(Registry::default());
8966        let supervisor_handle = SupervisorHandle::new();
8967        let supervisor =
8968            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8969                .with_handle(supervisor_handle.clone());
8970        let module = supervisor
8971            .spawn(ModuleSpec {
8972                module_id: "terminal-golden".to_string(),
8973                program: fake_aft_stub_path(),
8974                args: Vec::new(),
8975                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8976                reserved: false,
8977                reserved_prefixes: Vec::new(),
8978                protocol: ModuleProtocol::Subc,
8979                overlap: Default::default(),
8980            })
8981            .unwrap();
8982
8983        let deadline = Instant::now() + Duration::from_secs(5);
8984        while module.terminal_history().entries.len() != 2 {
8985            assert!(
8986                Instant::now() < deadline,
8987                "module did not retain two terminal exits: {:?}",
8988                module.terminal_history()
8989            );
8990            sleep(Duration::from_millis(10)).await;
8991        }
8992
8993        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8994        let request = ClientControlRequest::SupervisorTerminals {
8995            module_id: "terminal-golden".to_string(),
8996        };
8997        let frame = Frame::build(
8998            FrameType::Request,
8999            control_flags(),
9000            0,
9001            0,
9002            1,
9003            serde_json::to_vec(&request).unwrap(),
9004        )
9005        .unwrap();
9006        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9007        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9008        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9009        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9010            panic!("expected supervisor.terminals response");
9011        };
9012        assert_eq!(terminals.entries.len(), 2);
9013        assert_eq!(terminals.dropped, 0);
9014
9015        let mut rendered = serde_json::to_value(response).unwrap();
9016        // Wall-clock fields are the observation contract, but not stable fixture
9017        // bytes; normalize only them after the real handler has shaped the response.
9018        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9019        for (index, entry) in rendered["entries"]
9020            .as_array_mut()
9021            .expect("terminal response entries array")
9022            .iter_mut()
9023            .enumerate()
9024        {
9025            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9026        }
9027
9028        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9029            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9030        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9031        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9032            std::fs::write(&golden_path, &serialized).unwrap();
9033        }
9034        let expected: Value =
9035            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9036        assert_eq!(rendered, expected);
9037    }
9038
9039    #[test]
9040    fn hello_registers_manifest_and_returns_ack() {
9041        let registry = Arc::new(Registry::default());
9042        let handler = ControlHandler::new(Arc::clone(&registry));
9043        let conn = ConnectionId::new(1);
9044
9045        let responses = handler
9046            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9047            .unwrap();
9048
9049        assert_eq!(responses.len(), 1);
9050        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9051        assert_eq!(responses[0].header.channel, 0);
9052        assert_eq!(responses[0].header.corr, 7);
9053        let ack = parse_ack(&responses[0]);
9054        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9055        assert!(ack
9056            .subc_capabilities
9057            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9058        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9059        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9060        assert!(ack
9061            .subc_ops
9062            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9063        assert!(ack
9064            .subc_ops
9065            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9066
9067        let registration = registry.get_module("aft").unwrap().unwrap();
9068        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9069        assert_eq!(registration.state, ChannelState::Active);
9070        assert_eq!(registration.connection_id, conn);
9071        assert_eq!(registration.control_ops, module_baseline_control_ops());
9072    }
9073
9074    #[test]
9075    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9076        let invalid_identifiers = [
9077            ("case_change", "credentials-Provider/v1"),
9078            ("leading_zero", "credentials-provider/v01"),
9079            ("trailing_hyphen", "credentials-provider-/v1"),
9080            ("consecutive_hyphens", "credentials--provider/v1"),
9081            ("uppercase", "Credentials-provider/v1"),
9082            ("missing_v", "credentials-provider/1"),
9083            ("whitespace", "credentials provider/v1"),
9084            ("zero_version", "credentials-provider/v0"),
9085            ("out_of_range_version", "credentials-provider/v4294967296"),
9086            (
9087                "overlength_name",
9088                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9089            ),
9090        ];
9091        let mut cases = invalid_identifiers
9092            .into_iter()
9093            .map(|(name, identifier)| {
9094                (
9095                    format!("identifier_{name}"),
9096                    "capabilities.provides[0]".to_string(),
9097                    identifier.to_string(),
9098                    json!({ "provides": [identifier] }),
9099                    None,
9100                )
9101            })
9102            .collect::<Vec<_>>();
9103        cases.extend([
9104            (
9105                "unknown_need".to_string(),
9106                "capabilities.requires[0].need".to_string(),
9107                "deferred".to_string(),
9108                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9109                None,
9110            ),
9111            (
9112                "duplicate_provides".to_string(),
9113                "capabilities.provides[1]".to_string(),
9114                "credentials-provider/v1".to_string(),
9115                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9116                None,
9117            ),
9118            (
9119                "duplicate_must_never_reach".to_string(),
9120                "capabilities.must_never_reach[1]".to_string(),
9121                "credentials-provider/v1".to_string(),
9122                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9123                None,
9124            ),
9125            (
9126                "duplicate_requires_same_need".to_string(),
9127                "capabilities.requires[1]".to_string(),
9128                "credentials-provider/v1".to_string(),
9129                json!({ "requires": [
9130                    { "capability": "credentials-provider/v1", "need": "required" },
9131                    { "capability": "credentials-provider/v1", "need": "required" }
9132                ] }),
9133                None,
9134            ),
9135            (
9136                "duplicate_requires_conflicting_need".to_string(),
9137                "capabilities.requires[1]".to_string(),
9138                "credentials-provider/v1".to_string(),
9139                json!({ "requires": [
9140                    { "capability": "credentials-provider/v1", "need": "required" },
9141                    { "capability": "credentials-provider/v1", "need": "optional" }
9142                ] }),
9143                None,
9144            ),
9145            (
9146                "capabilities_root_pointer".to_string(),
9147                "runtime_computed[0]".to_string(),
9148                "/capabilities".to_string(),
9149                json!({}),
9150                Some(json!(["/capabilities"])),
9151            ),
9152            (
9153                "capabilities_descendant_pointer".to_string(),
9154                "runtime_computed[0]".to_string(),
9155                "/capabilities/provides".to_string(),
9156                json!({}),
9157                Some(json!(["/capabilities/provides"])),
9158            ),
9159            (
9160                "malformed_pointer_without_leading_slash".to_string(),
9161                "runtime_computed[0]".to_string(),
9162                "capabilities".to_string(),
9163                json!({}),
9164                Some(json!(["capabilities"])),
9165            ),
9166            (
9167                "malformed_pointer_escape".to_string(),
9168                "runtime_computed[0]".to_string(),
9169                "/roles/~2/tools".to_string(),
9170                json!({}),
9171                Some(json!(["/roles/~2/tools"])),
9172            ),
9173            (
9174                "unknown_capabilities_field".to_string(),
9175                "capabilities.future".to_string(),
9176                "<array>".to_string(),
9177                json!({ "future": [] }),
9178                None,
9179            ),
9180        ]);
9181
9182        for (index, (name, field, value, capabilities, runtime_computed)) in
9183            cases.into_iter().enumerate()
9184        {
9185            let registry = Arc::new(Registry::default());
9186            let handler = ControlHandler::new(Arc::clone(&registry));
9187            let response = handler
9188                .handle_control(
9189                    ConnectionId::new((index + 1) as u64),
9190                    capability_grammar_hello_frame(
9191                        capabilities,
9192                        runtime_computed,
9193                        index as u64 + 1,
9194                    ),
9195                )
9196                .expect("invalid HELLO returns a refusal");
9197
9198            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9199            let error = parse_error(&response[0]);
9200            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9201            let message = error["message"]
9202                .as_str()
9203                .expect("error message is a string");
9204            assert!(
9205                message.contains(&field),
9206                "{name}: field missing from {message}"
9207            );
9208            assert!(
9209                message.contains(&value),
9210                "{name}: value missing from {message}"
9211            );
9212            assert_eq!(
9213                registry
9214                    .active_registration_count()
9215                    .expect("registry reads"),
9216                0,
9217                "{name}: refused HELLO must not create a catalog entry"
9218            );
9219        }
9220    }
9221
9222    #[test]
9223    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9224        let registry = Arc::new(Registry::default());
9225        let handler = ControlHandler::new(Arc::clone(&registry));
9226        let capabilities = json!({
9227            "provides": ["credentials-provider/v1"],
9228            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9229            "must_never_reach": ["federation-transport/v1"]
9230        });
9231        let response = handler
9232            .handle_control(
9233                ConnectionId::new(99),
9234                capability_grammar_hello_frame(
9235                    capabilities.clone(),
9236                    Some(json!(["/roles/0/tools"])),
9237                    99,
9238                ),
9239            )
9240            .expect("valid HELLO registers");
9241        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9242
9243        let request = Frame::build(
9244            FrameType::Request,
9245            control_flags(),
9246            0,
9247            0,
9248            100,
9249            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9250                .expect("catalog request serializes"),
9251        )
9252        .expect("catalog request frame builds");
9253        let response = handler
9254            .handle_catalog_list(request, None)
9255            .expect("catalog list succeeds");
9256        let ClientControlResponse::CatalogList { modules, .. } =
9257            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9258        else {
9259            panic!("catalog request must return catalog.list");
9260        };
9261        assert_eq!(modules.len(), 1);
9262        assert_eq!(
9263            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9264            capabilities
9265        );
9266    }
9267
9268    #[test]
9269    fn catalog_list_mirrors_management_operation_description() {
9270        let registry = Arc::new(Registry::default());
9271        let handler = ControlHandler::new(Arc::clone(&registry));
9272        let description = "List managed records and return their identifiers and metadata.";
9273        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9274        manifest.provides = vec![ProviderRole::ManagementSurface {
9275            operations: vec![ManagementOperation {
9276                name: "records.list".to_string(),
9277                kind: ManagementOperationKind::Query,
9278                description: Some(description.to_string()),
9279            }],
9280            config_schema: json!({"type": "object"}),
9281            observability: vec![ObservabilitySurface {
9282                name: "records.stats".to_string(),
9283                kind: ObservabilityKind::Snapshot,
9284            }],
9285            identity_scope: vec![IdentityScope::Project],
9286            concurrency: Concurrency::ModuleManaged,
9287        }];
9288        registry
9289            .register_with_control_ops(
9290                manifest,
9291                PROTOCOL_VERSION,
9292                ConnectionId::new(99),
9293                Vec::new(),
9294            )
9295            .expect("described management manifest registers");
9296
9297        let request = Frame::build(
9298            FrameType::Request,
9299            control_flags(),
9300            0,
9301            0,
9302            100,
9303            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9304                .expect("catalog request serializes"),
9305        )
9306        .expect("catalog request frame builds");
9307        let response = handler
9308            .handle_catalog_list(request, None)
9309            .expect("catalog list succeeds");
9310        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9311        assert_eq!(
9312            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9313            "catalog.list must preserve the declared operation description verbatim"
9314        );
9315    }
9316
9317    #[test]
9318    fn catalog_list_mirrors_registered_operation_delivery() {
9319        let registry = Arc::new(Registry::default());
9320        let handler = ControlHandler::new(Arc::clone(&registry));
9321        let delivery =
9322            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9323        let mut body = serde_json::to_value(ModuleHelloBody {
9324            manifest: manifest("watch-management", PROTOCOL_VERSION),
9325            protocol_ver: PROTOCOL_VERSION,
9326            control_ops: None,
9327            launch_nonce: None,
9328        })
9329        .unwrap();
9330        body["manifest"]["provides"] = json!([{
9331            "role": "management_surface",
9332            "operations": [
9333                {"name": "status", "kind": "query"},
9334                {"name": "future", "kind": "query"},
9335                {"name": "reset", "kind": "mutate"}
9336            ],
9337            "config_schema": {}, "observability": [], "identity_scope": []
9338        }]);
9339        body["manifest"]["operation_delivery"] = delivery.clone();
9340        let hello = Frame::build(
9341            FrameType::Hello,
9342            control_flags(),
9343            0,
9344            0,
9345            99,
9346            serde_json::to_vec(&body).unwrap(),
9347        )
9348        .unwrap();
9349        let response = handler
9350            .handle_control(ConnectionId::new(99), hello)
9351            .unwrap();
9352        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9353
9354        let (_, registrations) = registry.list_modules().unwrap();
9355        let registered = &registrations[0].manifest;
9356        assert!(registered.is_watch_operation("status"));
9357        assert!(!registered.is_watch_operation("future"));
9358        assert!(!registered.is_watch_operation("reset"));
9359        assert!(!registered.is_watch_operation("missing"));
9360
9361        let request = Frame::build(
9362            FrameType::Request,
9363            control_flags(),
9364            0,
9365            0,
9366            100,
9367            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9368        )
9369        .unwrap();
9370        let response = handler.handle_catalog_list(request, None).unwrap();
9371        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9372        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9373        let ClientControlResponse::CatalogList { modules, .. } =
9374            serde_json::from_slice(&response[0].body).unwrap()
9375        else {
9376            panic!("expected catalog.list");
9377        };
9378        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9379    }
9380
9381    #[test]
9382    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9383        let registry = Arc::new(Registry::default());
9384        let supervisor = SupervisorHandle::new();
9385        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9386        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9387        let descriptor = json!({
9388            "schema": "ckios.richui.manifest/1",
9389            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9390            "bytes": 4096,
9391            "chunks": 4
9392        });
9393        let connection = ConnectionId::new(201);
9394        let hello = handler
9395            .handle_control(
9396                connection,
9397                hello_frame_with_rich_ui(
9398                    "vault",
9399                    PROTOCOL_VERSION,
9400                    201,
9401                    Some("the-real-nonce"),
9402                    descriptor.clone(),
9403                ),
9404            )
9405            .unwrap();
9406        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9407
9408        let request =
9409            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9410        let provides = registry
9411            .get_module_by_connection(connection)
9412            .unwrap()
9413            .unwrap()
9414            .manifest
9415            .provides;
9416        handler
9417            .handle_catalog_update(connection, request, provides, None, Some(false))
9418            .unwrap();
9419
9420        let request = Frame::build(
9421            FrameType::Request,
9422            control_flags(),
9423            0,
9424            0,
9425            203,
9426            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9427        )
9428        .unwrap();
9429        let response = handler.handle_catalog_list(request, None).unwrap();
9430        let ClientControlResponse::CatalogList { modules, .. } =
9431            serde_json::from_slice(&response[0].body).unwrap()
9432        else {
9433            panic!("expected catalog.list");
9434        };
9435        assert_eq!(
9436            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9437            descriptor
9438        );
9439        assert!(modules[0].nonce_verified);
9440    }
9441
9442    fn hello_with_role_declarations(declarations: Value) -> Frame {
9443        let mut body = serde_json::to_value(ModuleHelloBody {
9444            manifest: manifest("role-test", PROTOCOL_VERSION),
9445            protocol_ver: PROTOCOL_VERSION,
9446            control_ops: None,
9447            launch_nonce: None,
9448        })
9449        .unwrap();
9450        body["manifest"]["role_declarations"] = declarations;
9451        Frame::build(
9452            FrameType::Hello,
9453            control_flags(),
9454            0,
9455            0,
9456            301,
9457            serde_json::to_vec(&body).unwrap(),
9458        )
9459        .unwrap()
9460    }
9461
9462    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9463        let request = Frame::build(
9464            FrameType::Request,
9465            control_flags(),
9466            0,
9467            0,
9468            302,
9469            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9470        )
9471        .unwrap();
9472        let response = handler.handle_catalog_list(request, None).unwrap();
9473        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9474        let ClientControlResponse::CatalogList { mut modules, .. } =
9475            serde_json::from_slice(&response[0].body).unwrap()
9476        else {
9477            panic!("expected catalog.list");
9478        };
9479        (raw["modules"][0].clone(), modules.remove(0))
9480    }
9481
9482    #[test]
9483    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9484        let registry = Arc::new(Registry::default());
9485        let handler = ControlHandler::new(Arc::clone(&registry));
9486        let declarations = json!({
9487            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9488            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9489        });
9490        let connection = ConnectionId::new(301);
9491        let response = handler
9492            .handle_control(
9493                connection,
9494                hello_with_role_declarations(declarations.clone()),
9495            )
9496            .unwrap();
9497        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9498        let registered = registry
9499            .get_module_by_connection(connection)
9500            .unwrap()
9501            .unwrap()
9502            .manifest;
9503        // Compared as JSON values, not bytes: object key order is not part of
9504        // the relay's promise, and workspace builds unify serde_json's
9505        // preserve_order feature, which changes the order a literal keeps.
9506        assert_eq!(
9507            serde_json::to_value(&registered.role_declarations).unwrap(),
9508            declarations
9509        );
9510        let (raw, entry) = role_declarations_catalog(&handler);
9511        assert_eq!(raw["role_declarations"], declarations);
9512        assert_eq!(entry.role_declarations, registered.role_declarations);
9513
9514        let request =
9515            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9516        handler
9517            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9518            .unwrap();
9519        let (raw, entry) = role_declarations_catalog(&handler);
9520        assert_eq!(raw["role_declarations"], declarations);
9521        assert_eq!(entry.role_declarations, registered.role_declarations);
9522    }
9523
9524    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9525        let registry = Arc::new(Registry::default());
9526        let handler = ControlHandler::new(Arc::clone(&registry));
9527        let capture = EventCapture::default();
9528        let _subscriber =
9529            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9530        let response = handler
9531            .handle_control(
9532                ConnectionId::new(301),
9533                hello_with_role_declarations(declarations),
9534            )
9535            .unwrap();
9536        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9537        assert!(registry
9538            .get_module_by_connection(ConnectionId::new(301))
9539            .unwrap()
9540            .unwrap()
9541            .manifest
9542            .role_declarations
9543            .is_empty());
9544        let (raw, entry) = role_declarations_catalog(&handler);
9545        assert!(raw.get("role_declarations").is_none());
9546        assert!(entry.role_declarations.is_empty());
9547        let warnings: Vec<_> = capture
9548            .events()
9549            .into_iter()
9550            .filter(|event| {
9551                event.fields.get("message").map(String::as_str)
9552                    == Some("dropping whole role_declarations map from HELLO")
9553            })
9554            .collect();
9555        assert_eq!(warnings.len(), 1);
9556        assert_eq!(warnings[0].level, tracing::Level::WARN);
9557        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9558        assert_eq!(warnings[0].fields["reason"], reason);
9559    }
9560
9561    #[test]
9562    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9563        for value in [
9564            json!(null),
9565            json!(false),
9566            json!(0),
9567            json!("opaque"),
9568            json!([]),
9569        ] {
9570            assert_hello_drops_role_declarations_with_warning(
9571                json!({"routing/v1": {}, "broken/v1": value}),
9572                "role_declarations values must be objects",
9573            );
9574        }
9575        assert_hello_drops_role_declarations_with_warning(
9576            json!([]),
9577            "role_declarations must be an object",
9578        );
9579    }
9580
9581    #[test]
9582    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9583        assert_hello_drops_role_declarations_with_warning(
9584            json!({"routing/v1": {}, "bad key": {}}),
9585            "role_declarations key must use capability-name grammar",
9586        );
9587    }
9588
9589    #[test]
9590    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9591        assert_hello_drops_role_declarations_with_warning(
9592            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9593            "role_declarations exceeds 16384 compact JSON bytes",
9594        );
9595    }
9596
9597    #[test]
9598    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9599        let registry = Arc::new(Registry::default());
9600        let handler = ControlHandler::new(registry.clone());
9601        let descriptor = json!({
9602            "schema": "ckios.richui.manifest/1",
9603            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9604            "bytes": 128,
9605            "chunks": 2
9606        });
9607        let malformed = [
9608            ("schema", json!("")),
9609            ("schema", json!("a".repeat(65))),
9610            ("schema", json!("Bad schema")),
9611            (
9612                "manifest_digest",
9613                json!(format!("sha256:{}", "A".repeat(64))),
9614            ),
9615            ("manifest_digest", json!("sha256:abc")),
9616            ("bytes", json!(0)),
9617            ("bytes", json!(16_777_217)),
9618            ("bytes", json!("128")),
9619            ("chunks", json!(0)),
9620            ("chunks", json!(1025)),
9621            ("chunks", json!("2")),
9622        ];
9623        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9624            let mut invalid = descriptor.clone();
9625            invalid[field] = invalid_value;
9626            let response = handler
9627                .handle_control(
9628                    ConnectionId::new(210 + index as u64),
9629                    hello_frame_with_rich_ui(
9630                        &format!("bad-rich-ui-{index}"),
9631                        PROTOCOL_VERSION,
9632                        210 + index as u64,
9633                        None,
9634                        invalid,
9635                    ),
9636                )
9637                .unwrap();
9638            assert_eq!(response[0].header.ty, FrameType::Error);
9639            let error = parse_error(&response[0]);
9640            assert_eq!(error["code"], "invalid_manifest");
9641            assert!(error["message"]
9642                .as_str()
9643                .unwrap()
9644                .contains(&format!("rich_ui.{field}")));
9645            assert!(registry
9646                .get_module(&format!("bad-rich-ui-{index}"))
9647                .unwrap()
9648                .is_none());
9649        }
9650    }
9651
9652    #[test]
9653    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9654        let registry = Arc::new(Registry::default());
9655        let handler = ControlHandler::new(Arc::clone(&registry));
9656        let connection = ConnectionId::new(219);
9657        registry
9658            .register_with_control_ops(
9659                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9660                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9661                        "bad schema",
9662                        format!("sha256:{}", "a".repeat(64)),
9663                        128,
9664                        2,
9665                    )))
9666                    .build(),
9667                PROTOCOL_VERSION,
9668                connection,
9669                Vec::new(),
9670            )
9671            .unwrap();
9672        let request =
9673            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9674        let response = handler
9675            .handle_catalog_update(connection, request, Vec::new(), None, None)
9676            .unwrap();
9677        assert_eq!(response[0].header.ty, FrameType::Error);
9678        let error = parse_error(&response[0]);
9679        assert_eq!(error["code"], "invalid_manifest");
9680        assert!(error["message"]
9681            .as_str()
9682            .unwrap()
9683            .contains("rich_ui.schema"));
9684    }
9685
9686    #[test]
9687    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9688        let registry = Arc::new(Registry::default());
9689        let supervisor = SupervisorHandle::new();
9690        supervisor.open_swap("provider", "candidate-nonce".to_string());
9691        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9692        let connection = ConnectionId::new(218);
9693        let hello = handler
9694            .handle_control(
9695                connection,
9696                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9697            )
9698            .unwrap();
9699        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9700        assert!(registry.list_modules().unwrap().1.is_empty());
9701
9702        registry.promote_candidate("provider").unwrap().unwrap();
9703        let request = Frame::build(
9704            FrameType::Request,
9705            control_flags(),
9706            0,
9707            0,
9708            219,
9709            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9710        )
9711        .unwrap();
9712        let response = handler.handle_catalog_list(request, None).unwrap();
9713        let ClientControlResponse::CatalogList { modules, .. } =
9714            serde_json::from_slice(&response[0].body).unwrap()
9715        else {
9716            panic!("expected catalog.list");
9717        };
9718        assert_eq!(modules.len(), 1);
9719        assert!(modules[0].nonce_verified);
9720    }
9721
9722    #[test]
9723    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9724        let registry = Arc::new(Registry::default());
9725        let handler = ControlHandler::new(Arc::clone(&registry));
9726        let hello = Frame::build(
9727            FrameType::Hello,
9728            control_flags(),
9729            0,
9730            0,
9731            220,
9732            serde_json::to_vec(&ModuleHelloBody {
9733                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9734                protocol_ver: PROTOCOL_VERSION,
9735                control_ops: None,
9736                launch_nonce: Some("unverified".to_string()),
9737            })
9738            .unwrap(),
9739        )
9740        .unwrap();
9741        assert_eq!(
9742            handler
9743                .handle_control(ConnectionId::new(220), hello)
9744                .unwrap()[0]
9745                .header
9746                .ty,
9747            FrameType::HelloAck
9748        );
9749        let request = Frame::build(
9750            FrameType::Request,
9751            control_flags(),
9752            0,
9753            0,
9754            221,
9755            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9756        )
9757        .unwrap();
9758        let response = handler.handle_catalog_list(request, None).unwrap();
9759        let ClientControlResponse::CatalogList { modules, .. } =
9760            serde_json::from_slice(&response[0].body).unwrap()
9761        else {
9762            panic!("expected catalog.list");
9763        };
9764        assert_eq!(
9765            serde_json::to_vec(&modules[0]).unwrap(),
9766            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9767        );
9768        assert!(!modules[0].nonce_verified);
9769    }
9770
9771    #[test]
9772    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9773        let registry = Arc::new(Registry::default());
9774        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9775            [("vault".to_string(), true), ("squatter".to_string(), true)],
9776            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9777        );
9778        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9779        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9780            provides: vec!["credentials-provider/v1".to_string()],
9781            requires: Vec::new(),
9782            must_never_reach: Vec::new(),
9783        });
9784        let frame = Frame::build(
9785            FrameType::Hello,
9786            control_flags(),
9787            0,
9788            0,
9789            77,
9790            serde_json::to_vec(&ModuleHelloBody {
9791                manifest: squatter,
9792                protocol_ver: PROTOCOL_VERSION,
9793                control_ops: None,
9794                launch_nonce: None,
9795            })
9796            .expect("HELLO serializes"),
9797        )
9798        .expect("HELLO frame builds");
9799        let response = handler
9800            .handle_control(ConnectionId::new(77), frame)
9801            .expect("reserved claim receives a typed refusal");
9802        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9803        assert_eq!(
9804            registry
9805                .active_registration_count()
9806                .expect("registry reads"),
9807            0,
9808            "a reserved capability refusal must not leave a catalog entry"
9809        );
9810    }
9811
9812    #[test]
9813    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9814        for settlement in ["timeout", "inconclusive", "drop"] {
9815            let breakers = RouteBindBreakers::default();
9816            let RouteBindAdmission::Admitted {
9817                guard: mut old,
9818                probe: false,
9819            } = breakers.admit("prov")
9820            else {
9821                panic!("ordinary relay admitted")
9822            };
9823            let RouteBindAdmission::Admitted {
9824                guard: mut opener, ..
9825            } = breakers.admit("prov")
9826            else {
9827                panic!("second relay admitted")
9828            };
9829            assert!(
9830                !opener
9831                    .record_timeout(1, Duration::ZERO)
9832                    .unwrap()
9833                    .reopened_after_probe
9834            );
9835            let RouteBindAdmission::Admitted {
9836                guard: mut probe,
9837                probe: true,
9838            } = breakers.admit("prov")
9839            else {
9840                panic!("one cooldown probe admitted")
9841            };
9842            match settlement {
9843                "timeout" => assert!(
9844                    !old.record_timeout(1, Duration::ZERO)
9845                        .unwrap()
9846                        .reopened_after_probe
9847                ),
9848                "inconclusive" => old.record_inconclusive(),
9849                "drop" => drop(old),
9850                _ => unreachable!(),
9851            }
9852            assert!(
9853                matches!(
9854                    breakers.admit("prov"),
9855                    RouteBindAdmission::Refused {
9856                        probe_in_flight: true,
9857                        ..
9858                    }
9859                ),
9860                "{settlement} of a pre-open relay cannot release the real probe"
9861            );
9862            assert!(
9863                probe
9864                    .record_timeout(1, Duration::ZERO)
9865                    .unwrap()
9866                    .reopened_after_probe
9867            );
9868            assert!(matches!(
9869                breakers.admit("prov"),
9870                RouteBindAdmission::Admitted { probe: true, .. }
9871            ));
9872        }
9873        let breakers = RouteBindBreakers::default();
9874        let admit = || match breakers.admit("prov") {
9875            RouteBindAdmission::Admitted { guard, .. } => guard,
9876            _ => panic!("relay admitted"),
9877        };
9878        admit().record_timeout(1, Duration::ZERO);
9879        let mut old_probe = admit();
9880        breakers.reset_for_new_module_connection("prov");
9881        admit().record_timeout(1, Duration::ZERO);
9882        let _new_probe = admit();
9883        old_probe.record_inconclusive();
9884        assert!(matches!(
9885            breakers.admit("prov"),
9886            RouteBindAdmission::Refused {
9887                probe_in_flight: true,
9888                ..
9889            }
9890        ));
9891    }
9892
9893    #[tokio::test]
9894    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9895        for candidate in [false, true] {
9896            let registry = Arc::new(Registry::default());
9897            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9898                [("vault".to_string(), true), ("squatter".to_string(), true)],
9899                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9900            );
9901            let conn = ConnectionId::new(77);
9902            let (ctx, mut rx) = route_ctx(conn);
9903            let initial = capability_manifest("squatter", &[], &[]);
9904            if candidate {
9905                registry
9906                    .register_candidate_with_control_ops(
9907                        initial.clone(),
9908                        PROTOCOL_VERSION,
9909                        conn,
9910                        module_baseline_control_ops(),
9911                    )
9912                    .unwrap();
9913                handler
9914                    .forwarding
9915                    .register_candidate_module_connection(
9916                        conn,
9917                        "squatter".to_string(),
9918                        PROTOCOL_VERSION,
9919                        manifest_concurrency(&initial),
9920                        ctx.egress.clone(),
9921                    )
9922                    .unwrap();
9923            } else {
9924                hello_via_sink(
9925                    &handler,
9926                    &ctx,
9927                    &mut rx,
9928                    hello_frame_with_manifest(initial.clone(), 1),
9929                )
9930                .await;
9931            }
9932            let response = handler
9933                .handle_control_frame(
9934                    &ctx,
9935                    catalog_update_with_capabilities_frame(
9936                        2,
9937                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
9938                            .capabilities
9939                            .unwrap(),
9940                    ),
9941                )
9942                .await
9943                .unwrap();
9944            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9945            assert_eq!(
9946                registry
9947                    .get_module_by_connection(conn)
9948                    .unwrap()
9949                    .unwrap()
9950                    .manifest,
9951                initial
9952            );
9953        }
9954    }
9955
9956    #[test]
9957    fn server_describe_surfaces_required_capability_verdict_fields() {
9958        let registry = Arc::new(Registry::default());
9959        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9960            [
9961                ("consumer".to_string(), true),
9962                ("provider".to_string(), false),
9963            ],
9964            BTreeMap::new(),
9965        );
9966        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9967        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9968            provides: Vec::new(),
9969            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9970                capability: "credentials-provider/v1".to_string(),
9971                need: subc_protocol::manifest::CapabilityNeed::Required,
9972            }],
9973            must_never_reach: Vec::new(),
9974        });
9975        let hello = Frame::build(
9976            FrameType::Hello,
9977            control_flags(),
9978            0,
9979            0,
9980            78,
9981            serde_json::to_vec(&ModuleHelloBody {
9982                manifest: consumer,
9983                protocol_ver: PROTOCOL_VERSION,
9984                control_ops: None,
9985                launch_nonce: None,
9986            })
9987            .expect("HELLO serializes"),
9988        )
9989        .expect("HELLO frame builds");
9990        handler
9991            .handle_control(ConnectionId::new(78), hello)
9992            .expect("consumer registers");
9993        let describe = Frame::build(
9994            FrameType::Request,
9995            control_flags(),
9996            0,
9997            0,
9998            79,
9999            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10000                .expect("request serializes"),
10001        )
10002        .expect("describe frame builds");
10003        let response = handler
10004            .handle_server_describe(describe)
10005            .expect("server.describe succeeds");
10006        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10007        let requirement = &rendered["capability_requirements"][0];
10008        assert_eq!(requirement["consumer"], "consumer");
10009        assert_eq!(requirement["verdict"], "never_provided");
10010        assert_eq!(requirement["episode_seq"], 1);
10011        assert_eq!(requirement["config_satisfiable"], false);
10012        assert_eq!(requirement["runtime_available"], false);
10013        assert!(requirement["detail"]
10014            .as_str()
10015            .expect("detail string")
10016            .contains("credentials-provider/v1"));
10017    }
10018
10019    /// Register a consumer that requires `project-identity/v1` and the provider
10020    /// of it, optionally mark daemon shutdown as begun, then deregister the
10021    /// provider as its connection closes. Returns the `capability.requirement`
10022    /// events that deregistration logged and the verdict the evaluator then
10023    /// holds for the requirement (the one `route.open` refusals read).
10024    fn deregister_required_provider(
10025        daemon_shutdown_begun: bool,
10026    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10027        const CAPABILITY: &str = "project-identity/v1";
10028        let registry = Arc::new(Registry::default());
10029        let daemon_shutdown = DaemonShutdownFlag::default();
10030        let handler = ControlHandler::new(Arc::clone(&registry))
10031            .with_capability_config(
10032                [
10033                    ("consumer".to_string(), true),
10034                    ("provider".to_string(), true),
10035                ],
10036                BTreeMap::new(),
10037            )
10038            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10039        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10040        consumer.capabilities = Some(CapabilityDeclarations {
10041            provides: Vec::new(),
10042            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10043                capability: CAPABILITY.to_string(),
10044                need: subc_protocol::manifest::CapabilityNeed::Required,
10045            }],
10046            must_never_reach: Vec::new(),
10047        });
10048        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10049        let provider_conn = ConnectionId::new(182);
10050        for (conn, manifest, corr) in [
10051            (ConnectionId::new(181), consumer, 181),
10052            (provider_conn, provider, 182),
10053        ] {
10054            let hello = handler
10055                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10056                .expect("module registers");
10057            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10058        }
10059        assert_eq!(
10060            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10061            Some(CapabilityVerdict::Provided),
10062            "the registered provider must satisfy the requirement first"
10063        );
10064        if daemon_shutdown_begun {
10065            daemon_shutdown.set_for_test();
10066        }
10067
10068        let capture = EventCapture::default();
10069        let _subscriber =
10070            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10071        handler
10072            .cleanup_connection(provider_conn)
10073            .expect("provider deregisters");
10074        let events = capture
10075            .events()
10076            .into_iter()
10077            .filter(|event| {
10078                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10079            })
10080            .collect();
10081        (
10082            events,
10083            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10084        )
10085    }
10086
10087    #[test]
10088    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10089        let (events, verdict) = deregister_required_provider(true);
10090        assert_eq!(
10091            verdict,
10092            Some(CapabilityVerdict::NeverProvided),
10093            "shutdown must not change the verdict route.open refusals read"
10094        );
10095        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10096        let event = &events[0];
10097        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10098        assert_eq!(
10099            event.fields.get("cause").map(String::as_str),
10100            Some("\"daemon_shutdown\""),
10101            "{event:?}"
10102        );
10103        assert_eq!(
10104            event.fields.get("verdict").map(String::as_str),
10105            Some("\"never_provided\""),
10106            "{event:?}"
10107        );
10108    }
10109
10110    #[test]
10111    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10112        let (events, verdict) = deregister_required_provider(false);
10113        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10114        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10115        let event = &events[0];
10116        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10117        assert_eq!(event.fields.get("cause"), None, "{event:?}");
10118        assert_eq!(
10119            event.fields.get("verdict").map(String::as_str),
10120            Some("\"never_provided\""),
10121            "{event:?}"
10122        );
10123    }
10124
10125    #[test]
10126    fn catalog_list_omits_capabilities_for_legacy_manifest() {
10127        let registry = Arc::new(Registry::default());
10128        let handler = ControlHandler::new(Arc::clone(&registry));
10129        let hello = handler
10130            .handle_control(
10131                ConnectionId::new(101),
10132                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10133            )
10134            .expect("legacy HELLO registers");
10135        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10136
10137        let request = Frame::build(
10138            FrameType::Request,
10139            control_flags(),
10140            0,
10141            0,
10142            102,
10143            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10144                .expect("catalog request serializes"),
10145        )
10146        .expect("catalog request frame builds");
10147        let response = handler
10148            .handle_catalog_list(request, None)
10149            .expect("catalog list succeeds");
10150        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10151        assert!(
10152            body["modules"][0].get("capabilities").is_none(),
10153            "legacy manifest must retain an absent capabilities field on catalog.list"
10154        );
10155    }
10156
10157    #[test]
10158    fn hello_ack_omits_storage_when_no_storage_config() {
10159        let registry = Arc::new(Registry::default());
10160        let handler = ControlHandler::new(Arc::clone(&registry));
10161        let responses = handler
10162            .handle_control(
10163                ConnectionId::new(1),
10164                hello_frame("aft", PROTOCOL_VERSION, 7),
10165            )
10166            .unwrap();
10167        let ack = parse_ack(&responses[0]);
10168        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10169        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10170    }
10171
10172    #[tokio::test]
10173    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10174        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10175        let registry = Arc::new(Registry::default());
10176        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
10177        let responses = handler
10178            .handle_control(
10179                ConnectionId::new(1),
10180                hello_frame("aft", PROTOCOL_VERSION, 7),
10181            )
10182            .unwrap();
10183        let ack = parse_ack(&responses[0]);
10184        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10185
10186        let described = handler
10187            .handle_control_frame(
10188                &route_ctx(ConnectionId::new(2)).0,
10189                Frame::build(
10190                    FrameType::Request,
10191                    control_flags(),
10192                    0,
10193                    0,
10194                    9,
10195                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10196                )
10197                .unwrap(),
10198            )
10199            .await
10200            .unwrap();
10201        let ClientControlResponse::ServerDescribe { machine_id, .. } =
10202            serde_json::from_slice(&described[0].body).unwrap()
10203        else {
10204            panic!("server.describe answered with another shape");
10205        };
10206        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10207    }
10208
10209    #[test]
10210    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10211        // With a central sqlite storage policy, each registering module gets its
10212        // own resolved descriptor in HELLO_ACK, keyed by its module id.
10213        let registry = Arc::new(Registry::default());
10214        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
10215            crate::daemon_config::StorageConfig::Sqlite {
10216                data_home: std::path::PathBuf::from("/data"),
10217            },
10218        ));
10219
10220        let responses = handler
10221            .handle_control(
10222                ConnectionId::new(1),
10223                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10224            )
10225            .unwrap();
10226        let ack = parse_ack(&responses[0]);
10227        assert_eq!(
10228            ack.storage,
10229            Some(serde_json::json!({
10230                "module_id": "alfonso-routing",
10231                "storage_namespace": "default",
10232                "isolation": { "kind": "module" },
10233                "backend": {
10234                    "backend": "sqlite",
10235                    "path": "/data/cortexkit/alfonso-routing/store.db"
10236                }
10237            })),
10238            "the delivered descriptor is the module's own sqlite store path"
10239        );
10240    }
10241
10242    #[test]
10243    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10244        let registry = Arc::new(Registry::default());
10245        let handler = ControlHandler::new(Arc::clone(&registry));
10246        let conn = ConnectionId::new(1);
10247        let responses = handler
10248            .handle_control(
10249                conn,
10250                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10251            )
10252            .unwrap();
10253        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10254        let registration = registry.get_module("aft").unwrap().unwrap();
10255        assert_eq!(registration.control_ops, module_baseline_control_ops());
10256
10257        let frame =
10258            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10259        assert!(handler
10260            .guard_module_control_op(&frame, "aft", "route.bind")
10261            .unwrap()
10262            .is_none());
10263        let error = handler
10264            .guard_module_control_op(&frame, "aft", "test.synthetic")
10265            .unwrap()
10266            .expect("synthetic ungranted op should be rejected");
10267        assert_eq!(error.header.ty, FrameType::Error);
10268        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10269    }
10270
10271    #[test]
10272    fn hello_control_ops_some_adds_optional_grants() {
10273        let registry = Arc::new(Registry::default());
10274        let handler = ControlHandler::new(Arc::clone(&registry));
10275        handler
10276            .handle_control(
10277                ConnectionId::new(1),
10278                hello_frame_with_control_ops(
10279                    "aft",
10280                    PROTOCOL_VERSION,
10281                    7,
10282                    Some(vec![
10283                        "future.synthetic".to_string(),
10284                        "route.bind".to_string(),
10285                    ]),
10286                ),
10287            )
10288            .unwrap();
10289        let registration = registry.get_module("aft").unwrap().unwrap();
10290        assert_eq!(
10291            registration.control_ops,
10292            vec![
10293                "route.bind".to_string(),
10294                "route.status".to_string(),
10295                "future.synthetic".to_string(),
10296            ]
10297        );
10298        let frame =
10299            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10300        assert!(handler
10301            .guard_module_control_op(&frame, "aft", "future.synthetic")
10302            .unwrap()
10303            .is_none());
10304    }
10305
10306    #[tokio::test]
10307    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10308        let registry = Arc::new(Registry::default());
10309        let forwarding = Arc::new(ForwardingTable::default());
10310        let handler =
10311            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10312        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10313        hello_via_sink(
10314            &handler,
10315            &module_ctx,
10316            &mut module_rx,
10317            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10318        )
10319        .await;
10320
10321        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10322        let responses = handler
10323            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10324            .await
10325            .unwrap();
10326        assert_eq!(responses.len(), 1);
10327        assert_eq!(responses[0].header.ty, FrameType::Error);
10328        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10329        assert!(module_rx.try_recv().is_err());
10330    }
10331
10332    #[tokio::test]
10333    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10334        let registry = Arc::new(Registry::default());
10335        let forwarding = Arc::new(ForwardingTable::default());
10336        let handler =
10337            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10338        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10339        hello_via_sink(
10340            &handler,
10341            &module_ctx,
10342            &mut module_rx,
10343            hello_frame_with_control_ops(
10344                "aft",
10345                PROTOCOL_VERSION,
10346                7,
10347                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10348            ),
10349        )
10350        .await;
10351
10352        let project_root = unique_project_root("demux");
10353        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10354        let route_handler = handler.clone();
10355        let route_task = tokio::spawn(async move {
10356            route_handler
10357                .handle_control_frame(
10358                    &route_client_ctx,
10359                    route_open_frame(100, "aft", project_root),
10360                )
10361                .await
10362                .unwrap()
10363        });
10364        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10365            .await
10366            .unwrap()
10367            .unwrap();
10368        assert!(matches!(
10369            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10370            ModuleControlRequest::RouteBind { .. }
10371        ));
10372
10373        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10374        let health_handler = handler.clone();
10375        let health_task = tokio::spawn(async move {
10376            health_handler
10377                .handle_control_frame(
10378                    &health_client_ctx,
10379                    supervisor_health_probe_frame(101, "aft"),
10380                )
10381                .await
10382                .unwrap()
10383        });
10384        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10385            .await
10386            .unwrap()
10387            .unwrap();
10388        assert_eq!(
10389            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10390            ModuleControlRequest::HealthCheck {}
10391        );
10392
10393        handler
10394            .handle_control_frame(
10395                &module_ctx,
10396                health_response(health_frame.header.corr, HealthStatus::Degraded),
10397            )
10398            .await
10399            .unwrap();
10400        let health_response = health_task.await.unwrap();
10401        assert_eq!(health_response.len(), 1);
10402        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10403            ClientControlResponse::SupervisorHealthProbe {
10404                module_id,
10405                status,
10406                detail,
10407                metrics,
10408            } => {
10409                assert_eq!(module_id, "aft");
10410                assert_eq!(status, HealthStatus::Degraded);
10411                assert_eq!(detail.as_deref(), Some("warming"));
10412                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10413            }
10414            other => panic!("unexpected health response: {other:?}"),
10415        }
10416
10417        handler
10418            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10419            .await
10420            .unwrap();
10421        let route_response = route_task.await.unwrap();
10422        assert!(route_response.is_empty());
10423        let published = route_client_rx.recv().await.unwrap();
10424        assert!(matches!(
10425            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10426            ClientControlResponse::RouteOpen { .. }
10427        ));
10428    }
10429
10430    /// Start one `route.open` on `client_connection` and return its still-running
10431    /// handler task together with the `route.bind` the module received for it.
10432    /// The handler blocks until the module answers, so it has to run as a task
10433    /// while the test drives the module side.
10434    async fn relay_route_open(
10435        handler: &ControlHandler,
10436        client_connection: ConnectionId,
10437        client_egress: &FrameSink,
10438        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10439        corr: u64,
10440        module_id: &str,
10441        project_root_label: &str,
10442    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10443        let ctx = RouteCtx {
10444            connection_id: client_connection,
10445            egress: client_egress.clone(),
10446        };
10447        let handler = handler.clone();
10448        let project_root = unique_project_root(project_root_label);
10449        let module_id = module_id.to_string();
10450        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10451        let task = tokio::spawn(async move {
10452            let _guard = tracing::dispatcher::set_default(&dispatch);
10453            handler
10454                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10455                .await
10456                .unwrap()
10457        });
10458        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10459            .await
10460            .expect("module receives the relayed route.bind")
10461            .expect("module egress is open");
10462        (task, bind.frame)
10463    }
10464
10465    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10466        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10467            ModuleControlRequest::RouteBind {
10468                route_channel,
10469                epoch,
10470                ..
10471            } => (route_channel, epoch),
10472            other => panic!("expected a route.bind request, got {other:?}"),
10473        }
10474    }
10475
10476    fn published_route(frame: &Frame) -> (u16, u32) {
10477        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10478            ClientControlResponse::RouteOpen {
10479                route_channel,
10480                route_epoch,
10481            } => (route_channel, route_epoch),
10482            other => panic!("expected a route.open response, got {other:?}"),
10483        }
10484    }
10485
10486    /// Reproduction of a production outage. A client had `route.open`s in
10487    /// flight to a module and was already marked closing -- its egress had refused a
10488    /// module frame, so the daemon asked its connection to end -- while its sink
10489    /// was still open. When the module acked those binds, the daemon refused to
10490    /// commit a route for a closing client, and that refusal was returned from
10491    /// the MODULE connection's frame handler, where a router error that has no
10492    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
10493    /// the supervisor correctly did not respawn a clean exit, and every seat lost
10494    /// its tools for hours -- one client's teardown took down a connection
10495    /// carrying ~170 other routes.
10496    ///
10497    /// The window is opened here by calling the production path that opens it
10498    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
10499    /// state that matters is "in `closing_connections`, sink still open, relay
10500    /// still pending", and it lasts only from the close request until the
10501    /// connection loop reacts to it; a socket-level test can flood a client into
10502    /// that escalation but cannot pin the module's ack inside the window. Closing
10503    /// the socket instead takes the other path entirely -- connection teardown
10504    /// removes the pending relay under the same lock, so the ack finds nothing.
10505    #[tokio::test]
10506    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10507        let registry = Arc::new(Registry::default());
10508        let forwarding = Arc::new(ForwardingTable::default());
10509        let handler =
10510            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10511
10512        let module_connection = ConnectionId::new(30);
10513        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10514        hello_via_sink(
10515            &handler,
10516            &module_ctx,
10517            &mut module_rx,
10518            hello_frame("aft", PROTOCOL_VERSION, 7),
10519        )
10520        .await;
10521
10522        let dying_client = ConnectionId::new(31);
10523        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10524
10525        // A published route on the dying client. The escalation below only marks
10526        // a connection closing for a route it has already published.
10527        let (first_task, first_bind) = relay_route_open(
10528            &handler,
10529            dying_client,
10530            &dying_ctx.egress,
10531            &mut module_rx,
10532            100,
10533            "aft",
10534            "closing-first",
10535        )
10536        .await;
10537        handler
10538            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10539            .await
10540            .unwrap();
10541        assert!(first_task.await.unwrap().is_empty());
10542        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10543
10544        // A second route.open from the same client, relayed and awaiting its ack.
10545        let (second_task, second_bind) = relay_route_open(
10546            &handler,
10547            dying_client,
10548            &dying_ctx.egress,
10549            &mut module_rx,
10550            101,
10551            "aft",
10552            "closing-second",
10553        )
10554        .await;
10555        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10556
10557        // The window: the client is closing, its sink is still open, and its
10558        // second bind is still pending.
10559        assert!(forwarding
10560            .escalate_client_delivery_failure(
10561                dying_client,
10562                first_channel,
10563                first_epoch,
10564                CloseReason::new(
10565                    "module_to_client_delivery_failed",
10566                    "client egress refused a module frame",
10567                ),
10568                crate::forwarding::UndeliveredFrame {
10569                    module_id: None,
10570                    sink: &dying_ctx.egress,
10571                },
10572            )
10573            .unwrap());
10574        assert!(!dying_ctx.egress.is_closed());
10575
10576        // The frame that used to end the module connection.
10577        let ack = handler
10578            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10579            .await;
10580        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10581        if module_loop_error.is_some() {
10582            // What the server's connection loop does with a router error that has
10583            // no ERROR-frame translation: end the connection, which releases the
10584            // module's registration and every route on it.
10585            handler.cleanup_connection(module_connection).unwrap();
10586        }
10587        // Read the module's next frame before opening the co-tenant's route, so
10588        // the GOODBYE assertion below is about THIS ack and not about later
10589        // traffic. `None` means the module was told nothing.
10590        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10591            .await
10592            .ok()
10593            .flatten();
10594
10595        // 1. The module connection is still registered.
10596        assert!(
10597            registry
10598                .get_module_by_connection(module_connection)
10599                .unwrap()
10600                .is_some(),
10601            "one client's closing connection ended the shared module connection: \
10602             {module_loop_error:?}"
10603        );
10604        // ...and still serving: another client can open and use a route on it.
10605        let cotenant = ConnectionId::new(32);
10606        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10607        let (cotenant_task, cotenant_bind) = relay_route_open(
10608            &handler,
10609            cotenant,
10610            &cotenant_ctx.egress,
10611            &mut module_rx,
10612            102,
10613            "aft",
10614            "closing-cotenant",
10615        )
10616        .await;
10617        handler
10618            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10619            .await
10620            .unwrap();
10621        assert!(cotenant_task.await.unwrap().is_empty());
10622        let (cotenant_channel, cotenant_epoch) =
10623            published_route(&cotenant_rx.recv().await.unwrap());
10624        assert!(matches!(
10625            forwarding
10626                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10627                .unwrap(),
10628            DataRoute::Client(DataRouteState::Bound(_))
10629        ));
10630
10631        // 2. The module was told to drop the binding it created for the route
10632        //    that will never be published.
10633        let goodbye = post_ack_module_frame
10634            .expect("module receives a GOODBYE for the abandoned route channel");
10635        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10636        assert_eq!(goodbye.header.channel, abandoned_channel);
10637        assert_eq!(goodbye.header.epoch, abandoned_epoch);
10638
10639        // 3. The dying client received nothing: no route was ever published to
10640        //    it. Its route.open is answered as unavailable, which the connection
10641        //    loop would write to a socket that is already going away.
10642        assert!(dying_rx.try_recv().is_err());
10643        let second_response = second_task.await.unwrap();
10644        assert_eq!(second_response.len(), 1);
10645        assert_eq!(
10646            parse_error(&second_response[0])["code"],
10647            "target_unavailable"
10648        );
10649    }
10650
10651    /// One `catalog.list` as a client sees it: the generation and the listed
10652    /// module ids with their readiness.
10653    async fn catalog_snapshot(
10654        handler: &ControlHandler,
10655        client_ctx: &RouteCtx,
10656        corr: u64,
10657    ) -> (u64, Vec<(String, bool)>) {
10658        let body =
10659            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10660        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10661        let reply = handler
10662            .handle_control_frame(client_ctx, request)
10663            .await
10664            .unwrap()
10665            .pop()
10666            .unwrap();
10667        let ClientControlResponse::CatalogList {
10668            generation,
10669            modules,
10670            ..
10671        } = serde_json::from_slice(&reply.body).unwrap()
10672        else {
10673            panic!("not a catalog.list reply");
10674        };
10675        (
10676            generation,
10677            modules
10678                .into_iter()
10679                .map(|entry| (entry.module_id, entry.ready))
10680                .collect(),
10681        )
10682    }
10683
10684    /// A handler whose HELLO for module "aft" on connection 40 is parked
10685    /// between reserving its registry entry and installing its forwarding
10686    /// endpoint. Returns the handler, the two-party barrier (wait once to know
10687    /// the HELLO is parked, once more to release it), the HELLO's thread, and
10688    /// the module's context and receiver.
10689    fn hello_held_before_forwarding() -> (
10690        ControlHandler,
10691        Arc<std::sync::Barrier>,
10692        std::thread::JoinHandle<Vec<Frame>>,
10693        RouteCtx,
10694        mpsc::Receiver<crate::router::OutboundFrame>,
10695    ) {
10696        let hold = Arc::new(std::sync::Barrier::new(2));
10697        let handler = ControlHandler::with_forwarding(
10698            Arc::new(Registry::default()),
10699            Arc::new(ForwardingTable::default()),
10700        )
10701        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10702        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10703        let hello = {
10704            let handler = handler.clone();
10705            let module_ctx = module_ctx.clone();
10706            std::thread::spawn(move || {
10707                handler
10708                    .handle_hello(
10709                        module_ctx.connection_id,
10710                        Some(module_ctx.egress.clone()),
10711                        hello_frame("aft", PROTOCOL_VERSION, 7),
10712                    )
10713                    .unwrap()
10714            })
10715        };
10716        (handler, hold, hello, module_ctx, module_rx)
10717    }
10718
10719    /// While a HELLO has reserved its registry entry but its forwarding
10720    /// endpoint is not installed yet, `route.open` to the module can only be
10721    /// refused. `catalog.list` must not list the module in that window: a
10722    /// client that opens a route as soon as it sees a module listed and ready
10723    /// would be refused for a module the daemon had just advertised. The
10724    /// refusal in the window stays the retryable `target_unavailable`, never
10725    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10726    /// Once the HELLO completes, the module is listed and a route to it opens.
10727    #[tokio::test]
10728    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10729        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10730        let client = ConnectionId::new(41);
10731        let (client_ctx, mut client_rx) = route_ctx(client);
10732
10733        hold.wait();
10734        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10735        assert!(handler.registry.get_module("aft").unwrap().is_some());
10736        assert!(!handler
10737            .forwarding
10738            .has_live_module_connection("aft")
10739            .unwrap());
10740        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10741        assert!(
10742            listed.is_empty(),
10743            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10744        );
10745        let refused = handler
10746            .handle_control_frame(
10747                &client_ctx,
10748                route_open_frame(301, "aft", unique_project_root("listed-window")),
10749            )
10750            .await
10751            .unwrap();
10752        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10753        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10754        let duplicate = handler
10755            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10756            .await
10757            .unwrap();
10758        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10759
10760        hold.wait();
10761        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10762        let ack = module_rx.try_recv().unwrap().frame;
10763        assert_eq!(ack.header.ty, FrameType::HelloAck);
10764        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10765        assert_eq!(listed, vec![("aft".to_string(), true)]);
10766
10767        let (open_task, bind) = relay_route_open(
10768            &handler,
10769            client,
10770            &client_ctx.egress,
10771            &mut module_rx,
10772            303,
10773            "aft",
10774            "listed-after",
10775        )
10776        .await;
10777        assert!(handler
10778            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10779            .await
10780            .unwrap()
10781            .is_empty());
10782        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10783        published_route(&client_rx.try_recv().unwrap().frame);
10784    }
10785
10786    /// A catalog watcher keys on the generation: when it changes, it re-reads
10787    /// the listing. So the generation must advance exactly when the module
10788    /// appears. If it advanced while the module was still hidden, a watcher
10789    /// would read the new generation without the module, and the module would
10790    /// later appear under that same generation with nothing telling the
10791    /// watcher to look again.
10792    #[tokio::test]
10793    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10794        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10795        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10796        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10797
10798        hold.wait();
10799        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10800        assert!(listed.is_empty());
10801        assert_eq!(
10802            during, before,
10803            "the generation moved while the module was not listed"
10804        );
10805
10806        hold.wait();
10807        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10808        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10809        assert_eq!(listed, vec![("aft".to_string(), true)]);
10810        assert_eq!(
10811            after,
10812            before + 1,
10813            "the module appeared without exactly one generation bump"
10814        );
10815    }
10816
10817    /// A HELLO whose forwarding endpoint cannot be installed (here the
10818    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
10819    /// refused and leaves nothing behind: no registry entry, no catalog entry,
10820    /// and the catalog generation where it was.
10821    #[tokio::test]
10822    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10823        let registry = Arc::new(Registry::default());
10824        let handler = ControlHandler::with_forwarding(
10825            Arc::clone(&registry),
10826            Arc::new(ForwardingTable::default()),
10827        );
10828        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10829        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10830        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10831        drop(module_rx);
10832
10833        let replies = handler
10834            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10835            .await
10836            .unwrap();
10837        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10838        assert!(registry.get_module("aft").unwrap().is_none());
10839        assert!(registry
10840            .get_module_by_connection(module_ctx.connection_id)
10841            .unwrap()
10842            .is_none());
10843        assert_eq!(
10844            catalog_snapshot(&handler, &client_ctx, 301).await,
10845            (before, Vec::new())
10846        );
10847    }
10848
10849    /// A handler whose `route.open` parks after queueing the route.bind and
10850    /// before marking it enqueued, with module "aft" registered on connection
10851    /// 40. Returns the two-party barrier that parks and releases it.
10852    async fn route_open_held_after_bind_send() -> (
10853        ControlHandler,
10854        Arc<tokio::sync::Barrier>,
10855        RouteCtx,
10856        mpsc::Receiver<crate::router::OutboundFrame>,
10857    ) {
10858        let hold = Arc::new(tokio::sync::Barrier::new(2));
10859        let handler = ControlHandler::with_forwarding(
10860            Arc::new(Registry::default()),
10861            Arc::new(ForwardingTable::default()),
10862        )
10863        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10864        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10865        hello_via_sink(
10866            &handler,
10867            &module_ctx,
10868            &mut module_rx,
10869            hello_frame("aft", PROTOCOL_VERSION, 7),
10870        )
10871        .await;
10872        (handler, hold, module_ctx, module_rx)
10873    }
10874
10875    /// A fast module acks the route.bind, and the reader commits the route,
10876    /// before `route.open` has marked the relay enqueued. The daemon used to
10877    /// read the missing relay as an abandoned bind and send the module a
10878    /// GOODBYE for the route it had just committed: the client was told its
10879    /// route was open, the module unbound the channel, and the client's first
10880    /// request was dropped on the module side as unbound.
10881    #[tokio::test]
10882    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10883        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10884        let client = ConnectionId::new(41);
10885        let (client_ctx, mut client_rx) = route_ctx(client);
10886
10887        let (open_task, bind) = relay_route_open(
10888            &handler,
10889            client,
10890            &client_ctx.egress,
10891            &mut module_rx,
10892            200,
10893            "aft",
10894            "acked-before-mark",
10895        )
10896        .await;
10897        let (module_channel, module_epoch) = route_bind_channel(&bind);
10898        // route.open is now parked between queueing the bind and marking it.
10899        hold.wait().await;
10900
10901        // The module answers inside that window, and the route commits.
10902        assert!(handler
10903            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10904            .await
10905            .unwrap()
10906            .is_empty());
10907        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10908
10909        hold.wait().await;
10910        assert!(
10911            open_task.await.unwrap().is_empty(),
10912            "route.open answers accepted"
10913        );
10914
10915        // Every frame route.open could have sent is already queued, because it
10916        // has returned; the module must have been sent nothing.
10917        match module_rx.try_recv() {
10918            Err(mpsc::error::TryRecvError::Empty) => {}
10919            Ok(outbound) => panic!(
10920                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10921                outbound.frame.header.ty,
10922                outbound.frame.header.channel,
10923                outbound.frame.header.epoch
10924            ),
10925            Err(err) => panic!("module egress closed: {err}"),
10926        }
10927
10928        // The client's first request on the route is the next thing the module
10929        // reads, on the binding it just created.
10930        let router = Router::with_control_handler(Arc::new(handler.clone()));
10931        let request = Frame::build(
10932            FrameType::Request,
10933            Flags::new(true, Priority::Interactive, false),
10934            client_channel,
10935            client_epoch,
10936            201,
10937            b"first request".to_vec(),
10938        )
10939        .unwrap();
10940        router
10941            .route_for_connection(&client_ctx, request)
10942            .await
10943            .unwrap();
10944        let delivered = module_rx.try_recv().unwrap().frame;
10945        assert_eq!(delivered.header.ty, FrameType::Request);
10946        assert_eq!(
10947            (delivered.header.channel, delivered.header.epoch),
10948            (module_channel, module_epoch)
10949        );
10950        assert_eq!(delivered.body, b"first request");
10951    }
10952
10953    /// The client goes away while `route.open` is still queueing the bind, so
10954    /// the relay is dropped with no module reply. The teardown cannot send the
10955    /// GOODBYE itself (it could reach the module ahead of the bind), so
10956    /// `route.open` must send it once the bind is queued; otherwise the module
10957    /// keeps a binding the daemon will never commit.
10958    #[tokio::test]
10959    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10960        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10961        let client = ConnectionId::new(42);
10962        let (client_ctx, _client_rx) = route_ctx(client);
10963
10964        let (open_task, bind) = relay_route_open(
10965            &handler,
10966            client,
10967            &client_ctx.egress,
10968            &mut module_rx,
10969            300,
10970            "aft",
10971            "released-before-mark",
10972        )
10973        .await;
10974        let (module_channel, module_epoch) = route_bind_channel(&bind);
10975        hold.wait().await;
10976
10977        handler.cleanup_connection(client).unwrap();
10978        assert!(
10979            module_rx.try_recv().is_err(),
10980            "client teardown must not GOODBYE a bind route.open is still queueing"
10981        );
10982
10983        hold.wait().await;
10984        let response = open_task.await.unwrap();
10985        assert_eq!(response.len(), 1);
10986        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10987
10988        let goodbye = module_rx
10989            .try_recv()
10990            .expect("module is told to drop the binding the daemon abandoned")
10991            .frame;
10992        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10993        assert_eq!(
10994            (goodbye.header.channel, goodbye.header.epoch),
10995            (module_channel, module_epoch)
10996        );
10997        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10998    }
10999
11000    /// The module acks inside the same window, but the client's egress closed
11001    /// first, so the reader refuses to commit. The ack proves the module holds
11002    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
11003    /// send a second one.
11004    #[tokio::test]
11005    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11006        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11007        let client = ConnectionId::new(43);
11008        let (client_ctx, client_rx) = route_ctx(client);
11009
11010        let (open_task, bind) = relay_route_open(
11011            &handler,
11012            client,
11013            &client_ctx.egress,
11014            &mut module_rx,
11015            400,
11016            "aft",
11017            "gone-client-before-mark",
11018        )
11019        .await;
11020        let (module_channel, module_epoch) = route_bind_channel(&bind);
11021        hold.wait().await;
11022
11023        drop(client_rx);
11024        assert!(client_ctx.egress.is_closed());
11025        handler
11026            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11027            .await
11028            .unwrap();
11029
11030        hold.wait().await;
11031        let response = open_task.await.unwrap();
11032        assert_eq!(response.len(), 1);
11033        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11034
11035        let goodbye = module_rx
11036            .try_recv()
11037            .expect("module is told to drop the binding it acked")
11038            .frame;
11039        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11040        assert_eq!(
11041            (goodbye.header.channel, goodbye.header.epoch),
11042            (module_channel, module_epoch)
11043        );
11044        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11045        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11046    }
11047
11048    /// The fence at the module-loop boundary, stated as its own contract: which
11049    /// forwarding failures are allowed to end the module connection that is being
11050    /// served. A `ConnectionClosing` naming some client is about that client, and
11051    /// a module connection is shared; the same error naming the module's own
11052    /// connection is about this connection and must stay fatal, as must failures
11053    /// that are about the forwarding table itself.
11054    #[test]
11055    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11056        let handler = ControlHandler::default();
11057        let module_connection = ConnectionId::new(30);
11058        let client_connection = ConnectionId::new(31);
11059
11060        handler
11061            .refuse_to_end_module_connection_for_a_client(
11062                module_connection,
11063                77,
11064                ForwardingError::ConnectionClosing {
11065                    connection_id: client_connection,
11066                },
11067            )
11068            .expect("a closing client must never end the module connection");
11069
11070        assert!(matches!(
11071            handler.refuse_to_end_module_connection_for_a_client(
11072                module_connection,
11073                78,
11074                ForwardingError::ConnectionClosing {
11075                    connection_id: module_connection,
11076                },
11077            ),
11078            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11079                connection_id
11080            })) if connection_id == module_connection
11081        ));
11082        assert!(matches!(
11083            handler.refuse_to_end_module_connection_for_a_client(
11084                module_connection,
11085                79,
11086                ForwardingError::Poisoned,
11087            ),
11088            Err(RouterError::Forwarding(ForwardingError::Poisoned))
11089        ));
11090        assert!(matches!(
11091            handler.refuse_to_end_module_connection_for_a_client(
11092                module_connection,
11093                80,
11094                ForwardingError::StaleModuleEndpoint,
11095            ),
11096            Err(RouterError::Forwarding(
11097                ForwardingError::StaleModuleEndpoint
11098            ))
11099        ));
11100    }
11101
11102    /// The spawn-attestation guard is what stops a connected module from claiming
11103    /// another module's identity and being stamped `Reserved` for it. Every other
11104    /// test that supplies a consumer_identity supplies a CORRECT one, because a
11105    /// correct one is what the rest of the flow needs -- so the guard's rejection
11106    /// branch was never the subject of an assertion, only its acceptance branch.
11107    ///
11108    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
11109    /// whole subc-core library suite green; only the forwarding integration tests
11110    /// notice, and they notice for unrelated reasons. This test exists so the
11111    /// refusal itself is asserted where the guard lives: it fails if the identity
11112    /// check stops refusing, which is the direction that matters, since a guard
11113    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
11114    #[tokio::test]
11115    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11116        let registry = Arc::new(Registry::default());
11117        let forwarding = Arc::new(ForwardingTable::default());
11118        let supervisor = SupervisorHandle::new();
11119        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11120        let handler =
11121            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11122                .with_supervisor(supervisor);
11123
11124        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11125        hello_via_sink(
11126            &handler,
11127            &target_ctx,
11128            &mut target_rx,
11129            hello_frame("target", PROTOCOL_VERSION, 1),
11130        )
11131        .await;
11132
11133        // A real supervised module id presenting the wrong nonce. This is the
11134        // impersonation case: the attacker knows a privileged module_id, which is
11135        // public, and guesses at the nonce, which is not.
11136        let wrong_nonce = handler
11137            .handle_control_frame(
11138                &route_ctx(ConnectionId::new(91)).0,
11139                route_open_frame_with_admission_facts(
11140                    20,
11141                    "target",
11142                    unique_project_root("admission-facts"),
11143                    Some(subc_control::ConsumerIdentity {
11144                        module_id: "fed".to_string(),
11145                        launch_nonce: "not-the-real-nonce".to_string(),
11146                    }),
11147                    None,
11148                ),
11149            )
11150            .await
11151            .unwrap();
11152        assert_eq!(
11153            parse_error(&wrong_nonce[0])["code"],
11154            "bad_consumer_identity",
11155            "a mismatched launch nonce must be refused, not stamped Reserved"
11156        );
11157
11158        // A module id the supervisor never spawned at all, so no nonce exists to
11159        // compare against. An implementation that treats "no record" as "nothing
11160        // to check" fails open here while passing the case above.
11161        let never_spawned = handler
11162            .handle_control_frame(
11163                &route_ctx(ConnectionId::new(92)).0,
11164                route_open_frame_with_admission_facts(
11165                    21,
11166                    "target",
11167                    unique_project_root("admission-facts"),
11168                    Some(subc_control::ConsumerIdentity {
11169                        module_id: "never-spawned".to_string(),
11170                        launch_nonce: "any-nonce".to_string(),
11171                    }),
11172                    None,
11173                ),
11174            )
11175            .await
11176            .unwrap();
11177        assert_eq!(
11178            parse_error(&never_spawned[0])["code"],
11179            "bad_consumer_identity",
11180            "an unspawned module_id must be refused rather than accepted for lack of a record"
11181        );
11182    }
11183
11184    /// The refusal test above proves the guard says NO. Nothing proved it can say
11185    /// YES, and the difference is not academic: replacing the whole authorization
11186    /// with `false` -- admitting no consumer identity at all, revoking Reserved
11187    /// standing for every supervised module in the fleet -- leaves 110 of the 111
11188    /// library tests GREEN. The one that notices does so by HANGING, because it
11189    /// waits for a bind that can no longer happen.
11190    ///
11191    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
11192    /// or flaky test, invites a RETRY rather than an investigation, and the retry
11193    /// hangs too and gets blamed on the runner. So a total revocation of the
11194    /// daemon's trust grant would have shipped behind a symptom nobody attributes
11195    /// to code.
11196    ///
11197    /// The bias is structural rather than accidental. A REFUSAL looks like a
11198    /// failure someone writes a test for; a GRANT looks like the happy path. Every
11199    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
11200    /// suite for that reason, and this one is the purest case in the daemon.
11201    ///
11202    /// This test asserts the EFFECT rather than the absence of an error: the module
11203    /// receives a RouteBind and it carries `Reserved` naming the attested module.
11204    /// A guard that admitted nobody would produce no bind at all; one that admitted
11205    /// everybody would stamp the wrong principal, which the refusal test catches.
11206    #[tokio::test]
11207    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11208        let registry = Arc::new(Registry::default());
11209        let forwarding = Arc::new(ForwardingTable::default());
11210        let supervisor = SupervisorHandle::new();
11211        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11212        let handler =
11213            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11214                .with_supervisor(supervisor);
11215
11216        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11217        hello_via_sink(
11218            &handler,
11219            &target_ctx,
11220            &mut target_rx,
11221            hello_frame("target", PROTOCOL_VERSION, 1),
11222        )
11223        .await;
11224
11225        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11226        let route_handler = handler.clone();
11227        let route_task = tokio::spawn(async move {
11228            route_handler
11229                .handle_control_frame(
11230                    &client_ctx,
11231                    route_open_frame_with_admission_facts(
11232                        30,
11233                        "target",
11234                        unique_project_root("admission-facts"),
11235                        Some(subc_control::ConsumerIdentity {
11236                            module_id: "fed".to_string(),
11237                            launch_nonce: "fed-nonce".to_string(),
11238                        }),
11239                        None,
11240                    ),
11241                )
11242                .await
11243                .unwrap()
11244        });
11245
11246        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
11247        // the very mutation it exists to catch -- a guard that admits nobody -- no
11248        // bind is ever sent, so it HUNG rather than failing. That reproduces the
11249        // exact defect being fixed: a total revocation detected only as a stalled
11250        // suite, which reads as flakiness and invites a retry. An acceptance test
11251        // that waits for an effect must bound the wait, or a red becomes a hang.
11252        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11253            .await
11254            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11255            .expect("module control channel closed before route.bind");
11256        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11257        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11258            panic!("expected route.bind")
11259        };
11260        assert_eq!(
11261            principal,
11262            Some(Principal::Reserved {
11263                module_id: "fed".to_string()
11264            }),
11265            "a correctly attested consumer must be stamped Reserved for its own id"
11266        );
11267
11268        handler
11269            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11270            .await
11271            .unwrap();
11272        assert!(route_task.await.unwrap().is_empty());
11273        assert!(
11274            matches!(
11275                serde_json::from_slice::<ClientControlResponse>(
11276                    &client_rx.recv().await.unwrap().body
11277                )
11278                .unwrap(),
11279                ClientControlResponse::RouteOpen { .. }
11280            ),
11281            "the route must actually open, not merely avoid an error"
11282        );
11283    }
11284
11285    #[tokio::test(start_paused = true)]
11286    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11287        let registry = Arc::new(Registry::default());
11288        let forwarding = Arc::new(ForwardingTable::default());
11289        let supervisor = SupervisorHandle::new();
11290        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11291        let handler =
11292            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11293                .with_supervisor(supervisor);
11294
11295        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11296        hello_via_sink(
11297            &handler,
11298            &target_ctx,
11299            &mut target_rx,
11300            hello_frame("target", PROTOCOL_VERSION, 1),
11301        )
11302        .await;
11303
11304        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11305        let direct_handler = handler.clone();
11306        let direct_open = tokio::spawn(async move {
11307            direct_handler
11308                .handle_control_frame(
11309                    &direct_ctx,
11310                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
11311                )
11312                .await
11313                .unwrap()
11314        });
11315        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11316            .await
11317            .expect("no direct route.bind within 5s")
11318            .expect("target control channel closed before direct route.bind");
11319        handler
11320            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11321            .await
11322            .unwrap();
11323        assert!(direct_open.await.unwrap().is_empty());
11324        let _ = direct_rx.recv().await.unwrap();
11325
11326        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11327        let reserved_handler = handler.clone();
11328        let reserved_open = tokio::spawn(async move {
11329            reserved_handler
11330                .handle_control_frame(
11331                    &reserved_ctx,
11332                    route_open_frame_with_admission_facts(
11333                        3,
11334                        "target",
11335                        unique_project_root("admission-facts"),
11336                        Some(ConsumerIdentity {
11337                            module_id: "fed".to_string(),
11338                            launch_nonce: "fed-nonce".to_string(),
11339                        }),
11340                        None,
11341                    ),
11342                )
11343                .await
11344                .unwrap()
11345        });
11346        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11347            .await
11348            .expect("no reserved route.bind within 5s")
11349            .expect("target control channel closed before reserved route.bind");
11350        handler
11351            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11352            .await
11353            .unwrap();
11354        assert!(reserved_open.await.unwrap().is_empty());
11355        let _ = reserved_rx.recv().await.unwrap();
11356
11357        forwarding
11358            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11359            .unwrap();
11360        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11361        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11362            module_id: Some("target".to_string()),
11363        })
11364        .unwrap();
11365        let census_frame =
11366            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11367        let response = handler
11368            .handle_control_frame(&census_ctx, census_frame)
11369            .await
11370            .unwrap()
11371            .pop()
11372            .unwrap();
11373        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11374        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11375        assert!(matches!(
11376            decoded,
11377            ClientControlResponse::SupervisorRoutes { .. }
11378        ));
11379        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11380        assert_eq!(routes.len(), 2);
11381        assert!(routes.iter().all(|route| route["draining"] == true));
11382        // The census carries WHY: the reason the drain was begun with, in the
11383        // route.closing vocabulary, on every draining route this drain marked.
11384        assert!(
11385            routes.iter().all(|route| route["drain_reason"] == "reload"),
11386            "draining routes must name the drain's reason: {routes:?}"
11387        );
11388        assert!(routes.iter().any(|route| {
11389            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11390        }));
11391        assert!(routes.iter().any(|route| {
11392            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11393        }));
11394
11395        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11396            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11397        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11398            std::fs::write(
11399                &golden_path,
11400                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11401            )
11402            .unwrap();
11403        }
11404        let expected: Value =
11405            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11406        assert_eq!(actual, expected);
11407    }
11408
11409    async fn query_live_roots(
11410        handler: &ControlHandler,
11411        module_ctx: &RouteCtx,
11412    ) -> ModuleControlResponseToModule {
11413        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11414        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11415        let response = handler
11416            .handle_control_frame(module_ctx, frame)
11417            .await
11418            .unwrap()
11419            .pop()
11420            .unwrap();
11421        serde_json::from_slice(&response.body).unwrap()
11422    }
11423
11424    #[tokio::test(start_paused = true)]
11425    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11426        let registry = Arc::new(Registry::default());
11427        let forwarding = Arc::new(ForwardingTable::default());
11428        let handler = ControlHandler::with_forwarding(registry, forwarding);
11429        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11430        hello_via_sink(
11431            &handler,
11432            &target_ctx,
11433            &mut target_rx,
11434            hello_frame("target", PROTOCOL_VERSION, 1),
11435        )
11436        .await;
11437        let root = unique_project_root("live-roots-known");
11438        let path = ProjectRootId::from_path_allowing_missing(root.path())
11439            .unwrap()
11440            .as_path()
11441            .to_path_buf();
11442        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11443        let open_handler = handler.clone();
11444        let opened = tokio::spawn(async move {
11445            open_handler
11446                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11447                .await
11448                .unwrap()
11449        });
11450        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11451            .await
11452            .unwrap()
11453            .unwrap();
11454        handler
11455            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11456            .await
11457            .unwrap();
11458        assert!(opened.await.unwrap().is_empty());
11459        let _ = client_rx.recv().await.unwrap();
11460
11461        let root = unique_project_root("live-roots-pending");
11462        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11463            .unwrap()
11464            .as_path()
11465            .to_path_buf();
11466        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11467        let open_handler = handler.clone();
11468        let pending = tokio::spawn(async move {
11469            open_handler
11470                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11471                .await
11472                .unwrap()
11473        });
11474        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11475            .await
11476            .unwrap()
11477            .unwrap();
11478        let actual = query_live_roots(&handler, &target_ctx).await;
11479        let ModuleControlResponseToModule::LiveRoots {
11480            roots,
11481            unknown_root_bindings,
11482            total_bindings,
11483        } = actual
11484        else {
11485            panic!("expected live roots")
11486        };
11487        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11488        assert_eq!(unknown_root_bindings, 0);
11489        assert_eq!(
11490            roots.len(),
11491            2,
11492            "root-known arm must retain each canonical root"
11493        );
11494        assert_eq!(
11495            total_bindings,
11496            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11497        );
11498        let counts = roots
11499            .iter()
11500            .map(|root| (root.project_root.clone(), root.bound, root.pending))
11501            .collect::<Vec<_>>();
11502        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11503        expected.sort_by(|a, b| a.0.cmp(&b.0));
11504        assert_eq!(
11505            counts, expected,
11506            "roots must sort by path and count pending separately"
11507        );
11508        handler
11509            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11510            .await
11511            .unwrap();
11512        assert!(pending.await.unwrap().is_empty());
11513    }
11514
11515    #[tokio::test(start_paused = true)]
11516    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11517        let forwarding = Arc::new(ForwardingTable::default());
11518        let handler =
11519            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11520        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11521        hello_via_sink(
11522            &handler,
11523            &target_ctx,
11524            &mut target_rx,
11525            hello_frame("target", PROTOCOL_VERSION, 1),
11526        )
11527        .await;
11528        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11529        let pending = forwarding
11530            .begin_route_bind_relay_for_test(
11531                client_ctx.connection_id,
11532                client_ctx.egress.clone(),
11533                2,
11534                "target",
11535            )
11536            .unwrap();
11537        forwarding
11538            .complete_pending_relay(
11539                target_ctx.connection_id,
11540                pending.corr,
11541                RouteBindRelayOutcome::Accepted,
11542            )
11543            .unwrap();
11544        let actual = query_live_roots(&handler, &target_ctx).await;
11545        let ModuleControlResponseToModule::LiveRoots {
11546            roots,
11547            unknown_root_bindings,
11548            total_bindings,
11549        } = actual
11550        else {
11551            panic!("expected live roots")
11552        };
11553        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11554        assert_eq!(
11555            unknown_root_bindings, 1,
11556            "unknown-root arm must not read as no bindings"
11557        );
11558        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11559        assert_eq!(
11560            total_bindings,
11561            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11562        );
11563    }
11564
11565    /// A module reads HELLO_ACK as its first frame and exits on anything else,
11566    /// so the ack has to be on its outbound queue before the module is
11567    /// routable. The connection loop writes a handler's replies only after the
11568    /// handler returns; this test stops in exactly that gap, runs a real
11569    /// route.open from another connection, and only then writes whatever the
11570    /// HELLO handler returned, the way the loop would. If the ack were still a
11571    /// reply, the route.bind request would reach the module first.
11572    #[tokio::test(start_paused = true)]
11573    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11574        let forwarding = Arc::new(ForwardingTable::default());
11575        let handler =
11576            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11577        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11578        let replies = handler
11579            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11580            .await
11581            .unwrap();
11582        let queued_by_hello = module_rx.len();
11583
11584        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11585        let open_handler = handler.clone();
11586        let open = tokio::spawn(async move {
11587            open_handler
11588                .handle_control_frame(
11589                    &client_ctx,
11590                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11591                )
11592                .await
11593                .unwrap()
11594        });
11595        // Let the route.open run until its route.bind is on the module's queue.
11596        let mut spins = 0;
11597        while module_rx.len() == queued_by_hello {
11598            spins += 1;
11599            assert!(spins < 10_000, "route.open never queued a route.bind");
11600            tokio::task::yield_now().await;
11601        }
11602
11603        // Now the connection loop's half: write the HELLO handler's replies.
11604        for reply in replies {
11605            module_ctx.egress.send(reply).await.unwrap();
11606        }
11607
11608        let first = module_rx.recv().await.unwrap().frame;
11609        assert_eq!(
11610            first.header.ty,
11611            FrameType::HelloAck,
11612            "the first frame a registering module reads must be its HELLO_ACK"
11613        );
11614        assert_eq!(first.header.corr, 7);
11615        let second = module_rx.recv().await.unwrap().frame;
11616        assert_eq!(second.header.ty, FrameType::Request);
11617        assert!(
11618            matches!(
11619                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11620                ModuleControlRequest::RouteBind { .. }
11621            ),
11622            "the route.bind follows the ack"
11623        );
11624        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11625
11626        handler
11627            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11628            .await
11629            .unwrap();
11630        assert!(open.await.unwrap().is_empty());
11631        let _ = client_rx.recv().await.unwrap();
11632    }
11633
11634    #[tokio::test(start_paused = true)]
11635    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11636        let handler = ControlHandler::with_forwarding(
11637            Arc::new(Registry::default()),
11638            Arc::new(ForwardingTable::default()),
11639        );
11640        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11641        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11642        hello_via_sink(
11643            &handler,
11644            &first_ctx,
11645            &mut first_rx,
11646            hello_frame("first", PROTOCOL_VERSION, 1),
11647        )
11648        .await;
11649        hello_via_sink(
11650            &handler,
11651            &second_ctx,
11652            &mut second_rx,
11653            hello_frame("second", PROTOCOL_VERSION, 2),
11654        )
11655        .await;
11656        let root = unique_project_root("second-only");
11657        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11658        let cloned = handler.clone();
11659        let open = tokio::spawn(async move {
11660            cloned
11661                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11662                .await
11663                .unwrap()
11664        });
11665        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11666            .await
11667            .unwrap()
11668            .unwrap();
11669        let first = query_live_roots(&handler, &first_ctx).await;
11670        let second = query_live_roots(&handler, &second_ctx).await;
11671        assert!(
11672            matches!(
11673                first,
11674                ModuleControlResponseToModule::LiveRoots {
11675                    total_bindings: 0,
11676                    ..
11677                }
11678            ),
11679            "cross-module scope must not expose another module's roots"
11680        );
11681        assert!(
11682            matches!(
11683                second,
11684                ModuleControlResponseToModule::LiveRoots {
11685                    total_bindings: 1,
11686                    ..
11687                }
11688            ),
11689            "second module must see its pending route"
11690        );
11691        handler
11692            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11693            .await
11694            .unwrap();
11695        assert!(open.await.unwrap().is_empty());
11696    }
11697
11698    #[tokio::test(start_paused = true)]
11699    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11700        let handler = ControlHandler::with_forwarding(
11701            Arc::new(Registry::default()),
11702            Arc::new(ForwardingTable::default()),
11703        );
11704        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11705        hello_via_sink(
11706            &handler,
11707            &target_ctx,
11708            &mut target_rx,
11709            hello_frame("target", PROTOCOL_VERSION, 1),
11710        )
11711        .await;
11712        let actual = query_live_roots(&handler, &target_ctx).await;
11713        let ModuleControlResponseToModule::LiveRoots {
11714            roots,
11715            unknown_root_bindings,
11716            total_bindings,
11717        } = actual
11718        else {
11719            panic!("expected live roots")
11720        };
11721        assert!(roots.is_empty());
11722        assert_eq!(unknown_root_bindings, 0);
11723        assert_eq!(total_bindings, 0);
11724        assert_eq!(
11725            total_bindings,
11726            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11727        );
11728    }
11729
11730    /// Read the vendored fed corpus rather than hand-building a package.
11731    ///
11732    /// A hand-built object encodes what the test author believed the carrier
11733    /// emits. These vectors are what it actually emits, and one of them exists
11734    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11735    /// ignores additive unknown fields at the traversal emit terminus."
11736    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11737        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11738            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11739        let text = std::fs::read_to_string(&path)
11740            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11741        let vectors: Vec<(String, Value)> = text
11742            .lines()
11743            .filter(|line| !line.trim().is_empty())
11744            .map(|line| {
11745                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11746                let id = entry["corpus_id"]
11747                    .as_str()
11748                    .expect("every vector carries a corpus_id")
11749                    .to_string();
11750                (id, entry["package"].clone())
11751            })
11752            .collect();
11753        // Pin the count: a corpus that silently shrinks would take its coverage
11754        // with it, and a suite reading N-1 vectors reports the same clean pass
11755        // as one reading N.
11756        assert_eq!(
11757            vectors.len(),
11758            3,
11759            "vendored fed corpus changed size; re-sync from subc-federation"
11760        );
11761
11762        // Pin what makes the corpus DISCRIMINATING, not just present.
11763        //
11764        // The relay test below takes its expected value from the corpus, so the
11765        // corpus supplies the test's power to detect a lossy relay rather than
11766        // its correctness. A relay that dropped unrecognised fields would still
11767        // be caught -- but only by a package carrying fields it does not know.
11768        // Shrink every package to the handful of keys any implementation would
11769        // recognise and the test keeps passing over an input that can no longer
11770        // fail, which is the same clean green as a corpus that shrank away.
11771        //
11772        // So assert the precondition rather than duplicating the packages here:
11773        // at least one vector must carry a field beyond the small common set.
11774        // That is one claim to maintain instead of nine, and it fails loudly if
11775        // a re-sync ever flattens the corpus.
11776        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11777        let richest = vectors
11778            .iter()
11779            .filter_map(|(_, package)| package.as_object())
11780            .map(|object| {
11781                object
11782                    .keys()
11783                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11784                    .count()
11785            })
11786            .max()
11787            .unwrap_or(0);
11788        assert!(
11789            richest >= 2,
11790            "vendored corpus no longer carries a package with unmodelled fields, \
11791             so the relay test can no longer distinguish a verbatim relay from a lossy one"
11792        );
11793
11794        vectors
11795    }
11796
11797    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
11798    ///
11799    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
11800    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
11801    /// three-key object, so a relay that quietly dropped fields it did not
11802    /// recognise would satisfy it. These vectors carry nine keys including ones
11803    /// this crate has no type for, so a typed relay fails here and only here.
11804    #[tokio::test]
11805    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11806        for (corpus_id, package) in fed_admission_facts_vectors() {
11807            let registry = Arc::new(Registry::default());
11808            let forwarding = Arc::new(ForwardingTable::default());
11809            let supervisor = SupervisorHandle::new();
11810            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11811            let handler =
11812                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11813                    .with_supervisor(supervisor)
11814                    .with_admission_facts_config(
11815                        Some("fed".to_string()),
11816                        Some(vec!["target".to_string()]),
11817                    );
11818
11819            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11820            hello_via_sink(
11821                &handler,
11822                &target_ctx,
11823                &mut target_rx,
11824                hello_frame("target", PROTOCOL_VERSION, 1),
11825            )
11826            .await;
11827
11828            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11829            let route_handler = handler.clone();
11830            let expected = package.clone();
11831            let route_task = tokio::spawn(async move {
11832                route_handler
11833                    .handle_control_frame(
11834                        &client_ctx,
11835                        route_open_frame_with_admission_facts(
11836                            20,
11837                            "target",
11838                            unique_project_root("admission-facts"),
11839                            Some(subc_control::ConsumerIdentity {
11840                                module_id: "fed".to_string(),
11841                                launch_nonce: "fed-nonce".to_string(),
11842                            }),
11843                            Some(package),
11844                        ),
11845                    )
11846                    .await
11847                    .unwrap()
11848            });
11849
11850            let bind_frame = target_rx.recv().await.unwrap();
11851            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11852            let ModuleControlRequest::RouteBind {
11853                admission_facts, ..
11854            } = bind
11855            else {
11856                panic!("{corpus_id}: expected route.bind")
11857            };
11858            assert_eq!(
11859                admission_facts,
11860                Some(expected),
11861                "{corpus_id}: relay must not add, drop or reshape any field"
11862            );
11863
11864            handler
11865                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11866                .await
11867                .unwrap();
11868            route_task.await.unwrap();
11869        }
11870    }
11871
11872    #[tokio::test]
11873    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11874        let registry = Arc::new(Registry::default());
11875        let forwarding = Arc::new(ForwardingTable::default());
11876        let supervisor = SupervisorHandle::new();
11877        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11878        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11879        let handler =
11880            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11881                .with_supervisor(supervisor)
11882                .with_admission_facts_config(
11883                    Some("fed".to_string()),
11884                    Some(vec!["target".to_string()]),
11885                );
11886
11887        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11888        hello_via_sink(
11889            &handler,
11890            &target_ctx,
11891            &mut target_rx,
11892            hello_frame("target", PROTOCOL_VERSION, 1),
11893        )
11894        .await;
11895        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11896        hello_via_sink(
11897            &handler,
11898            &other_ctx,
11899            &mut other_rx,
11900            hello_frame("other", PROTOCOL_VERSION, 2),
11901        )
11902        .await;
11903
11904        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11905        let expected_facts = facts.clone();
11906        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11907        let route_handler = handler.clone();
11908        let route_task = tokio::spawn(async move {
11909            route_handler
11910                .handle_control_frame(
11911                    &client_ctx,
11912                    route_open_frame_with_admission_facts(
11913                        10,
11914                        "target",
11915                        unique_project_root("admission-facts"),
11916                        Some(subc_control::ConsumerIdentity {
11917                            module_id: "fed".to_string(),
11918                            launch_nonce: "fed-nonce".to_string(),
11919                        }),
11920                        Some(facts.clone()),
11921                    ),
11922                )
11923                .await
11924                .unwrap()
11925        });
11926        let bind_frame = target_rx.recv().await.unwrap();
11927        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11928        let ModuleControlRequest::RouteBind {
11929            admission_facts, ..
11930        } = bind
11931        else {
11932            panic!("expected route.bind")
11933        };
11934        assert_eq!(admission_facts, Some(expected_facts));
11935        handler
11936            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11937            .await
11938            .unwrap();
11939        assert!(route_task.await.unwrap().is_empty());
11940        assert!(matches!(
11941            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11942                .unwrap(),
11943            ClientControlResponse::RouteOpen { .. }
11944        ));
11945
11946        let direct = handler
11947            .handle_control_frame(
11948                &route_ctx(ConnectionId::new(73)).0,
11949                route_open_frame_with_admission_facts(
11950                    11,
11951                    "target",
11952                    unique_project_root("admission-facts"),
11953                    None,
11954                    Some(json!({"x": 1})),
11955                ),
11956            )
11957            .await
11958            .unwrap();
11959        assert_eq!(
11960            parse_error(&direct[0])["code"],
11961            "admission_facts_not_permitted"
11962        );
11963
11964        let different_reserved = handler
11965            .handle_control_frame(
11966                &route_ctx(ConnectionId::new(77)).0,
11967                route_open_frame_with_admission_facts(
11968                    15,
11969                    "target",
11970                    unique_project_root("admission-facts"),
11971                    Some(subc_control::ConsumerIdentity {
11972                        module_id: "other".to_string(),
11973                        launch_nonce: "other-nonce".to_string(),
11974                    }),
11975                    Some(json!({"x": 1})),
11976                ),
11977            )
11978            .await
11979            .unwrap();
11980        assert_eq!(
11981            parse_error(&different_reserved[0])["code"],
11982            "admission_facts_not_permitted"
11983        );
11984
11985        let other_target = handler
11986            .handle_control_frame(
11987                &route_ctx(ConnectionId::new(74)).0,
11988                route_open_frame_with_admission_facts(
11989                    12,
11990                    "other",
11991                    unique_project_root("admission-facts"),
11992                    Some(subc_control::ConsumerIdentity {
11993                        module_id: "fed".to_string(),
11994                        launch_nonce: "fed-nonce".to_string(),
11995                    }),
11996                    Some(json!({"x": 1})),
11997                ),
11998            )
11999            .await
12000            .unwrap();
12001        assert_eq!(
12002            parse_error(&other_target[0])["code"],
12003            "admission_facts_target_not_allowed"
12004        );
12005
12006        let nonexistent = handler
12007            .handle_control_frame(
12008                &route_ctx(ConnectionId::new(75)).0,
12009                route_open_frame_with_admission_facts(
12010                    13,
12011                    "missing",
12012                    unique_project_root("admission-facts"),
12013                    None,
12014                    Some(json!({"x": 1})),
12015                ),
12016            )
12017            .await
12018            .unwrap();
12019        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12020
12021        let described = handler
12022            .handle_control_frame(
12023                &route_ctx(ConnectionId::new(76)).0,
12024                Frame::build(
12025                    FrameType::Request,
12026                    control_flags(),
12027                    0,
12028                    0,
12029                    14,
12030                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12031                )
12032                .unwrap(),
12033            )
12034            .await
12035            .unwrap();
12036        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12037            serde_json::from_slice(&described[0].body).unwrap()
12038        else {
12039            panic!("expected server.describe response")
12040        };
12041        assert!(capabilities
12042            .iter()
12043            .any(|cap| cap == "admission_facts_relay_v1"));
12044    }
12045
12046    #[tokio::test]
12047    async fn admission_facts_without_configured_carrier_are_rejected() {
12048        let registry = Arc::new(Registry::default());
12049        let forwarding = Arc::new(ForwardingTable::default());
12050        let handler = ControlHandler::with_forwarding(registry, forwarding);
12051        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12052        hello_via_sink(
12053            &handler,
12054            &target_ctx,
12055            &mut target_rx,
12056            hello_frame("target", PROTOCOL_VERSION, 1),
12057        )
12058        .await;
12059
12060        let responses = handler
12061            .handle_control_frame(
12062                &route_ctx(ConnectionId::new(79)).0,
12063                route_open_frame_with_admission_facts(
12064                    16,
12065                    "target",
12066                    unique_project_root("admission-facts"),
12067                    None,
12068                    Some(json!({"x": 1})),
12069                ),
12070            )
12071            .await
12072            .unwrap();
12073        assert_eq!(
12074            parse_error(&responses[0])["code"],
12075            "admission_facts_not_permitted"
12076        );
12077    }
12078
12079    #[tokio::test]
12080    async fn route_open_relays_consumer_capabilities_verbatim() {
12081        let registry = Arc::new(Registry::default());
12082        let forwarding = Arc::new(ForwardingTable::default());
12083        let handler =
12084            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12085        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12086        hello_via_sink(
12087            &handler,
12088            &module_ctx,
12089            &mut module_rx,
12090            hello_frame("aft", PROTOCOL_VERSION, 7),
12091        )
12092        .await;
12093
12094        let expected = vec!["elicitation".to_string(), "roots".to_string()];
12095        let expected_for_request = expected.clone();
12096        let project_root = unique_project_root("consumer-capabilities-present");
12097        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12098        let route_handler = handler.clone();
12099        let route_task = tokio::spawn(async move {
12100            route_handler
12101                .handle_control_frame(
12102                    &client_ctx,
12103                    route_open_frame_with_consumer_capabilities(
12104                        401,
12105                        "aft",
12106                        project_root,
12107                        Some(expected_for_request),
12108                    ),
12109                )
12110                .await
12111                .unwrap()
12112        });
12113        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12114            .await
12115            .unwrap()
12116            .unwrap();
12117        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12118        let ModuleControlRequest::RouteBind {
12119            consumer_capabilities,
12120            ..
12121        } = bind
12122        else {
12123            panic!("expected route.bind request, got {bind:?}");
12124        };
12125        assert_eq!(consumer_capabilities, Some(expected.clone()));
12126
12127        handler
12128            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12129            .await
12130            .unwrap();
12131        let route_response = route_task.await.unwrap();
12132        assert!(route_response.is_empty());
12133        let published = client_rx.recv().await.unwrap();
12134        assert!(matches!(
12135            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12136            ClientControlResponse::RouteOpen { .. }
12137        ));
12138    }
12139
12140    #[tokio::test]
12141    async fn route_open_without_consumer_capabilities_relays_none() {
12142        let registry = Arc::new(Registry::default());
12143        let forwarding = Arc::new(ForwardingTable::default());
12144        let handler =
12145            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12146        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12147        hello_via_sink(
12148            &handler,
12149            &module_ctx,
12150            &mut module_rx,
12151            hello_frame("aft", PROTOCOL_VERSION, 7),
12152        )
12153        .await;
12154
12155        let project_root = unique_project_root("consumer-capabilities-absent");
12156        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12157        let route_handler = handler.clone();
12158        let route_task = tokio::spawn(async move {
12159            route_handler
12160                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12161                .await
12162                .unwrap()
12163        });
12164        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12165            .await
12166            .unwrap()
12167            .unwrap();
12168        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12169        let ModuleControlRequest::RouteBind {
12170            consumer_capabilities,
12171            ..
12172        } = bind
12173        else {
12174            panic!("expected route.bind request, got {bind:?}");
12175        };
12176        assert_eq!(consumer_capabilities, None);
12177
12178        handler
12179            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12180            .await
12181            .unwrap();
12182        let route_response = route_task.await.unwrap();
12183        assert!(route_response.is_empty());
12184        let published = client_rx.recv().await.unwrap();
12185        assert!(matches!(
12186            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12187            ClientControlResponse::RouteOpen { .. }
12188        ));
12189    }
12190
12191    /// Opens a route to a freshly registered `aft` with `sent` as the
12192    /// route.open's role_versions, acks the bind, and returns the role_versions
12193    /// the module's bind carried.
12194    async fn bind_role_versions_for(
12195        sent: Option<BTreeMap<String, String>>,
12196        connection: u64,
12197    ) -> Option<BTreeMap<String, String>> {
12198        let registry = Arc::new(Registry::default());
12199        let forwarding = Arc::new(ForwardingTable::default());
12200        let handler =
12201            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12202        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12203        hello_via_sink(
12204            &handler,
12205            &module_ctx,
12206            &mut module_rx,
12207            hello_frame("aft", PROTOCOL_VERSION, 7),
12208        )
12209        .await;
12210        let project_root = unique_project_root("role-versions");
12211        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12212        let route_handler = handler.clone();
12213        let route_task = tokio::spawn(async move {
12214            route_handler
12215                .handle_control_frame(
12216                    &client_ctx,
12217                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
12218                )
12219                .await
12220                .unwrap()
12221        });
12222        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12223            .await
12224            .expect("a well-formed route.open reaches the module as a bind")
12225            .unwrap();
12226        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12227        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12228            panic!("expected route.bind request, got {bind:?}");
12229        };
12230        handler
12231            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12232            .await
12233            .unwrap();
12234        assert!(route_task.await.unwrap().is_empty());
12235        let published = client_rx.recv().await.unwrap();
12236        assert!(matches!(
12237            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12238            ClientControlResponse::RouteOpen { .. }
12239        ));
12240        role_versions
12241    }
12242
12243    #[tokio::test]
12244    async fn route_open_relays_role_versions_verbatim() {
12245        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12246        assert_eq!(
12247            bind_role_versions_for(Some(sent.clone()), 141).await,
12248            Some(sent)
12249        );
12250    }
12251
12252    /// An empty map declares nothing, so the provider sees no field rather
12253    /// than an empty object it would have to treat as a second "none".
12254    #[tokio::test]
12255    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12256        assert_eq!(bind_role_versions_for(None, 143).await, None);
12257        assert_eq!(
12258            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12259            None
12260        );
12261    }
12262
12263    #[tokio::test]
12264    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12265        let registry = Arc::new(Registry::default());
12266        let forwarding = Arc::new(ForwardingTable::default());
12267        let handler =
12268            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12269        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12270        hello_via_sink(
12271            &handler,
12272            &module_ctx,
12273            &mut module_rx,
12274            hello_frame("aft", PROTOCOL_VERSION, 7),
12275        )
12276        .await;
12277
12278        let nine: BTreeMap<String, String> = (0..9)
12279            .map(|index| (format!("role-{index}"), "v1".to_string()))
12280            .collect();
12281        for (label, malformed) in [
12282            (
12283                "invalid role name",
12284                role_versions(&[("Tool_Provider", "v1")]),
12285            ),
12286            ("invalid version", role_versions(&[("tool-provider", "v0")])),
12287            ("nine entries", nine),
12288        ] {
12289            // A refused open answers at once; one that reached the module
12290            // would wait for its bind ack and trip this timeout.
12291            let responses = tokio::time::timeout(
12292                Duration::from_secs(1),
12293                handler.handle_control_frame(
12294                    &route_ctx(ConnectionId::new(148)).0,
12295                    route_open_frame_with_role_versions(
12296                        404,
12297                        "aft",
12298                        unique_project_root("role-versions-malformed"),
12299                        Some(malformed),
12300                    ),
12301                ),
12302            )
12303            .await
12304            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12305            .unwrap();
12306            assert_eq!(responses.len(), 1, "{label}");
12307            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12308            let error = parse_error(&responses[0]);
12309            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12310            assert_eq!(
12311                error["detail"]["field"], "role_versions",
12312                "{label}: {error}"
12313            );
12314            assert!(
12315                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12316                "{label}: a malformed declaration is terminal"
12317            );
12318            assert!(
12319                module_rx.try_recv().is_err(),
12320                "{label}: the module must never see a bind"
12321            );
12322        }
12323    }
12324
12325    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
12326    /// consumer can tell this daemon forwards the field from one that would
12327    /// drop it.
12328    #[tokio::test]
12329    async fn route_role_versions_capability_is_advertised() {
12330        let handler = ControlHandler::new(Arc::new(Registry::default()));
12331        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12332        let ack = hello_via_sink(
12333            &handler,
12334            &ctx,
12335            &mut rx,
12336            hello_frame("m", PROTOCOL_VERSION, 1),
12337        )
12338        .await;
12339        let ack = parse_ack(&ack);
12340        assert!(
12341            ack.subc_capabilities
12342                .iter()
12343                .any(|c| c == "route-role-versions/v1"),
12344            "{:?}",
12345            ack.subc_capabilities
12346        );
12347
12348        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12349        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12350        let reply = handler
12351            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12352            .await
12353            .unwrap()
12354            .pop()
12355            .unwrap();
12356        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12357            serde_json::from_slice(&reply.body).unwrap()
12358        else {
12359            panic!("not a server.describe reply");
12360        };
12361        assert!(
12362            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12363            "{capabilities:?}"
12364        );
12365    }
12366
12367    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12368    /// catalog entry without `events` (the module declares none) from a daemon
12369    /// that never reports declared events.
12370    #[tokio::test]
12371    async fn catalog_events_capability_is_advertised() {
12372        let handler = ControlHandler::new(Arc::new(Registry::default()));
12373        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12374        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12375        let reply = handler
12376            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12377            .await
12378            .unwrap()
12379            .pop()
12380            .unwrap();
12381        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12382            serde_json::from_slice(&reply.body).unwrap()
12383        else {
12384            panic!("not a server.describe reply");
12385        };
12386        assert!(
12387            capabilities
12388                .iter()
12389                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12390            "{capabilities:?}"
12391        );
12392    }
12393
12394    #[tokio::test]
12395    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12396        let handler = ControlHandler::new(Arc::new(Registry::default()));
12397        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12398        let ack = hello_via_sink(
12399            &handler,
12400            &module,
12401            &mut module_rx,
12402            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12403        )
12404        .await;
12405        let ack = parse_ack(&ack);
12406        assert!(
12407            ack.subc_capabilities
12408                .iter()
12409                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12410            "{:?}",
12411            ack.subc_capabilities
12412        );
12413        assert!(ack
12414            .subc_capabilities
12415            .iter()
12416            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12417
12418        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12419        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12420        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12421        let reply = handler
12422            .handle_control_frame(&client, frame)
12423            .await
12424            .unwrap()
12425            .pop()
12426            .unwrap();
12427        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12428            serde_json::from_slice(&reply.body).unwrap()
12429        else {
12430            panic!("not a server.describe reply");
12431        };
12432        assert!(
12433            capabilities
12434                .iter()
12435                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12436            "{capabilities:?}"
12437        );
12438        assert!(capabilities
12439            .iter()
12440            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12441    }
12442
12443    #[tokio::test]
12444    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12445        let handler = ControlHandler::new(Arc::new(Registry::default()));
12446        let (module, mut rx) = route_ctx(ConnectionId::new(156));
12447        let ack = hello_via_sink(
12448            &handler,
12449            &module,
12450            &mut rx,
12451            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12452        )
12453        .await;
12454        assert!(parse_ack(&ack)
12455            .subc_capabilities
12456            .iter()
12457            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12458        assert!(parse_ack(&ack)
12459            .subc_capabilities
12460            .iter()
12461            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12462        let (client, _rx) = route_ctx(ConnectionId::new(157));
12463        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12464        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12465        let reply = handler
12466            .handle_control_frame(&client, frame)
12467            .await
12468            .unwrap()
12469            .pop()
12470            .unwrap();
12471        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12472            serde_json::from_slice(&reply.body).unwrap()
12473        else {
12474            panic!("server.describe reply")
12475        };
12476        assert!(capabilities
12477            .iter()
12478            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12479        assert!(capabilities
12480            .iter()
12481            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12482    }
12483
12484    #[tokio::test]
12485    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12486        let handler = ControlHandler::new(Arc::new(Registry::default()));
12487        let (module, mut rx) = route_ctx(ConnectionId::new(158));
12488        let ack = hello_via_sink(
12489            &handler,
12490            &module,
12491            &mut rx,
12492            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12493        )
12494        .await;
12495        assert!(parse_ack(&ack)
12496            .subc_capabilities
12497            .iter()
12498            .any(|cap| cap == "scope-workdir/v1"));
12499        let (client, _rx) = route_ctx(ConnectionId::new(159));
12500        let frame = Frame::build(
12501            FrameType::Request,
12502            control_flags(),
12503            0,
12504            0,
12505            5,
12506            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12507        )
12508        .unwrap();
12509        let reply = handler
12510            .handle_control_frame(&client, frame)
12511            .await
12512            .unwrap()
12513            .remove(0);
12514        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12515            serde_json::from_slice(&reply.body).unwrap()
12516        else {
12517            panic!("server.describe reply")
12518        };
12519        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12520    }
12521
12522    #[tokio::test]
12523    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12524        let handler = ControlHandler::new(Arc::new(Registry::default()));
12525        let (module, mut rx) = route_ctx(ConnectionId::new(154));
12526        let ack = hello_via_sink(
12527            &handler,
12528            &module,
12529            &mut rx,
12530            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12531        )
12532        .await;
12533        assert!(parse_ack(&ack)
12534            .subc_capabilities
12535            .iter()
12536            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12537        let (client, _rx) = route_ctx(ConnectionId::new(155));
12538        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12539        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12540        let reply = handler
12541            .handle_control_frame(&client, frame)
12542            .await
12543            .unwrap()
12544            .pop()
12545            .unwrap();
12546        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12547            serde_json::from_slice(&reply.body).unwrap()
12548        else {
12549            panic!("not a server.describe reply")
12550        };
12551        assert!(capabilities
12552            .iter()
12553            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12554    }
12555
12556    #[tokio::test]
12557    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12558        let registry = Arc::new(Registry::default());
12559        let forwarding = Arc::new(ForwardingTable::default());
12560        let handler =
12561            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12562                .with_health_probe_timeout(Duration::from_secs(5));
12563        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12564        hello_via_sink(
12565            &handler,
12566            &module_ctx,
12567            &mut module_rx,
12568            non_routable_hello_frame_with_control_ops(
12569                "mcp",
12570                300,
12571                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12572            ),
12573        )
12574        .await;
12575        assert!(registry
12576            .get_module("mcp")
12577            .unwrap()
12578            .unwrap()
12579            .manifest
12580            .provides
12581            .is_empty());
12582
12583        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12584        let route_response = handler
12585            .handle_control_frame(
12586                &route_client_ctx,
12587                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12588            )
12589            .await
12590            .unwrap();
12591        assert_eq!(route_response[0].header.ty, FrameType::Error);
12592        assert_eq!(
12593            parse_error(&route_response[0])["code"],
12594            "target_unavailable"
12595        );
12596        assert!(parse_error(&route_response[0])["message"]
12597            .as_str()
12598            .unwrap()
12599            .contains("does not provide the requested target"));
12600        assert!(module_rx.try_recv().is_err());
12601
12602        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12603        let health_handler = handler.clone();
12604        let health_task = tokio::spawn(async move {
12605            health_handler
12606                .handle_control_frame(
12607                    &health_client_ctx,
12608                    supervisor_health_probe_frame(302, "mcp"),
12609                )
12610                .await
12611                .unwrap()
12612        });
12613        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12614            .await
12615            .unwrap()
12616            .unwrap();
12617        assert_eq!(
12618            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12619            ModuleControlRequest::HealthCheck {}
12620        );
12621        handler
12622            .handle_control_frame(
12623                &module_ctx,
12624                health_response(health_frame.header.corr, HealthStatus::Ok),
12625            )
12626            .await
12627            .unwrap();
12628        let health_response = health_task.await.unwrap();
12629        assert_eq!(health_response[0].header.ty, FrameType::Response);
12630        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12631            ClientControlResponse::SupervisorHealthProbe {
12632                module_id, status, ..
12633            } => {
12634                assert_eq!(module_id, "mcp");
12635                assert_eq!(status, HealthStatus::Ok);
12636            }
12637            other => panic!("unexpected health response: {other:?}"),
12638        }
12639
12640        // Exercise the forwarding cleanup path directly while leaving the registry
12641        // advertisement in place. If cleanup leaves a stale control sink behind,
12642        // the next probe will enqueue onto it and wait for the long probe timeout
12643        // instead of returning an immediate no-connection error.
12644        forwarding
12645            .cleanup_connection(module_ctx.connection_id)
12646            .unwrap();
12647        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12648        let cleanup_response = tokio::time::timeout(
12649            Duration::from_millis(200),
12650            handler.handle_control_frame(
12651                &cleanup_probe_ctx,
12652                supervisor_health_probe_frame(303, "mcp"),
12653            ),
12654        )
12655        .await
12656        .expect("probe should fail immediately when the control lane is gone")
12657        .unwrap();
12658        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12659        assert_eq!(
12660            parse_error(&cleanup_response[0])["code"],
12661            "target_unavailable"
12662        );
12663        assert!(parse_error(&cleanup_response[0])["message"]
12664            .as_str()
12665            .unwrap()
12666            .contains("no module connection"));
12667
12668        handler
12669            .cleanup_connection(module_ctx.connection_id)
12670            .unwrap();
12671    }
12672
12673    #[tokio::test]
12674    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12675        let registry = Arc::new(Registry::default());
12676        let supervisor_handle = SupervisorHandle::new();
12677        let supervisor =
12678            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12679                .with_handle(supervisor_handle.clone())
12680                .with_connection_file_path(
12681                    std::env::temp_dir()
12682                        .join(format!("subc-route-open-warming-{}", std::process::id())),
12683                );
12684        let module = supervisor
12685            .supervise_configured(
12686                ModuleSpec {
12687                    module_id: "warming".to_string(),
12688                    program: fake_aft_stub_path(),
12689                    args: Vec::new(),
12690                    env: Vec::new(),
12691                    reserved: false,
12692                    reserved_prefixes: Vec::new(),
12693                    protocol: ModuleProtocol::Subc,
12694                    overlap: Default::default(),
12695                },
12696                true,
12697            )
12698            .unwrap();
12699        assert_eq!(module.state().unwrap(), ModuleState::Running);
12700
12701        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12702        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12703        let response = handler
12704            .handle_control_frame(
12705                &ctx,
12706                route_open_frame(304, "warming", unique_project_root("warming")),
12707            )
12708            .await
12709            .unwrap();
12710        module.stop().await.unwrap();
12711
12712        assert_eq!(response[0].header.ty, FrameType::Error);
12713        let error = parse_error(&response[0]);
12714        assert_eq!(error["code"], "module_warming");
12715        assert!(error["message"]
12716            .as_str()
12717            .unwrap()
12718            .contains("state=running, enabled=true, live=false"));
12719    }
12720
12721    #[test]
12722    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12723        let handler = ControlHandler::new(Arc::new(Registry::default()));
12724        let capture = EventCapture::default();
12725        let _subscriber =
12726            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12727        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12728        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12729        let pending = (0..limit).collect::<Vec<_>>();
12730        let response = handler
12731            .route_open_capacity_refusal(
12732                &ctx,
12733                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12734                "busy",
12735                pending.len(),
12736                limit,
12737            )
12738            .unwrap();
12739        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12740        let event = capture
12741            .events()
12742            .into_iter()
12743            .find(|event| {
12744                event.target == "control"
12745                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12746            })
12747            .expect("connection admission refusal event");
12748        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12749        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12750        // Refused before `handle_route_open` parses the request, so the caller
12751        // fields are read from the frame itself.
12752        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
12753        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
12754    }
12755
12756    #[test]
12757    fn route_open_target_cap_logs_admission_reason_and_capacity() {
12758        let handler = ControlHandler::new(Arc::new(Registry::default()));
12759        let capture = EventCapture::default();
12760        let _subscriber =
12761            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12762        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12763        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12764        let guards = (0..limit)
12765            .map(|_| {
12766                handler
12767                    .route_bind_concurrency
12768                    .try_admit("busy", limit)
12769                    .unwrap()
12770            })
12771            .collect::<Vec<_>>();
12772        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12773            Err(in_flight) => in_flight,
12774            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12775        };
12776        let response = handler
12777            .route_open_target_capacity_refusal(
12778                &ctx,
12779                &RouteOpenCaller::unknown(),
12780                &route_open_frame(397, "busy", unique_project_root("target-cap")),
12781                "busy",
12782                in_flight,
12783            )
12784            .unwrap();
12785        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12786        let event = capture
12787            .events()
12788            .into_iter()
12789            .find(|event| {
12790                event.target == "control"
12791                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12792            })
12793            .expect("target admission refusal event");
12794        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12795        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12796        drop(guards);
12797    }
12798
12799    #[cfg(unix)]
12800    #[tokio::test(start_paused = true)]
12801    async fn supervisor_list_shows_a_failed_modules_next_retry() {
12802        let registry = Arc::new(Registry::default());
12803        let handle = SupervisorHandle::new();
12804        let supervisor =
12805            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
12806                .with_handle(handle.clone());
12807        let module = supervisor
12808            .spawn(crate::ModuleSpec {
12809                module_id: "retry-list".into(),
12810                program: PathBuf::from("/bin/sh"),
12811                args: vec!["-c".into(), "exit 17".into()],
12812                env: vec![],
12813                reserved: false,
12814                reserved_prefixes: vec![],
12815                protocol: ModuleProtocol::None,
12816                overlap: Default::default(),
12817            })
12818            .unwrap();
12819        let deadline = std::time::Instant::now() + Duration::from_secs(10);
12820        while module.state().unwrap() != crate::ModuleState::Failed {
12821            assert!(
12822                std::time::Instant::now() < deadline,
12823                "waiting for budget failure"
12824            );
12825            tokio::task::yield_now().await;
12826        }
12827        let handler = ControlHandler::new(registry).with_supervisor(handle);
12828        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12829        let replies = handler.handle_supervisor_list(frame).await.unwrap();
12830        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12831        assert_eq!(body["modules"][0]["state"], "failed");
12832        let retry = body["modules"][0]["next_retry_at_ms"]
12833            .as_u64()
12834            .expect("scheduled retry on the wire");
12835        let now = std::time::SystemTime::now()
12836            .duration_since(std::time::UNIX_EPOCH)
12837            .unwrap()
12838            .as_millis() as u64;
12839        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
12840        module.set_enabled(false).await.unwrap();
12841        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12842        let replies = handler.handle_supervisor_list(frame).await.unwrap();
12843        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12844        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
12845    }
12846
12847    #[tokio::test]
12848    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12849        let registry = Arc::new(Registry::default());
12850        let handle = SupervisorHandle::new();
12851        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12852            .with_handle(handle.clone());
12853        supervisor
12854            .supervise_configured(
12855                crate::ModuleSpec {
12856                    module_id: "drift".into(),
12857                    program: PathBuf::from("disabled-module"),
12858                    args: Vec::new(),
12859                    env: Vec::new(),
12860                    reserved: false,
12861                    reserved_prefixes: Vec::new(),
12862                    protocol: ModuleProtocol::Subc,
12863                    overlap: Default::default(),
12864                },
12865                false,
12866            )
12867            .unwrap();
12868        let handler = ControlHandler::new(registry).with_supervisor(handle);
12869        for configured in [None, Some(false), Some(true)] {
12870            handler.capability_evaluator.configure(
12871                configured.map(|enabled| ("drift".to_string(), enabled)),
12872                BTreeMap::new(),
12873            );
12874            let frame =
12875                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12876            let replies = handler
12877                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12878                .await
12879                .unwrap();
12880            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12881            assert!(body.get("configured_enabled").is_none());
12882            assert_eq!(body["op"], "supervisor.ack");
12883            let frame =
12884                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12885            let replies = handler.handle_supervisor_list(frame).await.unwrap();
12886            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12887            assert_eq!(
12888                body["modules"][0].get("configured_enabled"),
12889                configured.map(Value::Bool).as_ref()
12890            );
12891        }
12892    }
12893
12894    #[tokio::test(flavor = "current_thread")]
12895    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12896        let handler = ControlHandler::new(Arc::new(Registry::default()));
12897        let capture = EventCapture::default();
12898        let _subscriber =
12899            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12900        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12901        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12902
12903        let _ = handler
12904            .handle_client_control_request(
12905                &ctx,
12906                frame,
12907                ClientControlRequest::SupervisorSetEnabled {
12908                    module_id: "broca".to_string(),
12909                    enabled: false,
12910                },
12911                &mut None,
12912            )
12913            .await
12914            .unwrap();
12915
12916        let events = capture
12917            .events()
12918            .into_iter()
12919            .filter(|event| {
12920                event.target == "control"
12921                    && event.fields.get("message").map(String::as_str)
12922                        == Some("supervisor request received")
12923            })
12924            .collect::<Vec<_>>();
12925        assert_eq!(events.len(), 1, "one supervisor request log line");
12926        let event = &events[0];
12927        assert_eq!(event.level, tracing::Level::INFO);
12928        assert_eq!(
12929            event.fields.get("op"),
12930            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12931        );
12932        assert_eq!(
12933            event.fields.get("module_id"),
12934            Some(&"\"broca\"".to_string())
12935        );
12936        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12937        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12938        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12939    }
12940
12941    #[tokio::test(flavor = "current_thread")]
12942    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12943        let registry = Arc::new(Registry::default());
12944        let module_connection = ConnectionId::new(303);
12945        registry
12946            .register_with_control_ops(
12947                manifest("aft", PROTOCOL_VERSION),
12948                PROTOCOL_VERSION,
12949                module_connection,
12950                module_baseline_control_ops(),
12951            )
12952            .unwrap();
12953        let handler = ControlHandler::new(registry);
12954        let capture = EventCapture::default();
12955        let _subscriber =
12956            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12957        let (ctx, _rx) = route_ctx(module_connection);
12958        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12959
12960        let _ = handler
12961            .handle_client_control_request(
12962                &ctx,
12963                frame,
12964                ClientControlRequest::SupervisorSetEnabled {
12965                    module_id: "broca".to_string(),
12966                    enabled: false,
12967                },
12968                &mut None,
12969            )
12970            .await
12971            .unwrap();
12972
12973        let callers = capture
12974            .events()
12975            .into_iter()
12976            .filter(|event| {
12977                event.fields.get("message").map(String::as_str)
12978                    == Some("supervisor request received")
12979            })
12980            .map(|event| event.fields.get("caller").cloned())
12981            .collect::<Vec<_>>();
12982        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12983    }
12984
12985    #[tokio::test(flavor = "current_thread")]
12986    async fn supervisor_list_does_not_log_request_received() {
12987        let handler = ControlHandler::new(Arc::new(Registry::default()));
12988        let capture = EventCapture::default();
12989        let _subscriber =
12990            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12991        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12992        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12993
12994        handler
12995            .handle_client_control_request(
12996                &ctx,
12997                frame,
12998                ClientControlRequest::SupervisorList {},
12999                &mut None,
13000            )
13001            .await
13002            .unwrap();
13003
13004        assert!(capture.events().into_iter().all(|event| {
13005            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13006        }));
13007    }
13008
13009    #[tokio::test(flavor = "current_thread")]
13010    async fn supervisor_rescan_preview_does_not_log_request_received() {
13011        let handler = ControlHandler::new(Arc::new(Registry::default()));
13012        let capture = EventCapture::default();
13013        let _subscriber =
13014            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13015        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13016        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13017
13018        handler
13019            .handle_client_control_request(
13020                &ctx,
13021                frame,
13022                ClientControlRequest::SupervisorRescan { preview: true },
13023                &mut None,
13024            )
13025            .await
13026            .unwrap();
13027
13028        assert!(capture.events().into_iter().all(|event| {
13029            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13030        }));
13031    }
13032
13033    struct ApproveOperator;
13034    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13035        fn prompt(
13036            &self,
13037            _: &str,
13038            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13039        ) -> crate::operator_confirm::ProviderResult {
13040            crate::operator_confirm::ProviderResult::Approved
13041        }
13042    }
13043
13044    #[tokio::test]
13045    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13046        assert_eq!(
13047            ControlHandler::default().privileged_operator_prompt,
13048            PrivilegedOperatorPrompt::Log
13049        );
13050        for (initial, changed, running) in [
13051            ("log", "enforce", PrivilegedOperatorPrompt::Log),
13052            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13053        ] {
13054            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13055            let path = dir.join("subc.jsonc");
13056            std::fs::write(
13057                &path,
13058                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13059            )
13060            .unwrap();
13061            let config = crate::daemon_config::load(&path).unwrap().unwrap();
13062            let registry = Arc::new(Registry::default());
13063            let handle = SupervisorHandle::new();
13064            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13065                .with_handle(handle.clone());
13066            let handler = ControlHandler::new(registry)
13067                .with_supervisor(handle)
13068                .with_privileged_operator_prompt(config.privileged_operator_prompt)
13069                .with_supervisor_rescan(supervisor, &path, config.port);
13070            handler.forwarding.operator_confirms().configure(
13071                Arc::new(ApproveOperator),
13072                crate::operator_confirm::OperatorLimits::default(),
13073            );
13074            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13075
13076            for (disk_mode, needs_restart) in [
13077                (Some(initial), false),
13078                (Some(changed), true),
13079                (Some(changed), true),
13080                (None, running == PrivilegedOperatorPrompt::Enforce),
13081                (Some(initial), false),
13082            ] {
13083                let mut doc = json!({"version":1});
13084                if let Some(mode) = disk_mode {
13085                    doc["privileged_operator_prompt"] = json!(mode);
13086                }
13087                std::fs::write(&path, doc.to_string()).unwrap();
13088                for preview in [true, false] {
13089                    let request = ClientControlRequest::SupervisorRescan { preview };
13090                    let frame = Frame::build(
13091                        FrameType::Request,
13092                        control_flags(),
13093                        0,
13094                        0,
13095                        1,
13096                        serde_json::to_vec(&request).unwrap(),
13097                    )
13098                    .unwrap();
13099                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13100                    assert_eq!(frames.len(), 1);
13101                    let ClientControlResponse::SupervisorRescan { result } =
13102                        serde_json::from_slice(&frames[0].body).unwrap()
13103                    else {
13104                        panic!("expected a rescan response");
13105                    };
13106                    let expected = if needs_restart {
13107                        vec!["privileged_operator_prompt"]
13108                    } else {
13109                        Vec::new()
13110                    };
13111                    assert_eq!(result.restart_required, expected);
13112                    assert_eq!(handler.privileged_operator_prompt, running);
13113                    assert_eq!(
13114                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13115                        running
13116                    );
13117                }
13118            }
13119        }
13120    }
13121
13122    #[tokio::test]
13123    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13124        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13125        let path = dir.join("subc.jsonc");
13126        let registry = Arc::new(Registry::default());
13127        let handle = SupervisorHandle::new();
13128        let supervisor =
13129            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13130        let handler = ControlHandler::new(registry)
13131            .with_supervisor(handle)
13132            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13133            .with_supervisor_rescan(supervisor, &path, None);
13134        handler.forwarding.operator_confirms().configure(
13135            Arc::new(ApproveOperator),
13136            crate::operator_confirm::OperatorLimits::default(),
13137        );
13138        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13139        for preview in [true, false] {
13140            std::fs::write(
13141                &path,
13142                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13143            )
13144            .unwrap();
13145            let request = ClientControlRequest::SupervisorRescan { preview };
13146            let frame = Frame::build(
13147                FrameType::Request,
13148                control_flags(),
13149                0,
13150                0,
13151                1,
13152                serde_json::to_vec(&request).unwrap(),
13153            )
13154            .unwrap();
13155            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13156            assert_eq!(frames.len(), 1);
13157            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13158            assert_eq!(
13159                handler.privileged_operator_prompt,
13160                PrivilegedOperatorPrompt::Enforce
13161            );
13162        }
13163    }
13164
13165    /// One wire code has several senders, so the refusal line names the check
13166    /// that refused. This drives the shared refusal path for ordinary refusals
13167    /// with an unregistered
13168    /// target and requires the branch label on the event.
13169    #[tokio::test]
13170    async fn route_open_refusal_names_the_check_that_refused() {
13171        let handler = ControlHandler::new(Arc::new(Registry::default()));
13172        let capture = EventCapture::default();
13173        let _subscriber =
13174            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13175        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13176        let response = handler
13177            .handle_control_frame(
13178                &ctx,
13179                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13180            )
13181            .await
13182            .unwrap();
13183
13184        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13185        let event = capture
13186            .events()
13187            .into_iter()
13188            .find(|event| {
13189                event.target == "control"
13190                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13191            })
13192            .expect("route.open refusal event");
13193        assert_eq!(
13194            event.fields.get("reason"),
13195            Some(&"\"not_registered\"".to_string())
13196        );
13197    }
13198
13199    #[tokio::test]
13200    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13201        let registry = Arc::new(Registry::default());
13202        let supervisor_handle = SupervisorHandle::new();
13203        let supervisor =
13204            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13205                .with_handle(supervisor_handle.clone())
13206                .with_connection_file_path(std::env::temp_dir().join(format!(
13207                    "subc-route-open-refusal-info-{}",
13208                    std::process::id()
13209                )));
13210        let module = supervisor
13211            .supervise_configured(
13212                ModuleSpec {
13213                    module_id: "warming".to_string(),
13214                    program: fake_aft_stub_path(),
13215                    args: Vec::new(),
13216                    env: Vec::new(),
13217                    reserved: false,
13218                    reserved_prefixes: Vec::new(),
13219                    protocol: ModuleProtocol::Subc,
13220                    overlap: Default::default(),
13221                },
13222                true,
13223            )
13224            .unwrap();
13225        assert_eq!(module.state().unwrap(), ModuleState::Running);
13226
13227        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13228        assert!(handler
13229            .counters()
13230            .snapshot()
13231            .get("route_open_refused_by_code")
13232            .is_none());
13233        let capture = EventCapture::default();
13234        let _subscriber =
13235            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13236        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13237        let response = handler
13238            .handle_control_frame(
13239                &ctx,
13240                route_open_frame(394, "warming", unique_project_root("refusal-info")),
13241            )
13242            .await
13243            .unwrap();
13244        module.stop().await.unwrap();
13245
13246        assert_eq!(parse_error(&response[0])["code"], "module_warming");
13247        let event = capture
13248            .events()
13249            .into_iter()
13250            .find(|event| {
13251                event.target == "control"
13252                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13253            })
13254            .expect("route.open refusal event");
13255        assert_eq!(
13256            event.fields.get("module_id"),
13257            Some(&"\"warming\"".to_string())
13258        );
13259        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13260        assert_eq!(
13261            event.fields.get("reason"),
13262            Some(&"\"supervised_not_registered\"".to_string())
13263        );
13264        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13265        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13266        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13267        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13268        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13269        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13270        assert_eq!(
13271            handler.counters().snapshot()["route_open_refused_by_code"],
13272            json!({ "module_warming": 1 })
13273        );
13274    }
13275
13276    const OUTAGE_START: &str = "route.open refusing module: not serving";
13277    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13278
13279    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13280        capture
13281            .events()
13282            .into_iter()
13283            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13284            .collect()
13285    }
13286
13287    fn supervise_stub(
13288        registry: &Arc<Registry>,
13289        module_id: &str,
13290        enabled: bool,
13291    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13292        let supervisor_handle = SupervisorHandle::new();
13293        let supervisor =
13294            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13295                .with_handle(supervisor_handle.clone())
13296                .with_connection_file_path(std::env::temp_dir().join(format!(
13297                    "subc-route-outage-{module_id}-{}",
13298                    std::process::id()
13299                )));
13300        let module = supervisor
13301            .supervise_configured(
13302                ModuleSpec {
13303                    module_id: module_id.to_string(),
13304                    program: fake_aft_stub_path(),
13305                    args: Vec::new(),
13306                    env: Vec::new(),
13307                    reserved: false,
13308                    reserved_prefixes: Vec::new(),
13309                    protocol: ModuleProtocol::Subc,
13310                    overlap: Default::default(),
13311                },
13312                enabled,
13313            )
13314            .unwrap();
13315        (supervisor_handle, module)
13316    }
13317
13318    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13319        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13320            module_id: module_id.to_string(),
13321            drain_timeout_ms: Some(50),
13322        })
13323        .unwrap();
13324        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13325    }
13326
13327    /// Two handlers built over one forwarding table must share one outage
13328    /// tracker; separate trackers would each log their own opening line for
13329    /// the same outage.
13330    #[test]
13331    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13332        let registry = Arc::new(Registry::default());
13333        let forwarding = Arc::new(ForwardingTable::default());
13334        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13335        let second = ControlHandler::with_forwarding(registry, forwarding);
13336        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13337    }
13338
13339    /// A client can name any module id it likes. Refusing an unknown one,
13340    /// however often, must not create outage state or outage lines, or the
13341    /// tracker would be a memory sink any client could fill.
13342    #[tokio::test(flavor = "current_thread")]
13343    async fn route_open_unknown_module_refusals_add_no_outage_state() {
13344        let handler = ControlHandler::new(Arc::new(Registry::default()));
13345        let capture = EventCapture::default();
13346        let _subscriber =
13347            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13348        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13349        for corr in 0..8 {
13350            let response = handler
13351                .handle_control_frame(
13352                    &ctx,
13353                    route_open_frame(
13354                        380 + corr,
13355                        &format!("nobody-{corr}"),
13356                        unique_project_root("outage-unknown"),
13357                    ),
13358                )
13359                .await
13360                .unwrap();
13361            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13362        }
13363
13364        assert_eq!(handler.route_outages.tracked_module_count(), 0);
13365        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13366        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13367    }
13368
13369    /// Drives the refusal path end to end: a supervised module that served
13370    /// before and stopped being registered with no instruction to stop is a
13371    /// WARN, and the same module refused after an operator `supervisor.restart`
13372    /// is an INFO.
13373    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13374    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13375        let registry = Arc::new(Registry::default());
13376        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
13377        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13378        let capture = EventCapture::default();
13379        let _subscriber =
13380            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13381        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13382        // The stub never registers, so pretend it served once: otherwise every
13383        // refusal would fall in its startup window.
13384        handler.route_outages.record_accepted("outage-restart");
13385
13386        let response = handler
13387            .handle_control_frame(
13388                &ctx,
13389                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13390            )
13391            .await
13392            .unwrap();
13393        assert_eq!(response[0].header.ty, FrameType::Error);
13394        let starts = outage_lines(&capture, OUTAGE_START);
13395        assert_eq!(starts.len(), 1, "{starts:?}");
13396        assert_eq!(starts[0].level, tracing::Level::WARN);
13397        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13398        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13399        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13400        handler.route_outages.record_accepted("outage-restart");
13401        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13402
13403        let restart = handler
13404            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13405            .await
13406            .unwrap();
13407        assert_eq!(
13408            restart[0].header.ty,
13409            FrameType::Response,
13410            "{:?}",
13411            parse_error(&restart[0])
13412        );
13413        handler
13414            .handle_control_frame(
13415                &ctx,
13416                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13417            )
13418            .await
13419            .unwrap();
13420        module.stop().await.unwrap();
13421
13422        let starts = outage_lines(&capture, OUTAGE_START);
13423        assert_eq!(starts.len(), 2, "{starts:?}");
13424        assert_eq!(starts[1].level, tracing::Level::INFO);
13425        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13426    }
13427
13428    /// A restart refused before it touched the module (here: the module is
13429    /// disabled) must clear its operator mark, so the next real outage is
13430    /// still reported as a warning.
13431    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13432    async fn failed_operator_restart_leaves_no_operator_mark() {
13433        let registry = Arc::new(Registry::default());
13434        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
13435        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13436        let capture = EventCapture::default();
13437        let _subscriber =
13438            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13439        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13440        handler.route_outages.record_accepted("outage-disabled");
13441
13442        let restart = handler
13443            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13444            .await
13445            .unwrap();
13446        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13447        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13448
13449        handler
13450            .handle_control_frame(
13451                &ctx,
13452                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13453            )
13454            .await
13455            .unwrap();
13456        let starts = outage_lines(&capture, OUTAGE_START);
13457        assert_eq!(starts.len(), 1, "{starts:?}");
13458        assert_eq!(starts[0].level, tracing::Level::WARN);
13459    }
13460
13461    #[tokio::test(flavor = "current_thread")]
13462    async fn route_open_unknown_module_escapes_target_module_id() {
13463        let handler = ControlHandler::new(Arc::new(Registry::default()));
13464        let capture = EventCapture::default();
13465        let _subscriber =
13466            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13467        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13468        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13469        let response = handler
13470            .handle_control_frame(
13471                &ctx,
13472                route_open_frame(
13473                    395,
13474                    hostile_module_id,
13475                    unique_project_root("hostile-target-module-id"),
13476                ),
13477            )
13478            .await
13479            .unwrap();
13480
13481        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13482        let event = capture
13483            .events()
13484            .into_iter()
13485            .find(|event| {
13486                event.target == "control"
13487                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13488            })
13489            .expect("route.open unknown-module refusal event");
13490        let logged = event.fields.get("module_id").expect("module_id field");
13491        assert!(!logged.bytes().any(|byte| byte < 0x20));
13492        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13493    }
13494
13495    #[tokio::test(flavor = "current_thread")]
13496    async fn route_open_module_rejection_uses_daemon_counter_key() {
13497        let registry = Arc::new(Registry::default());
13498        let forwarding = Arc::new(ForwardingTable::default());
13499        let handler =
13500            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13501        let module_connection = ConnectionId::new(95);
13502        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13503        hello_via_sink(
13504            &handler,
13505            &module_ctx,
13506            &mut module_rx,
13507            hello_frame("aft", PROTOCOL_VERSION, 395),
13508        )
13509        .await;
13510
13511        let client_connection = ConnectionId::new(96);
13512        let (client_ctx, _client_rx) = route_ctx(client_connection);
13513        let capture = EventCapture::default();
13514        let _subscriber =
13515            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13516        let (route_task, bind) = relay_route_open(
13517            &handler,
13518            client_connection,
13519            &client_ctx.egress,
13520            &mut module_rx,
13521            396,
13522            "aft",
13523            "hostile-module-code",
13524        )
13525        .await;
13526        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13527        let rejection = Frame::build(
13528            FrameType::Error,
13529            control_flags(),
13530            0,
13531            0,
13532            bind.header.corr,
13533            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13534        )
13535        .unwrap();
13536        handler
13537            .handle_control_frame(&module_ctx, rejection)
13538            .await
13539            .unwrap();
13540
13541        let response = route_task.await.unwrap();
13542        assert_eq!(parse_error(&response[0])["code"], hostile_code);
13543        let counters = handler.counters().snapshot();
13544        assert_eq!(
13545            counters["route_open_refused_by_code"],
13546            json!({ "module_rejected": 1 })
13547        );
13548        assert!(counters["route_open_refused_by_code"]
13549            .get(hostile_code)
13550            .is_none());
13551
13552        let event = capture
13553            .events()
13554            .into_iter()
13555            .find(|event| {
13556                event.target == "control"
13557                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13558            })
13559            .expect("route.open module-rejection refusal event");
13560        let logged = event.fields.get("module_code").expect("module_code field");
13561        assert!(!logged.bytes().any(|byte| byte < 0x20));
13562        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13563    }
13564
13565    #[tokio::test]
13566    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13567        let registry = Arc::new(Registry::default());
13568        let supervisor_handle = SupervisorHandle::new();
13569        let missing_program = std::env::temp_dir().join(format!(
13570            "subc-route-open-missing-program-{}",
13571            std::process::id()
13572        ));
13573        let supervisor =
13574            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13575                .with_handle(supervisor_handle.clone());
13576        let module = supervisor
13577            .supervise_configured(
13578                ModuleSpec {
13579                    module_id: "failed".to_string(),
13580                    program: missing_program,
13581                    args: Vec::new(),
13582                    env: Vec::new(),
13583                    reserved: false,
13584                    reserved_prefixes: Vec::new(),
13585                    protocol: ModuleProtocol::Subc,
13586                    overlap: Default::default(),
13587                },
13588                true,
13589            )
13590            .unwrap();
13591        assert_eq!(module.state().unwrap(), ModuleState::Failed);
13592
13593        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13594        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13595        let response = handler
13596            .handle_control_frame(
13597                &ctx,
13598                route_open_frame(305, "failed", unique_project_root("failed")),
13599            )
13600            .await
13601            .unwrap();
13602
13603        assert_eq!(response[0].header.ty, FrameType::Error);
13604        let error = parse_error(&response[0]);
13605        assert_eq!(error["code"], "target_unavailable");
13606        assert!(error["message"]
13607            .as_str()
13608            .unwrap()
13609            .contains("state=failed, enabled=true, live=false"));
13610    }
13611
13612    #[tokio::test]
13613    async fn route_open_role_mismatch_remains_target_unavailable() {
13614        let registry = Arc::new(Registry::default());
13615        let handler = ControlHandler::new(Arc::clone(&registry));
13616        handler
13617            .handle_control(
13618                ConnectionId::new(41),
13619                non_routable_hello_frame_with_control_ops("health-only", 306, None),
13620            )
13621            .unwrap();
13622
13623        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13624        let response = handler
13625            .handle_control_frame(
13626                &ctx,
13627                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13628            )
13629            .await
13630            .unwrap();
13631
13632        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13633        assert!(parse_error(&response[0])["message"]
13634            .as_str()
13635            .unwrap()
13636            .contains("does not provide the requested target"));
13637    }
13638
13639    #[tokio::test]
13640    async fn route_open_inactive_registration_remains_target_unavailable() {
13641        let registry = Arc::new(Registry::default());
13642        let handler = ControlHandler::new(Arc::clone(&registry));
13643        handler
13644            .handle_control(
13645                ConnectionId::new(43),
13646                hello_frame("inactive", PROTOCOL_VERSION, 308),
13647            )
13648            .unwrap();
13649        assert!(registry
13650            .set_module_state_for_test("inactive", ChannelState::Closed)
13651            .unwrap());
13652
13653        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13654        let response = handler
13655            .handle_control_frame(
13656                &ctx,
13657                route_open_frame(309, "inactive", unique_project_root("inactive")),
13658            )
13659            .await
13660            .unwrap();
13661
13662        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13663        assert!(parse_error(&response[0])["message"]
13664            .as_str()
13665            .unwrap()
13666            .contains("is not active"));
13667    }
13668
13669    #[tokio::test]
13670    async fn late_health_reply_is_recorded_through_the_module_response_path() {
13671        let registry = Arc::new(Registry::default());
13672        let forwarding = Arc::new(ForwardingTable::default());
13673        let supervisor_handle = SupervisorHandle::new();
13674        let supervisor =
13675            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
13676                .with_forwarding(Arc::clone(&forwarding))
13677                .with_handle(supervisor_handle.clone());
13678        let module = supervisor
13679            .supervise_configured(
13680                crate::ModuleSpec {
13681                    module_id: "late-health-response".to_string(),
13682                    program: PathBuf::from("disabled-module"),
13683                    args: Vec::new(),
13684                    env: Vec::new(),
13685                    reserved: false,
13686                    reserved_prefixes: Vec::new(),
13687                    protocol: ModuleProtocol::Subc,
13688                    overlap: Default::default(),
13689                },
13690                false,
13691            )
13692            .unwrap();
13693        let handler =
13694            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13695                .with_supervisor(supervisor_handle);
13696        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13697        handler
13698            .handle_control_frame(
13699                &module_ctx,
13700                hello_frame_with_control_ops(
13701                    "late-health-response",
13702                    PROTOCOL_VERSION,
13703                    7,
13704                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13705                ),
13706            )
13707            .await
13708            .unwrap();
13709        let probe_started_at = Instant::now() - Duration::from_millis(80);
13710        let pending = forwarding
13711            .begin_health_probe_rpc_for(
13712                "late-health-response",
13713                MODULE_CONTROL_OP_HEALTH_CHECK,
13714                probe_started_at,
13715                Instant::now() - Duration::from_millis(1),
13716            )
13717            .unwrap();
13718        assert!(forwarding
13719            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13720            .unwrap());
13721
13722        let responses = handler
13723            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13724            .await
13725            .unwrap();
13726
13727        assert!(responses.is_empty());
13728        let health = module.status().unwrap().health;
13729        assert_eq!(health.late_answer_count, 1);
13730        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13731    }
13732
13733    #[tokio::test]
13734    async fn health_probe_timeout_and_module_death_are_typed() {
13735        let registry = Arc::new(Registry::default());
13736        let forwarding = Arc::new(ForwardingTable::default());
13737        let handler =
13738            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13739                .with_health_probe_timeout(Duration::from_millis(50));
13740        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13741        hello_via_sink(
13742            &handler,
13743            &module_ctx,
13744            &mut module_rx,
13745            hello_frame_with_control_ops(
13746                "aft",
13747                PROTOCOL_VERSION,
13748                7,
13749                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13750            ),
13751        )
13752        .await;
13753
13754        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13755        let responses = handler
13756            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13757            .await
13758            .unwrap();
13759        assert_eq!(responses[0].header.ty, FrameType::Error);
13760        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13761        let _ = module_rx.try_recv();
13762
13763        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13764        let health_handler = handler.clone();
13765        let death_task = tokio::spawn(async move {
13766            health_handler
13767                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13768                .await
13769                .unwrap()
13770        });
13771        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13772            .await
13773            .unwrap()
13774            .unwrap();
13775        handler
13776            .cleanup_connection(module_ctx.connection_id)
13777            .unwrap();
13778        let responses = death_task.await.unwrap();
13779        assert_eq!(responses[0].header.ty, FrameType::Error);
13780        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13781    }
13782
13783    #[test]
13784    fn hello_requires_exact_protocol_version() {
13785        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13786            let registry = Arc::new(Registry::default());
13787            let handler = ControlHandler::new(Arc::clone(&registry));
13788            let responses = handler
13789                .handle_control(
13790                    ConnectionId::new(connection),
13791                    hello_frame("aft", offered, 9),
13792                )
13793                .unwrap();
13794
13795            assert_eq!(responses.len(), 1);
13796            assert_eq!(responses[0].header.ty, FrameType::Error);
13797            let error = parse_error(&responses[0]);
13798            assert_eq!(error["code"], "version_unsupported");
13799            assert!(registry.get_module("aft").unwrap().is_none());
13800            assert_eq!(registry.active_registration_count().unwrap(), 0);
13801        }
13802    }
13803
13804    #[test]
13805    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13806        let registry = Arc::new(Registry::default());
13807        let forwarding = Arc::new(ForwardingTable::default());
13808        let handler =
13809            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13810        let module_connection = ConnectionId::new(301);
13811        let registration = registry
13812            .register_with_control_ops(
13813                manifest("aft-push", PROTOCOL_VERSION),
13814                PROTOCOL_VERSION,
13815                module_connection,
13816                module_baseline_control_ops(),
13817            )
13818            .unwrap();
13819        let (module_tx, _module_rx) = mpsc::channel(8);
13820        let endpoint = forwarding
13821            .register_module_connection(
13822                module_connection,
13823                "aft-push".to_string(),
13824                PROTOCOL_VERSION,
13825                manifest_concurrency(&registration.manifest),
13826                FrameSink::new(module_tx),
13827            )
13828            .unwrap();
13829
13830        // A push op this version does not know is ignored (forward-compat), not errored.
13831        let unknown = Frame::build(
13832            FrameType::Push,
13833            control_flags(),
13834            0,
13835            0,
13836            5,
13837            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13838        )
13839        .unwrap();
13840        let out = handler.handle_status_update(endpoint, unknown).unwrap();
13841        assert!(
13842            out.is_empty(),
13843            "unknown push op must be ignored, got {out:?}"
13844        );
13845
13846        // A malformed body for a KNOWN op is a real error worth surfacing.
13847        let malformed = Frame::build(
13848            FrameType::Push,
13849            control_flags(),
13850            0,
13851            0,
13852            6,
13853            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13854        )
13855        .unwrap();
13856        let out = handler.handle_status_update(endpoint, malformed).unwrap();
13857        assert_eq!(out.len(), 1);
13858        assert_eq!(out[0].header.ty, FrameType::Error);
13859        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13860    }
13861
13862    #[test]
13863    fn hello_rejected_when_connection_already_owns_client_routes() {
13864        let registry = Arc::new(Registry::default());
13865        let forwarding = Arc::new(ForwardingTable::default());
13866        let handler =
13867            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13868        // Commits a client route on connection 202 (bound to a module on conn 101).
13869        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
13870        let client_connection = ConnectionId::new(202);
13871
13872        // That same connection now tries to register as a module: rejected, so one
13873        // connection never holds both client-route and module-endpoint state.
13874        let responses = handler
13875            .handle_control(
13876                client_connection,
13877                hello_frame("aft-second", PROTOCOL_VERSION, 9),
13878            )
13879            .unwrap();
13880        assert_eq!(responses[0].header.ty, FrameType::Error);
13881        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13882        assert!(registry.get_module("aft-second").unwrap().is_none());
13883    }
13884
13885    #[tokio::test]
13886    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13887        let registry = Arc::new(Registry::default());
13888        let forwarding = Arc::new(ForwardingTable::default());
13889        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13890        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13891        hello_via_sink(
13892            &handler,
13893            &module_ctx,
13894            &mut module_rx,
13895            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13896        )
13897        .await;
13898        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13899        let pending = forwarding
13900            .begin_route_bind_relay_for_test(
13901                client_ctx.connection_id,
13902                client_ctx.egress.clone(),
13903                2,
13904                "alpha",
13905            )
13906            .unwrap();
13907        forwarding
13908            .complete_pending_relay(
13909                module_ctx.connection_id,
13910                pending.corr,
13911                RouteBindRelayOutcome::Accepted,
13912            )
13913            .unwrap();
13914        client_rx.try_recv().unwrap();
13915        for module_id in ["beta", "alpha"] {
13916            let replies = handler
13917                .handle_control_frame(
13918                    &module_ctx,
13919                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13920                )
13921                .await
13922                .unwrap();
13923            assert_eq!(replies.len(), 1, "second HELLO must be refused");
13924            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13925        }
13926        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13927        assert!(registry.get_module("beta").unwrap().is_none());
13928        assert!(matches!(
13929            forwarding
13930                .lookup_data_route(
13931                    client_ctx.connection_id,
13932                    pending.client_channel,
13933                    pending.client_epoch,
13934                )
13935                .unwrap(),
13936            DataRoute::Client(DataRouteState::Bound(_))
13937        ));
13938        assert!(handler
13939            .hello_launch_nonces
13940            .lock()
13941            .unwrap()
13942            .presented(module_ctx.connection_id, Some("alpha-nonce")));
13943        assert!(module_rx.try_recv().is_err());
13944    }
13945
13946    #[test]
13947    fn reserved_module_hello_requires_matching_launch_nonce() {
13948        let registry = Arc::new(Registry::default());
13949        let supervisor = SupervisorHandle::new();
13950        // The supervisor recorded the nonce it injected when it spawned the reserved
13951        // module; the HELLO verifier checks against the same shared handle.
13952        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13953        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13954
13955        // A HELLO with NO nonce is rejected.
13956        let no_nonce = handler
13957            .handle_control(
13958                ConnectionId::new(1),
13959                hello_frame("vault", PROTOCOL_VERSION, 1),
13960            )
13961            .unwrap();
13962        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13963        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13964        assert!(registry.get_module("vault").unwrap().is_none());
13965
13966        // A HELLO with the WRONG nonce is rejected.
13967        let wrong = handler
13968            .handle_control(
13969                ConnectionId::new(2),
13970                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13971            )
13972            .unwrap();
13973        assert_eq!(wrong[0].header.ty, FrameType::Error);
13974        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13975        assert!(registry.get_module("vault").unwrap().is_none());
13976
13977        // A HELLO with the CORRECT nonce registers.
13978        let ok = handler
13979            .handle_control(
13980                ConnectionId::new(3),
13981                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13982            )
13983            .unwrap();
13984        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13985        assert!(registry.get_module("vault").unwrap().is_some());
13986    }
13987
13988    #[test]
13989    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13990        let registry = Arc::new(Registry::default());
13991        let supervisor = SupervisorHandle::new();
13992        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13993        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13994        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13995
13996        let squat = handler
13997            .handle_control(
13998                ConnectionId::new(1),
13999                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14000            )
14001            .unwrap();
14002        assert_eq!(squat[0].header.ty, FrameType::Error);
14003        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14004        assert!(parse_error(&squat[0])["message"]
14005            .as_str()
14006            .unwrap()
14007            .contains("fed:"));
14008
14009        let accepted_peer = handler
14010            .handle_control(
14011                ConnectionId::new(2),
14012                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14013            )
14014            .unwrap();
14015        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14016
14017        let accepted_short = handler
14018            .handle_control(
14019                ConnectionId::new(3),
14020                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14021            )
14022            .unwrap();
14023        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14024
14025        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14026            let response = handler
14027                .handle_control(
14028                    ConnectionId::new(conn),
14029                    hello_frame(module_id, PROTOCOL_VERSION, conn),
14030                )
14031                .unwrap();
14032            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14033        }
14034    }
14035
14036    #[test]
14037    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14038        let registry = Arc::new(Registry::default());
14039        let supervisor = SupervisorHandle::new();
14040        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14041        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14042        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14043        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14044
14045        let owner_nonce = handler
14046            .handle_control(
14047                ConnectionId::new(1),
14048                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14049            )
14050            .unwrap();
14051        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14052        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14053        assert!(registry.get_module("fed:special").unwrap().is_none());
14054
14055        let exact_nonce = handler
14056            .handle_control(
14057                ConnectionId::new(2),
14058                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14059            )
14060            .unwrap();
14061        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14062        assert!(registry.get_module("fed:special").unwrap().is_some());
14063    }
14064
14065    #[test]
14066    fn non_reserved_module_ignores_launch_nonce() {
14067        let registry = Arc::new(Registry::default());
14068        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
14069        // registration succeeds whether a spawned process echoes a nonce or not.
14070        let handler = ControlHandler::new(Arc::clone(&registry));
14071        let no_nonce = handler
14072            .handle_control(
14073                ConnectionId::new(1),
14074                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14075            )
14076            .unwrap();
14077        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14078        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14079
14080        let echoed_nonce = handler
14081            .handle_control(
14082                ConnectionId::new(2),
14083                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14084            )
14085            .unwrap();
14086        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14087        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14088    }
14089
14090    #[test]
14091    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14092        let handler = ControlHandler::default();
14093        let conn = ConnectionId::new(1);
14094        let malformed = Frame::build(
14095            FrameType::Hello,
14096            control_flags(),
14097            0,
14098            0,
14099            3,
14100            b"{not json".to_vec(),
14101        )
14102        .unwrap();
14103
14104        let error = handler.handle_control(conn, malformed).unwrap();
14105        assert_eq!(error[0].header.ty, FrameType::Error);
14106        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14107
14108        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14109        let pong = handler.handle_control(conn, ping).unwrap();
14110        assert_eq!(pong[0].header.ty, FrameType::Pong);
14111        assert_eq!(pong[0].header.corr, 4);
14112    }
14113
14114    #[test]
14115    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14116        let registry = Arc::new(Registry::default());
14117        let handler = ControlHandler::new(Arc::clone(&registry));
14118
14119        handler
14120            .handle_control(
14121                ConnectionId::new(1),
14122                hello_frame("aft", PROTOCOL_VERSION, 1),
14123            )
14124            .unwrap();
14125        let duplicate = handler
14126            .handle_control(
14127                ConnectionId::new(2),
14128                hello_frame("aft", PROTOCOL_VERSION, 2),
14129            )
14130            .unwrap();
14131
14132        assert_eq!(duplicate[0].header.ty, FrameType::Error);
14133        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14134        let registration = registry.get_module("aft").unwrap().unwrap();
14135        assert_eq!(registration.connection_id, ConnectionId::new(1));
14136    }
14137
14138    #[test]
14139    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14140        let registry = Arc::new(Registry::default());
14141        let forwarding = Arc::new(ForwardingTable::default());
14142        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14143        let handler =
14144            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14145                .with_process_liveness(process_liveness);
14146        let (ctx, route_channel, route_epoch) =
14147            bind_liveness_route(&registry, &forwarding, "aft-dead");
14148        let responses = handler
14149            .handle_route_poll(
14150                &ctx,
14151                route_poll_frame(41, PollKind::Liveness, route_channel),
14152                route_channel,
14153                route_epoch,
14154                PollKind::Liveness,
14155            )
14156            .unwrap();
14157
14158        assert_eq!(responses.len(), 1);
14159        assert_eq!(responses[0].header.ty, FrameType::Response);
14160        assert_route_poll_liveness(&responses[0], false);
14161    }
14162
14163    #[test]
14164    fn liveness_poll_without_process_source_uses_bound_route() {
14165        let registry = Arc::new(Registry::default());
14166        let forwarding = Arc::new(ForwardingTable::default());
14167        let handler =
14168            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14169        let (ctx, route_channel, route_epoch) =
14170            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
14171        let responses = handler
14172            .handle_route_poll(
14173                &ctx,
14174                route_poll_frame(42, PollKind::Liveness, route_channel),
14175                route_channel,
14176                route_epoch,
14177                PollKind::Liveness,
14178            )
14179            .unwrap();
14180
14181        assert_route_poll_liveness(&responses[0], true);
14182    }
14183
14184    #[test]
14185    fn liveness_poll_untracked_process_source_uses_bound_route() {
14186        let registry = Arc::new(Registry::default());
14187        let forwarding = Arc::new(ForwardingTable::default());
14188        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14189        let handler =
14190            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14191                .with_process_liveness(process_liveness);
14192        let (ctx, route_channel, route_epoch) =
14193            bind_liveness_route(&registry, &forwarding, "aft-untracked");
14194        let responses = handler
14195            .handle_route_poll(
14196                &ctx,
14197                route_poll_frame(43, PollKind::Liveness, route_channel),
14198                route_channel,
14199                route_epoch,
14200                PollKind::Liveness,
14201            )
14202            .unwrap();
14203
14204        assert_route_poll_liveness(&responses[0], true);
14205    }
14206
14207    #[tokio::test]
14208    async fn unknown_op_returns_unknown_control_op() {
14209        let handler = ControlHandler::default();
14210        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14211        let request = Frame::build(
14212            FrameType::Request,
14213            control_flags(),
14214            0,
14215            0,
14216            55,
14217            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14218        )
14219        .unwrap();
14220
14221        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14222
14223        assert_eq!(response.len(), 1);
14224        assert_eq!(response[0].header.ty, FrameType::Error);
14225        assert_eq!(response[0].header.corr, 55);
14226        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14227    }
14228
14229    #[tokio::test]
14230    async fn supervisor_provenance_rejects_unknown_exact_module() {
14231        let handler = ControlHandler::default();
14232        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14233        let request = Frame::build(
14234            FrameType::Request,
14235            control_flags(),
14236            0,
14237            0,
14238            57,
14239            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14240        )
14241        .unwrap();
14242
14243        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14244
14245        assert_eq!(response.len(), 1);
14246        assert_eq!(response[0].header.ty, FrameType::Error);
14247        assert_eq!(response[0].header.corr, 57);
14248        let error = parse_error(&response[0]);
14249        assert_eq!(error["code"], "unknown_module");
14250        assert_eq!(error["message"], "module_id 'missing' is not supervised");
14251    }
14252
14253    #[test]
14254    fn provenance_probe_override_keeps_handler_tests_deterministic() {
14255        let expected = subc_control::RunningImageAgreement::Unavailable {
14256            reason: subc_control::RunningImageUnavailableReason::HashFailed,
14257        };
14258        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14259        assert_eq!(handler.provenance_probe_override, Some(expected));
14260    }
14261
14262    #[test]
14263    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14264        let verdict = reload_verdict(
14265            std::path::Path::new("/bin/new"),
14266            Some(std::path::Path::new("/bin/old")),
14267            subc_control::RunningImageAgreement::Unavailable {
14268                reason: subc_control::RunningImageUnavailableReason::HashFailed,
14269            },
14270        );
14271        assert!(matches!(
14272            verdict.path,
14273            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14274                if configured == std::path::Path::new("/bin/new")
14275                    && spawned_from == std::path::Path::new("/bin/old")
14276        ));
14277    }
14278
14279    #[test]
14280    fn launch_spec_without_running_digest_is_unavailable_not_match() {
14281        use subc_control::{
14282            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14283        };
14284        let configured = crate::supervise::ModuleSpec {
14285            module_id: "legacy".into(),
14286            program: "/module".into(),
14287            args: Vec::new(),
14288            env: Vec::new(),
14289            reserved: false,
14290            reserved_prefixes: Vec::new(),
14291            protocol: subc_control::ModuleProtocol::Subc,
14292            overlap: Default::default(),
14293        };
14294        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14295        assert_eq!(
14296            launch_spec_agreement(&configured, false, Some(&digest)),
14297            Agreement::Unavailable {
14298                reason: Reason::NotRunning
14299            }
14300        );
14301        assert_eq!(
14302            launch_spec_agreement(&configured, true, None),
14303            Agreement::Unavailable {
14304                reason: Reason::SpawnedSpecUnavailable
14305            }
14306        );
14307        assert_eq!(
14308            launch_spec_agreement(&configured, true, Some(&digest)),
14309            Agreement::Match
14310        );
14311    }
14312
14313    #[cfg(windows)]
14314    #[test]
14315    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14316        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14317        let configured = dir.join("module.exe");
14318        std::fs::write(&configured, b"executable path fixture").unwrap();
14319        let spawned = std::fs::canonicalize(&configured).unwrap();
14320        assert_ne!(
14321            configured, spawned,
14322            "fixture must exercise different path spellings"
14323        );
14324        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14325        let lower_case = configured.to_string_lossy().to_lowercase();
14326        for spelling in [
14327            &configured,
14328            Path::new(&forward_slashes),
14329            Path::new(&lower_case),
14330        ] {
14331            let verdict = reload_verdict(
14332                spelling,
14333                Some(&spawned),
14334                subc_control::RunningImageAgreement::Unavailable {
14335                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14336                },
14337            );
14338            assert_eq!(
14339                verdict.path,
14340                ReloadPathAgreement::Match,
14341                "configured {spelling:?}, spawned {spawned:?}"
14342            );
14343        }
14344    }
14345
14346    #[cfg(windows)]
14347    #[test]
14348    fn windows_reload_path_keeps_different_existing_program_pending() {
14349        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14350        let old = dir.join("old.exe");
14351        let new = dir.join("new.exe");
14352        std::fs::write(&old, b"same executable content").unwrap();
14353        std::fs::copy(&old, &new).unwrap();
14354        let spawned = std::fs::canonicalize(&old).unwrap();
14355        let image = subc_control::RunningImageAgreement::Unavailable {
14356            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14357        };
14358        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14359        assert!(
14360            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14361            if configured == new && spawned_from == spawned)
14362        );
14363        let missing = dir.join("missing.exe");
14364        assert!(matches!(
14365            reload_verdict(&missing, Some(&spawned), image).path,
14366            ReloadPathAgreement::Mismatch { .. }
14367        ));
14368    }
14369
14370    #[test]
14371    fn reload_verdict_detects_replaced_image_at_same_path() {
14372        let image = subc_control::RunningImageAgreement::Mismatch {
14373            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14374                digest: "old".into(),
14375            },
14376            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14377                digest: "new".into(),
14378            },
14379        };
14380        let verdict = reload_verdict(
14381            std::path::Path::new("/bin/same"),
14382            Some(std::path::Path::new("/bin/same")),
14383            image.clone(),
14384        );
14385        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14386        assert_eq!(verdict.image, image);
14387    }
14388
14389    #[test]
14390    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14391        let image = subc_control::RunningImageAgreement::Unavailable {
14392            reason: subc_control::RunningImageUnavailableReason::NotRunning,
14393        };
14394        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14395        assert_eq!(
14396            verdict.path,
14397            subc_control::ReloadPathAgreement::Unavailable {
14398                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14399            }
14400        );
14401        assert_eq!(verdict.image, image);
14402
14403        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14404            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14405        };
14406        let verdict = reload_verdict(
14407            std::path::Path::new("/bin/same"),
14408            Some(std::path::Path::new("/bin/same")),
14409            unconfirmed.clone(),
14410        );
14411        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14412        assert_eq!(verdict.image, unconfirmed);
14413    }
14414
14415    #[test]
14416    fn reload_verdict_preserves_each_image_unavailability_reason() {
14417        use subc_control::RunningImageUnavailableReason as Reason;
14418
14419        for reason in [
14420            Reason::NotRunning,
14421            Reason::UnsupportedPlatform,
14422            Reason::RunningExecutableUnreadable,
14423            Reason::SpawnedPathUnreadable,
14424            Reason::HashFailed,
14425            Reason::ProcessIdentityUnconfirmed,
14426            Reason::Unknown("future_probe_reason".to_string()),
14427        ] {
14428            let image = subc_control::RunningImageAgreement::Unavailable {
14429                reason: reason.clone(),
14430            };
14431            let verdict = reload_verdict(
14432                std::path::Path::new("/bin/same"),
14433                Some(std::path::Path::new("/bin/same")),
14434                image.clone(),
14435            );
14436            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14437            assert_eq!(verdict.image, image, "{reason:?}");
14438        }
14439    }
14440
14441    #[tokio::test]
14442    async fn malformed_control_bodies_return_invalid_control_body() {
14443        let handler = ControlHandler::default();
14444        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14445
14446        for (corr, body) in [
14447            (56, br#"{"route_channel":1}"#.as_slice()),
14448            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14449            (
14450                58,
14451                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14452            ),
14453        ] {
14454            let request = Frame::build(
14455                FrameType::Request,
14456                control_flags(),
14457                0,
14458                0,
14459                corr,
14460                body.to_vec(),
14461            )
14462            .unwrap();
14463            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14464
14465            assert_eq!(response.len(), 1);
14466            assert_eq!(response[0].header.ty, FrameType::Error);
14467            assert_eq!(response[0].header.corr, corr);
14468            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14469        }
14470    }
14471
14472    #[tokio::test]
14473    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14474        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14475        let registry = Arc::new(Registry::default());
14476        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14477        let router = Router::with_control_handler(Arc::clone(&control));
14478        let connection = router.begin_connection();
14479        let (ctx, mut rx) = route_ctx(connection.id());
14480
14481        router
14482            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14483            .await
14484            .unwrap();
14485        let response = rx.recv().await.unwrap();
14486        let ack = parse_ack(&response);
14487        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14488        let channel = 1;
14489
14490        let goodbye =
14491            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14492        router.route_for_connection(&ctx, goodbye).await.unwrap();
14493        assert!(rx.try_recv().is_err());
14494        assert!(registry.get_module("aft").unwrap().is_none());
14495
14496        router
14497            .route_for_connection(&ctx, channel_request(channel, 13))
14498            .await
14499            .unwrap();
14500        let error_frame = rx.recv().await.unwrap();
14501        assert_eq!(error_frame.header.ty, FrameType::Error);
14502        assert_eq!(error_frame.header.channel, channel);
14503        let captured = crate::router::test_log::captured_logs(&logs);
14504        assert_eq!(
14505            captured
14506                .lines()
14507                .filter(|line| {
14508                    line.contains("module_id=aft")
14509                        && line.contains("reason=explicit_goodbye")
14510                        && line.contains("module registration ended")
14511                })
14512                .count(),
14513            1,
14514            "unexpected GOODBYE registry log: {captured}"
14515        );
14516    }
14517
14518    #[tokio::test]
14519    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14520        let registry = Arc::new(Registry::default());
14521        let handler = ControlHandler::new(registry).with_capability_config(
14522            [("prov".to_string(), true), ("cons".to_string(), true)],
14523            BTreeMap::new(),
14524        );
14525        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14526        register_capability_manifest(
14527            &handler,
14528            &provider_ctx,
14529            &mut provider_rx,
14530            capability_manifest("prov", &["thing/v1"], &[]),
14531            1,
14532        )
14533        .await;
14534        let mut consumer = capability_manifest("cons", &[], &[]);
14535        consumer.capabilities.as_mut().unwrap().requires.push(
14536            subc_protocol::manifest::CapabilityRequirement {
14537                capability: "thing/v1".to_string(),
14538                need: subc_protocol::manifest::CapabilityNeed::Required,
14539            },
14540        );
14541        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14542        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14543        assert_eq!(
14544            handler.capability_evaluator.verdict("cons", "thing/v1"),
14545            Some(CapabilityVerdict::Provided)
14546        );
14547        let (mut client_rx, _) = open_route_for_capability_test(
14548            &handler,
14549            &provider_ctx,
14550            &mut provider_rx,
14551            703,
14552            3,
14553            "prov",
14554            None,
14555        )
14556        .await;
14557        let goodbye =
14558            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14559        handler
14560            .handle_control_frame(&provider_ctx, goodbye)
14561            .await
14562            .unwrap();
14563        assert_eq!(
14564            handler.capability_evaluator.verdict("cons", "thing/v1"),
14565            Some(CapabilityVerdict::NeverProvided)
14566        );
14567        let closed = client_rx
14568            .try_recv()
14569            .expect("GOODBYE pushes route.closed before route GOODBYE");
14570        assert!(
14571            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14572            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14573        );
14574        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14575        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14576    }
14577
14578    struct RouteClosedFixture {
14579        handler: ControlHandler,
14580        #[cfg(unix)]
14581        supervisor: Supervisor,
14582        module_ctx: RouteCtx,
14583        #[cfg(unix)]
14584        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14585        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14586        route: Arc<crate::forwarding::RouteBinding>,
14587    }
14588
14589    async fn route_closed_fixture() -> RouteClosedFixture {
14590        let registry = Arc::new(Registry::default());
14591        let handler = ControlHandler::new(Arc::clone(&registry));
14592        #[cfg(unix)]
14593        let supervisor = Supervisor::new(registry, RestartPolicy::default())
14594            .with_forwarding(Arc::clone(&handler.forwarding));
14595        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14596        register_capability_manifest(
14597            &handler,
14598            &module_ctx,
14599            &mut module_rx,
14600            capability_manifest("prov", &[], &[]),
14601            1,
14602        )
14603        .await;
14604        let (client_rx, bind) = open_route_for_capability_test(
14605            &handler,
14606            &module_ctx,
14607            &mut module_rx,
14608            702,
14609            2,
14610            "prov",
14611            None,
14612        )
14613        .await;
14614        let ModuleControlRequest::RouteBind {
14615            route_channel: channel,
14616            epoch: route_epoch,
14617            ..
14618        } = bind
14619        else {
14620            panic!("expected route.bind");
14621        };
14622        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14623            handler
14624                .forwarding
14625                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14626                .unwrap()
14627        else {
14628            panic!("the acknowledged route must be bound");
14629        };
14630        RouteClosedFixture {
14631            handler,
14632            #[cfg(unix)]
14633            supervisor,
14634            module_ctx,
14635            #[cfg(unix)]
14636            module_rx,
14637            client_rx,
14638            route,
14639        }
14640    }
14641
14642    impl RouteClosedFixture {
14643        #[cfg(unix)]
14644        async fn cut(&mut self) {
14645            let notices = async {
14646                let mut draining = self.module_rx.recv().await.unwrap();
14647                assert!(matches!(
14648                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14649                        &draining.body
14650                    )
14651                    .unwrap(),
14652                    subc_protocol::session::ModuleControlCommand::Draining {
14653                        reason: RouteCloseReason::Restart,
14654                        ..
14655                    }
14656                ));
14657                draining.flushed.take().unwrap().send(()).unwrap();
14658                let mut closing = self.client_rx.recv().await.unwrap();
14659                assert!(matches!(
14660                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14661                    ClientControlPush::RouteClosing {
14662                        reason: RouteCloseReason::Restart,
14663                        ..
14664                    }
14665                ));
14666                closing.flushed.take().unwrap().send(()).unwrap();
14667            };
14668            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14669            result.unwrap();
14670        }
14671
14672        fn disconnect(&mut self) -> ClientControlPush {
14673            self.handler
14674                .cleanup_connection(self.module_ctx.connection_id)
14675                .unwrap();
14676            let closed = self.client_rx.try_recv().unwrap();
14677            assert_eq!(closed.header.ty, FrameType::Push);
14678            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14679            assert!(matches!(
14680                &push,
14681                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14682                    if channels == &[self.route.client_channel]
14683            ));
14684            assert_eq!(
14685                self.client_rx.try_recv().unwrap().header.ty,
14686                FrameType::Goodbye
14687            );
14688            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14689            push
14690        }
14691    }
14692
14693    #[cfg(unix)]
14694    #[tokio::test(start_paused = true)]
14695    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14696        let mut fixture = route_closed_fixture().await;
14697        fixture.cut().await;
14698        assert!(matches!(
14699            fixture.disconnect(),
14700            ClientControlPush::RouteClosed {
14701                reason: RouteCloseReason::Restart,
14702                drained: true,
14703                ..
14704            }
14705        ));
14706    }
14707
14708    #[cfg(unix)]
14709    #[tokio::test(start_paused = true)]
14710    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14711        let mut fixture = route_closed_fixture().await;
14712        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14713        let started = tokio::time::Instant::now();
14714        fixture.cut().await;
14715        assert!(started.elapsed() >= Duration::from_secs(2));
14716        assert_eq!(fixture.route.flow.in_flight(), 1);
14717        assert!(matches!(
14718            fixture.disconnect(),
14719            ClientControlPush::RouteClosed {
14720                reason: RouteCloseReason::Restart,
14721                drained: false,
14722                ..
14723            }
14724        ));
14725    }
14726
14727    #[tokio::test]
14728    async fn module_connection_crash_reports_crash_and_not_drained() {
14729        let mut fixture = route_closed_fixture().await;
14730        // The server calls cleanup_connection on module EOF. Without a planned
14731        // daemon shutdown, that disconnect must remain a crash with no prior
14732        // route.closing notice, even when the route has no in-flight calls.
14733        assert!(matches!(
14734            fixture.disconnect(),
14735            ClientControlPush::RouteClosed {
14736                reason: RouteCloseReason::Crash,
14737                drained: false,
14738                ..
14739            }
14740        ));
14741    }
14742
14743    #[tokio::test]
14744    async fn dropping_router_connection_releases_registration() {
14745        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14746        let registry = Arc::new(Registry::default());
14747        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14748        let router = Router::with_control_handler(Arc::clone(&control));
14749        let connection = router.begin_connection();
14750        let connection_id = connection.id();
14751        let (ctx, mut rx) = route_ctx(connection_id);
14752
14753        router
14754            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14755            .await
14756            .unwrap();
14757        let response = rx.recv().await.unwrap();
14758        let ack = parse_ack(&response);
14759        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14760        assert!(registry.get_module("aft").unwrap().is_some());
14761
14762        drop(connection);
14763
14764        assert!(registry.get_module("aft").unwrap().is_none());
14765        assert_eq!(registry.active_registration_count().unwrap(), 0);
14766
14767        control
14768            .cleanup_connection(ConnectionId::new(u64::MAX))
14769            .unwrap();
14770        let captured = crate::router::test_log::captured_logs(&logs);
14771        println!("captured registry lifecycle logs:\n{captured}");
14772        let events: Vec<_> = captured
14773            .lines()
14774            .filter(|line| {
14775                line.contains("module registered module_id=aft ")
14776                    || line.contains("module registration ended")
14777            })
14778            .collect();
14779        assert_eq!(
14780            events.len(),
14781            2,
14782            "unexpected registry lifecycle logs: {captured}"
14783        );
14784        assert!(events[0].contains("module registered module_id=aft "));
14785        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14786        assert!(events[1].contains(&format!(
14787            "module_id=aft connection_id={}",
14788            connection_id.get()
14789        )));
14790        assert!(events[1].contains("reason=connection_closed"));
14791        assert!(events[1].contains("module registration ended"));
14792        assert_eq!(
14793            captured
14794                .lines()
14795                .filter(|line| line.contains("module registered module_id=aft "))
14796                .count(),
14797            1,
14798            "legacy registration admission line must appear once: {captured}"
14799        );
14800    }
14801
14802    #[tokio::test]
14803    async fn hello_registration_keeps_legacy_module_registered_line_once() {
14804        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14805        let registry = Arc::new(Registry::default());
14806        let control = ControlHandler::new(Arc::clone(&registry));
14807        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14808        hello_via_sink(
14809            &control,
14810            &ctx,
14811            &mut rx,
14812            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14813        )
14814        .await;
14815
14816        let captured = crate::router::test_log::captured_logs(&logs);
14817        let admissions: Vec<_> = captured
14818            .lines()
14819            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14820            .collect();
14821        assert_eq!(
14822            admissions.len(),
14823            1,
14824            "legacy admission line must remain exactly once: {captured}"
14825        );
14826        assert!(admissions[0].contains("routable_provider=true"));
14827        assert!(admissions[0].contains("connection_id=777"));
14828    }
14829
14830    fn capability_manifest(
14831        module_id: &str,
14832        provides: &[&str],
14833        must_never_reach: &[&str],
14834    ) -> ModuleManifest {
14835        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14836        manifest.capabilities = Some(CapabilityDeclarations {
14837            provides: provides
14838                .iter()
14839                .map(|capability| (*capability).to_string())
14840                .collect(),
14841            requires: Vec::new(),
14842            must_never_reach: must_never_reach
14843                .iter()
14844                .map(|capability| (*capability).to_string())
14845                .collect(),
14846        });
14847        manifest
14848    }
14849
14850    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14851        Frame::build(
14852            FrameType::Hello,
14853            control_flags(),
14854            0,
14855            0,
14856            corr,
14857            serde_json::to_vec(&ModuleHelloBody {
14858                protocol_ver: manifest.protocol_ver,
14859                manifest,
14860                control_ops: None,
14861                launch_nonce: None,
14862            })
14863            .expect("capability test HELLO serializes"),
14864        )
14865        .expect("capability test HELLO frame builds")
14866    }
14867
14868    fn catalog_update_with_capabilities_frame(
14869        corr: u64,
14870        capabilities: CapabilityDeclarations,
14871    ) -> Frame {
14872        Frame::build(
14873            FrameType::Request,
14874            control_flags(),
14875            0,
14876            0,
14877            corr,
14878            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14879                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14880                capabilities: Some(capabilities),
14881                ready: None,
14882            })
14883            .expect("capability catalog.update serializes"),
14884        )
14885        .expect("capability catalog.update frame builds")
14886    }
14887
14888    async fn register_capability_manifest(
14889        handler: &ControlHandler,
14890        ctx: &RouteCtx,
14891        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14892        manifest: ModuleManifest,
14893        corr: u64,
14894    ) {
14895        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14896    }
14897
14898    async fn open_route_for_capability_test(
14899        handler: &ControlHandler,
14900        target_ctx: &RouteCtx,
14901        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14902        client_connection_id: u64,
14903        corr: u64,
14904        target_module_id: &str,
14905        consumer_identity: Option<ConsumerIdentity>,
14906    ) -> (
14907        mpsc::Receiver<crate::router::OutboundFrame>,
14908        ModuleControlRequest,
14909    ) {
14910        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14911        let route_handler = handler.clone();
14912        let target_module_id = target_module_id.to_string();
14913        let route_task = tokio::spawn(async move {
14914            route_handler
14915                .handle_control_frame(
14916                    &client_ctx,
14917                    route_open_frame_with_admission_facts(
14918                        corr,
14919                        &target_module_id,
14920                        unique_project_root("admission-facts"),
14921                        consumer_identity,
14922                        None,
14923                    ),
14924                )
14925                .await
14926                .expect("capability test route.open succeeds")
14927        });
14928        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14929            .await
14930            .expect("capability test route.open must reach route.bind")
14931            .expect("target control receiver stays open");
14932        let bind_request: ModuleControlRequest =
14933            serde_json::from_slice(&bind.body).expect("route.bind decodes");
14934        handler
14935            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14936            .await
14937            .expect("capability test route.bind ACK succeeds");
14938        assert!(route_task.await.expect("route.open task joins").is_empty());
14939        let opened = client_rx
14940            .recv()
14941            .await
14942            .expect("successful route.open publishes a response");
14943        assert!(matches!(
14944            serde_json::from_slice::<ClientControlResponse>(&opened.body),
14945            Ok(ClientControlResponse::RouteOpen { .. })
14946        ));
14947        (client_rx, bind_request)
14948    }
14949
14950    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
14951        assert_eq!(frame.header.ty, FrameType::Push);
14952        assert_eq!(frame.header.channel, 0);
14953        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14954            .expect("route.closed control push decodes");
14955        let ClientControlPush::RouteClosed { channels, .. } = &push else {
14956            panic!("expected route.closed");
14957        };
14958        assert_eq!(channels.len(), 1, "exactly one violating route closed");
14959        let closed_channel = channels[0];
14960        let channels = channels.clone();
14961        assert_eq!(
14962            push,
14963            ClientControlPush::RouteClosed {
14964                module_id: target_module_id.to_string(),
14965                channels,
14966                reason: RouteCloseReason::CapabilityDenied,
14967                drained: false,
14968                abandoned: 0,
14969                excluded_subscriptions: 0,
14970                terminal: Some(false),
14971            }
14972        );
14973        closed_channel
14974    }
14975
14976    #[tokio::test]
14977    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14978        let registry = Arc::new(Registry::default());
14979        let forwarding = Arc::new(ForwardingTable::default());
14980        let supervisor = SupervisorHandle::new();
14981        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14982        let handler =
14983            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14984                .with_supervisor(supervisor);
14985        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14986        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14987        register_capability_manifest(
14988            &handler,
14989            &target_ctx,
14990            &mut target_rx,
14991            capability_manifest("target", &["credentials-provider/v1"], &[]),
14992            1,
14993        )
14994        .await;
14995        register_capability_manifest(
14996            &handler,
14997            &opener_ctx,
14998            &mut opener_rx,
14999            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15000            2,
15001        )
15002        .await;
15003
15004        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15005        let replies = handler
15006            .handle_control_frame(
15007                &client_ctx,
15008                route_open_frame_with_admission_facts(
15009                    3,
15010                    "target",
15011                    unique_project_root("admission-facts"),
15012                    Some(ConsumerIdentity {
15013                        module_id: "opener".to_string(),
15014                        launch_nonce: "opener-nonce".to_string(),
15015                    }),
15016                    None,
15017                ),
15018            )
15019            .await
15020            .expect("denied route.open returns a typed frame");
15021        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15022        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15023        assert!(
15024            target_rx.try_recv().is_err(),
15025            "forbidden route.open must not relay route.bind"
15026        );
15027    }
15028
15029    #[tokio::test]
15030    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15031        let registry = Arc::new(Registry::default());
15032        let forwarding = Arc::new(ForwardingTable::default());
15033        let supervisor = SupervisorHandle::new();
15034        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15035        let handler =
15036            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15037                .with_supervisor(supervisor);
15038        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15039        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15040        register_capability_manifest(
15041            &handler,
15042            &target_ctx,
15043            &mut target_rx,
15044            capability_manifest("target", &["credentials-provider/v1"], &[]),
15045            1,
15046        )
15047        .await;
15048        register_capability_manifest(
15049            &handler,
15050            &old_opener_ctx,
15051            &mut old_opener_rx,
15052            capability_manifest("opener", &[], &[]),
15053            2,
15054        )
15055        .await;
15056        let (mut client_rx, _) = open_route_for_capability_test(
15057            &handler,
15058            &target_ctx,
15059            &mut target_rx,
15060            712,
15061            3,
15062            "target",
15063            Some(ConsumerIdentity {
15064                module_id: "opener".to_string(),
15065                launch_nonce: "opener-nonce".to_string(),
15066            }),
15067        )
15068        .await;
15069        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15070
15071        handler
15072            .cleanup_connection(old_opener_ctx.connection_id)
15073            .expect("old opener registration cleans up");
15074        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15075        register_capability_manifest(
15076            &handler,
15077            &new_opener_ctx,
15078            &mut new_opener_rx,
15079            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15080            4,
15081        )
15082        .await;
15083
15084        let closed_channel = assert_capability_denied_push(
15085            client_rx
15086                .try_recv()
15087                .expect("HELLO deny addition must emit route.closed")
15088                .frame,
15089            "target",
15090        );
15091        let goodbye = client_rx
15092            .try_recv()
15093            .expect("HELLO deny addition must send the client a route GOODBYE")
15094            .frame;
15095        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15096        assert_eq!(goodbye.header.channel, closed_channel);
15097        assert_ne!(goodbye.header.channel, 0);
15098        assert_ne!(goodbye.header.epoch, 0);
15099        assert!(client_rx.try_recv().is_err());
15100        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15101        assert!(matches!(
15102            target_rx.try_recv(),
15103            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15104        ));
15105    }
15106
15107    #[tokio::test]
15108    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15109        let registry = Arc::new(Registry::default());
15110        let forwarding = Arc::new(ForwardingTable::default());
15111        let supervisor = SupervisorHandle::new();
15112        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15113        let handler =
15114            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15115                .with_supervisor(supervisor);
15116        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15117        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15118        register_capability_manifest(
15119            &handler,
15120            &target_ctx,
15121            &mut target_rx,
15122            capability_manifest("target", &[], &[]),
15123            1,
15124        )
15125        .await;
15126        register_capability_manifest(
15127            &handler,
15128            &opener_ctx,
15129            &mut opener_rx,
15130            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15131            2,
15132        )
15133        .await;
15134        let (mut client_rx, _) = open_route_for_capability_test(
15135            &handler,
15136            &target_ctx,
15137            &mut target_rx,
15138            722,
15139            3,
15140            "target",
15141            Some(ConsumerIdentity {
15142                module_id: "opener".to_string(),
15143                launch_nonce: "opener-nonce".to_string(),
15144            }),
15145        )
15146        .await;
15147        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15148
15149        let replies = handler
15150            .handle_control_frame(
15151                &target_ctx,
15152                catalog_update_with_capabilities_frame(
15153                    4,
15154                    CapabilityDeclarations {
15155                        provides: vec!["credentials-provider/v1".to_string()],
15156                        requires: Vec::new(),
15157                        must_never_reach: Vec::new(),
15158                    },
15159                ),
15160            )
15161            .await
15162            .expect("claim catalog.update succeeds");
15163        assert!(matches!(
15164            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15165            Ok(ModuleControlResponseToModule::CatalogUpdate {})
15166        ));
15167        let closed_channel = assert_capability_denied_push(
15168            client_rx
15169                .try_recv()
15170                .expect("claim addition must emit route.closed")
15171                .frame,
15172            "target",
15173        );
15174        let goodbye = client_rx
15175            .try_recv()
15176            .expect("claim addition must send the client a route GOODBYE")
15177            .frame;
15178        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15179        assert_eq!(goodbye.header.channel, closed_channel);
15180        assert_ne!(goodbye.header.channel, 0);
15181        assert_ne!(goodbye.header.epoch, 0);
15182        assert!(client_rx.try_recv().is_err());
15183        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15184        assert!(matches!(
15185            target_rx.try_recv(),
15186            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15187        ));
15188    }
15189
15190    #[tokio::test]
15191    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15192        let registry = Arc::new(Registry::default());
15193        let forwarding = Arc::new(ForwardingTable::default());
15194        let supervisor = SupervisorHandle::new();
15195        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15196        let handler =
15197            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15198                .with_supervisor(supervisor);
15199        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15200        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15201        register_capability_manifest(
15202            &handler,
15203            &target_ctx,
15204            &mut target_rx,
15205            capability_manifest("target", &["credentials-provider/v1"], &[]),
15206            1,
15207        )
15208        .await;
15209        register_capability_manifest(
15210            &handler,
15211            &opener_ctx,
15212            &mut opener_rx,
15213            capability_manifest("opener", &[], &[]),
15214            2,
15215        )
15216        .await;
15217        let (mut client_rx, _) = open_route_for_capability_test(
15218            &handler,
15219            &target_ctx,
15220            &mut target_rx,
15221            732,
15222            3,
15223            "target",
15224            Some(ConsumerIdentity {
15225                module_id: "opener".to_string(),
15226                launch_nonce: "opener-nonce".to_string(),
15227            }),
15228        )
15229        .await;
15230        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15231
15232        handler
15233            .handle_control_frame(
15234                &target_ctx,
15235                catalog_update_with_capabilities_frame(
15236                    4,
15237                    CapabilityDeclarations {
15238                        provides: Vec::new(),
15239                        requires: Vec::new(),
15240                        must_never_reach: Vec::new(),
15241                    },
15242                ),
15243            )
15244            .await
15245            .expect("claim removal catalog.update succeeds");
15246        assert_eq!(
15247            forwarding.active_binding_count().unwrap(),
15248            1,
15249            "removing an attested target claim must leave the route census unchanged"
15250        );
15251        assert!(
15252            client_rx.try_recv().is_err(),
15253            "claim removal must not emit route.closed or a client route GOODBYE"
15254        );
15255        assert!(
15256            target_rx.try_recv().is_err(),
15257            "claim removal must not send the target a route GOODBYE"
15258        );
15259    }
15260
15261    /// A direct client may open a route to a denied capability provider; this
15262    /// policy applies only to attested supervised module origins, not to direct clients.
15263    #[tokio::test]
15264    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15265        let registry = Arc::new(Registry::default());
15266        let forwarding = Arc::new(ForwardingTable::default());
15267        let supervisor = SupervisorHandle::new();
15268        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15269        let handler =
15270            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15271                .with_supervisor(supervisor);
15272        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15273        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15274        register_capability_manifest(
15275            &handler,
15276            &target_ctx,
15277            &mut target_rx,
15278            capability_manifest("target", &["credentials-provider/v1"], &[]),
15279            1,
15280        )
15281        .await;
15282        register_capability_manifest(
15283            &handler,
15284            &opener_ctx,
15285            &mut opener_rx,
15286            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15287            2,
15288        )
15289        .await;
15290
15291        let (_client_rx, bind) = open_route_for_capability_test(
15292            &handler,
15293            &target_ctx,
15294            &mut target_rx,
15295            742,
15296            3,
15297            "target",
15298            None,
15299        )
15300        .await;
15301        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15302            panic!("direct scope-honesty route must bind");
15303        };
15304        assert_eq!(principal, Some(Principal::Direct));
15305        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15306    }
15307
15308    /// A module that denies a capability receives no self-route exemption when it
15309    /// also attestedly provides that capability.
15310    #[tokio::test]
15311    async fn must_never_reach_self_route_is_capability_forbidden() {
15312        let registry = Arc::new(Registry::default());
15313        let forwarding = Arc::new(ForwardingTable::default());
15314        let supervisor = SupervisorHandle::new();
15315        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15316        let handler =
15317            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15318                .with_supervisor(supervisor);
15319        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15320        register_capability_manifest(
15321            &handler,
15322            &self_ctx,
15323            &mut self_rx,
15324            capability_manifest(
15325                "self-provider",
15326                &["credentials-provider/v1"],
15327                &["credentials-provider/v1"],
15328            ),
15329            1,
15330        )
15331        .await;
15332
15333        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15334        let replies = handler
15335            .handle_control_frame(
15336                &client_ctx,
15337                route_open_frame_with_admission_facts(
15338                    2,
15339                    "self-provider",
15340                    unique_project_root("admission-facts"),
15341                    Some(ConsumerIdentity {
15342                        module_id: "self-provider".to_string(),
15343                        launch_nonce: "self-nonce".to_string(),
15344                    }),
15345                    None,
15346                ),
15347            )
15348            .await
15349            .expect("self-route refusal returns a typed frame");
15350        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15351        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15352        assert!(
15353            self_rx.try_recv().is_err(),
15354            "self denial must not relay route.bind"
15355        );
15356    }
15357
15358    #[test]
15359    fn unsupported_channel_zero_frame_returns_error() {
15360        let handler = ControlHandler::default();
15361        let request = Frame::build(
15362            FrameType::Request,
15363            control_flags(),
15364            0,
15365            0,
15366            21,
15367            b"opaque".to_vec(),
15368        )
15369        .unwrap();
15370
15371        let response = handler
15372            .handle_control(ConnectionId::new(1), request)
15373            .unwrap();
15374
15375        assert_eq!(response[0].header.ty, FrameType::Error);
15376        assert_eq!(
15377            parse_error(&response[0])["code"],
15378            "unsupported_control_frame"
15379        );
15380    }
15381
15382    /// Blue/green swap at the control-plane boundary. The supervisor that opens
15383    /// a swap is not wired yet, so the candidate is registered here directly
15384    /// into the registry and forwarding candidate slots, the way the swap's
15385    /// HELLO admission will.
15386    mod swap {
15387        use super::*;
15388
15389        const INCUMBENT: ConnectionId = ConnectionId::new(30);
15390        const CANDIDATE: ConnectionId = ConnectionId::new(40);
15391
15392        struct Swap {
15393            registry: Arc<Registry>,
15394            forwarding: Arc<ForwardingTable>,
15395            handler: ControlHandler,
15396            incumbent_ctx: RouteCtx,
15397            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15398            candidate_ctx: RouteCtx,
15399            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15400        }
15401
15402        async fn swap_with_incumbent() -> Swap {
15403            let registry = Arc::new(Registry::default());
15404            let forwarding = Arc::new(ForwardingTable::default());
15405            let handler =
15406                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15407            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15408            hello_via_sink(
15409                &handler,
15410                &incumbent_ctx,
15411                &mut incumbent_rx,
15412                hello_frame("aft", PROTOCOL_VERSION, 7),
15413            )
15414            .await;
15415            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15416            Swap {
15417                registry,
15418                forwarding,
15419                handler,
15420                incumbent_ctx,
15421                incumbent_rx,
15422                candidate_ctx,
15423                candidate_rx,
15424            }
15425        }
15426
15427        fn register_candidate(swap: &Swap, ready: Option<bool>) {
15428            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15429            candidate_manifest.ready = ready;
15430            let registration = swap
15431                .registry
15432                .register_candidate_with_control_ops(
15433                    candidate_manifest,
15434                    PROTOCOL_VERSION,
15435                    CANDIDATE,
15436                    module_baseline_control_ops(),
15437                )
15438                .unwrap();
15439            swap.forwarding
15440                .register_candidate_module_connection(
15441                    CANDIDATE,
15442                    "aft".to_string(),
15443                    PROTOCOL_VERSION,
15444                    manifest_concurrency(&registration.manifest),
15445                    swap.candidate_ctx.egress.clone(),
15446                )
15447                .unwrap();
15448        }
15449
15450        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15451            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15452            swap.registry.promote_candidate("aft").unwrap().unwrap();
15453            cutover.incumbent.unwrap()
15454        }
15455
15456        /// A swap candidate's HELLO reserves its candidate registry entry
15457        /// before installing its candidate forwarding endpoint. The swap
15458        /// supervisor promotes a ready candidate as soon as it reads it as
15459        /// registered, and promotion cuts forwarding over to the candidate
15460        /// endpoint, so the lookup the supervisor reads
15461        /// (`Registry::published_registration`) must not return the candidate
15462        /// until that endpoint exists. Otherwise the cutover finds no candidate
15463        /// endpoint and the swap fails as if the candidate had disconnected.
15464        #[tokio::test]
15465        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15466            let mut swap = swap_with_incumbent().await;
15467            let hold = Arc::new(std::sync::Barrier::new(2));
15468            let handler = swap
15469                .handler
15470                .clone()
15471                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15472            handler
15473                .supervisor
15474                .open_swap("aft", "candidate-nonce".to_string());
15475            let hello = {
15476                let handler = handler.clone();
15477                let candidate_ctx = swap.candidate_ctx.clone();
15478                std::thread::spawn(move || {
15479                    handler
15480                        .handle_hello(
15481                            candidate_ctx.connection_id,
15482                            Some(candidate_ctx.egress.clone()),
15483                            hello_frame_with_nonce(
15484                                "aft",
15485                                PROTOCOL_VERSION,
15486                                8,
15487                                Some("candidate-nonce"),
15488                            ),
15489                        )
15490                        .unwrap()
15491                })
15492            };
15493
15494            hold.wait();
15495            // Parked: the candidate slot is claimed but not published, and
15496            // forwarding has no candidate endpoint.
15497            assert!(swap
15498                .registry
15499                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15500                .unwrap()
15501                .is_some());
15502            assert!(swap
15503                .registry
15504                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15505                .unwrap()
15506                .is_none());
15507            assert!(swap
15508                .forwarding
15509                .module_endpoint_for_connection(CANDIDATE)
15510                .unwrap()
15511                .is_none());
15512
15513            hold.wait();
15514            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15515            assert_eq!(
15516                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15517                FrameType::HelloAck
15518            );
15519            let published = swap
15520                .registry
15521                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15522                .unwrap()
15523                .expect("candidate is published once its HELLO completes");
15524            assert_eq!(published.connection_id, CANDIDATE);
15525            assert!(swap
15526                .forwarding
15527                .module_endpoint_for_connection(CANDIDATE)
15528                .unwrap()
15529                .is_some());
15530        }
15531
15532        fn keyed_total(counters: &Value, key: &str) -> u64 {
15533            counters[key]
15534                .as_object()
15535                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15536                .unwrap_or(0)
15537        }
15538
15539        #[tokio::test]
15540        async fn replacement_logs_old_end_and_new_admission_once() {
15541            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15542            let mut swap = swap_with_incumbent().await;
15543            swap.handler
15544                .supervisor
15545                .open_swap("aft", "candidate-nonce".to_string());
15546            hello_via_sink(
15547                &swap.handler,
15548                &swap.candidate_ctx,
15549                &mut swap.candidate_rx,
15550                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15551            )
15552            .await;
15553            cutover(&swap);
15554            swap.handler.cleanup_connection(INCUMBENT).unwrap();
15555
15556            let captured = crate::router::test_log::captured_logs(&logs);
15557            println!("captured replacement registry logs:\n{captured}");
15558            let new_admissions: Vec<_> = captured
15559                .lines()
15560                .filter(|line| {
15561                    line.contains("connection_id=40")
15562                        && line.contains("swap candidate registered; not routable until cutover")
15563                })
15564                .collect();
15565            assert_eq!(
15566                new_admissions.len(),
15567                1,
15568                "unexpected admission logs: {captured}"
15569            );
15570            assert!(new_admissions[0].contains("module_id=aft"));
15571            assert!(new_admissions[0].contains("connection_id=40"));
15572            assert!(
15573                new_admissions[0].contains("swap candidate registered; not routable until cutover")
15574            );
15575            assert!(new_admissions[0].contains("ready=true"));
15576
15577            let old_admissions: Vec<_> = captured
15578                .lines()
15579                .filter(|line| {
15580                    line.contains("module registered module_id=aft ")
15581                        && line.contains("connection_id=30")
15582                })
15583                .collect();
15584            assert_eq!(
15585                old_admissions.len(),
15586                1,
15587                "unexpected incumbent admission: {captured}"
15588            );
15589
15590            let old_ends: Vec<_> = captured
15591                .lines()
15592                .filter(|line| {
15593                    line.contains("connection_id=30") && line.contains("module registration ended")
15594                })
15595                .collect();
15596            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15597            assert!(old_ends[0].contains("module_id=aft"));
15598            assert!(old_ends[0].contains("connection_id=30"));
15599            assert!(old_ends[0].contains("reason=replaced"));
15600            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15601            assert_eq!(
15602                captured
15603                    .lines()
15604                    .filter(|line| line.contains("module registration promoted"))
15605                    .count(),
15606                1,
15607                "unexpected promotion logs: {captured}"
15608            );
15609        }
15610
15611        /// An ack from the incumbent for a bind it was sent before cutover,
15612        /// arriving before the incumbent is drained. The incumbent is the live
15613        /// connection carrying every other client's routes, so the ack must
15614        /// not end it: the waiting client is told to retry, the reservation is
15615        /// given back, and the incumbent is told to drop just that binding.
15616        #[tokio::test]
15617        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15618            let mut swap = swap_with_incumbent().await;
15619            let handler = swap.handler.clone();
15620
15621            // A co-tenant route, bound on the incumbent before the swap.
15622            let cotenant = ConnectionId::new(31);
15623            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15624            let (cotenant_task, cotenant_bind) = relay_route_open(
15625                &handler,
15626                cotenant,
15627                &cotenant_ctx.egress,
15628                &mut swap.incumbent_rx,
15629                100,
15630                "aft",
15631                "swap-cotenant",
15632            )
15633            .await;
15634            handler
15635                .handle_control_frame(
15636                    &swap.incumbent_ctx,
15637                    route_bind_ack(cotenant_bind.header.corr),
15638                )
15639                .await
15640                .unwrap();
15641            assert!(cotenant_task.await.unwrap().is_empty());
15642            let (cotenant_channel, cotenant_epoch) =
15643                published_route(&cotenant_rx.recv().await.unwrap());
15644
15645            // A second route.open, relayed to the incumbent and not yet acked.
15646            let caller = ConnectionId::new(32);
15647            let (caller_ctx, mut caller_rx) = route_ctx(caller);
15648            let (caller_task, caller_bind) = relay_route_open(
15649                &handler,
15650                caller,
15651                &caller_ctx.egress,
15652                &mut swap.incumbent_rx,
15653                101,
15654                "aft",
15655                "swap-caller",
15656            )
15657            .await;
15658            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15659
15660            register_candidate(&swap, None);
15661            cutover(&swap);
15662
15663            // The incumbent acks after promotion and before any drain.
15664            let ack = handler
15665                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15666                .await;
15667            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15668            if module_loop_error.is_some() {
15669                // What the connection loop does with an untranslated router
15670                // error: end the connection, releasing every route on it.
15671                handler.cleanup_connection(INCUMBENT).unwrap();
15672            }
15673
15674            // 1. The incumbent's other routes survive.
15675            assert!(
15676                cotenant_rx.try_recv().is_err(),
15677                "the co-tenant route on the incumbent was torn down by one late ack: \
15678                 {module_loop_error:?}"
15679            );
15680            assert!(matches!(
15681                swap.forwarding
15682                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15683                    .unwrap(),
15684                DataRoute::Client(DataRouteState::Bound(_))
15685            ));
15686            assert_eq!(module_loop_error, None);
15687            assert!(swap
15688                .registry
15689                .get_module_by_connection(INCUMBENT)
15690                .unwrap()
15691                .is_some());
15692
15693            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
15694            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15695                .await
15696                .expect("the incumbent is told to drop the abandoned binding")
15697                .unwrap()
15698                .frame;
15699            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15700            assert_eq!(goodbye.header.channel, abandoned_channel);
15701            assert_eq!(goodbye.header.epoch, abandoned_epoch);
15702            assert!(swap.incumbent_rx.try_recv().is_err());
15703
15704            // 3. The waiting client gets a retryable refusal and no route.
15705            let response = caller_task.await.unwrap();
15706            assert_eq!(response.len(), 1);
15707            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15708            assert!(caller_rx.try_recv().is_err());
15709
15710            // 4. The reservation pair is given back, and the pending bind
15711            //    settled exactly once: one accepted open (the co-tenant) and one
15712            //    refused open (the caller), nothing counted twice.
15713            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15714            let counters = handler.counters().snapshot();
15715            assert_eq!(
15716                keyed_total(&counters, "route_open_accepted_by_principal"),
15717                1
15718            );
15719            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15720            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15721        }
15722
15723        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
15724        /// id would resolve to the promoted candidate and every new route.open
15725        /// would be refused as reloading, leaving neither process routable.
15726        #[tokio::test]
15727        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15728            let mut swap = swap_with_incumbent().await;
15729            register_candidate(&swap, None);
15730            let incumbent = cutover(&swap);
15731            swap.forwarding
15732                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15733                .unwrap()
15734                .expect("the incumbent is still registered");
15735
15736            let client = ConnectionId::new(33);
15737            let (client_ctx, mut client_rx) = route_ctx(client);
15738            let route_handler = swap.handler.clone();
15739            let open_ctx = RouteCtx {
15740                connection_id: client,
15741                egress: client_ctx.egress.clone(),
15742            };
15743            let mut route_task = tokio::spawn(async move {
15744                route_handler
15745                    .handle_control_frame(
15746                        &open_ctx,
15747                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15748                    )
15749                    .await
15750                    .unwrap()
15751            });
15752            let bind = tokio::select! {
15753                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15754                response = &mut route_task => {
15755                    let response = response.unwrap();
15756                    panic!(
15757                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
15758                        parse_error(&response[0])["code"]
15759                    );
15760                }
15761            };
15762            swap.handler
15763                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15764                .await
15765                .unwrap();
15766            assert!(route_task.await.unwrap().is_empty());
15767            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15768            match swap
15769                .forwarding
15770                .lookup_data_route(client, channel, epoch)
15771                .unwrap()
15772            {
15773                DataRoute::Client(DataRouteState::Bound(route)) => {
15774                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15775                }
15776                other => panic!("expected a bound route on the candidate, got {other:?}"),
15777            }
15778            assert!(swap.incumbent_rx.try_recv().is_err());
15779        }
15780
15781        /// A candidate declares itself ready with `catalog.update` on its own
15782        /// connection. If the connection-keyed registry lookups searched only the
15783        /// active slot, this would answer `not_registered` and the candidate
15784        /// would never become ready.
15785        #[tokio::test]
15786        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15787            let swap = swap_with_incumbent().await;
15788            register_candidate(&swap, Some(false));
15789            let update = Frame::build(
15790                FrameType::Request,
15791                control_flags(),
15792                0,
15793                0,
15794                55,
15795                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15796                    provides: manifest("aft", PROTOCOL_VERSION).provides,
15797                    capabilities: None,
15798                    ready: Some(true),
15799                })
15800                .unwrap(),
15801            )
15802            .unwrap();
15803
15804            let replies = swap
15805                .handler
15806                .handle_control_frame(&swap.candidate_ctx, update)
15807                .await
15808                .unwrap();
15809
15810            assert_eq!(replies.len(), 1);
15811            assert_eq!(
15812                replies[0].header.ty,
15813                FrameType::Response,
15814                "candidate catalog.update was refused: {:?}",
15815                serde_json::from_slice::<Value>(&replies[0].body).ok()
15816            );
15817            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15818            assert_eq!(
15819                swap.registry
15820                    .get_module("aft")
15821                    .unwrap()
15822                    .unwrap()
15823                    .connection_id,
15824                INCUMBENT
15825            );
15826        }
15827    }
15828
15829    /// The HELLO gate while the supervisor has a swap open: only the nonce it
15830    /// minted for the candidate admits a second process, into the candidate
15831    /// slot, and that check runs ahead of the reserved-module gate.
15832    mod swap_admission {
15833        use super::*;
15834
15835        const INCUMBENT_NONCE: &str = "incumbent-nonce";
15836        const CANDIDATE_NONCE: &str = "candidate-nonce";
15837
15838        fn handler_with_incumbent(
15839            module_id: &str,
15840            reserved: bool,
15841        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15842            let registry = Arc::new(Registry::default());
15843            let supervisor = SupervisorHandle::new();
15844            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15845            if reserved {
15846                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15847            }
15848            let handler =
15849                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
15850            let incumbent = handler
15851                .handle_control(
15852                    ConnectionId::new(1),
15853                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15854                )
15855                .unwrap();
15856            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15857            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15858            (registry, supervisor, handler)
15859        }
15860
15861        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
15862        /// admits every nonce, so while a swap is open the swap gate is the only
15863        /// thing between a key-holder and the candidate slot. A nonce the
15864        /// supervisor did not mint, or none at all, is refused, and neither the
15865        /// incumbent's registration nor the candidate slot moves.
15866        #[test]
15867        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15868            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15869
15870            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15871                let replies = handler
15872                    .handle_control(
15873                        ConnectionId::new(connection),
15874                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15875                    )
15876                    .unwrap();
15877                assert_eq!(replies[0].header.ty, FrameType::Error);
15878                assert_eq!(
15879                    parse_error(&replies[0])["code"],
15880                    "swap_token_invalid",
15881                    "nonce {nonce:?}"
15882                );
15883            }
15884            assert!(registry.get_candidate("aft").unwrap().is_none());
15885            assert_eq!(
15886                registry.get_module("aft").unwrap().unwrap().connection_id,
15887                ConnectionId::new(1)
15888            );
15889
15890            // Control: the minted token is admitted, into the candidate slot,
15891            // and only once.
15892            let admitted = handler
15893                .handle_control(
15894                    ConnectionId::new(4),
15895                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15896                )
15897                .unwrap();
15898            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15899            assert_eq!(
15900                registry
15901                    .get_candidate("aft")
15902                    .unwrap()
15903                    .unwrap()
15904                    .connection_id,
15905                ConnectionId::new(4)
15906            );
15907            assert_eq!(
15908                registry.get_module("aft").unwrap().unwrap().connection_id,
15909                ConnectionId::new(1),
15910                "the candidate must not take the active slot"
15911            );
15912            let replayed = handler
15913                .handle_control(
15914                    ConnectionId::new(5),
15915                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15916                )
15917                .unwrap();
15918            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15919
15920            // The case only this gate covers: the incumbent has died mid-swap,
15921            // so its duplicate refusal is gone too, and without the gate a
15922            // key-holder would take the id's ACTIVE slot.
15923            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15924            let squatter = handler
15925                .handle_control(
15926                    ConnectionId::new(6),
15927                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15928                )
15929                .unwrap();
15930            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15931            assert!(
15932                registry.get_module("aft").unwrap().is_none(),
15933                "a squatter took the active slot of an id being swapped"
15934            );
15935        }
15936
15937        /// Design mutation arm (iii). A reserved module's candidate presents a
15938        /// nonce the reserved gate has never seen (that gate holds the
15939        /// incumbent's), so the swap gate must run first or the candidate is
15940        /// refused `reserved_module` and a reserved module can never be swapped.
15941        #[test]
15942        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15943            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15944
15945            let replies = handler
15946                .handle_control(
15947                    ConnectionId::new(2),
15948                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15949                )
15950                .unwrap();
15951
15952            assert_eq!(
15953                replies[0].header.ty,
15954                FrameType::HelloAck,
15955                "reserved candidate refused: {:?}",
15956                serde_json::from_slice::<Value>(&replies[0].body).ok()
15957            );
15958            assert_eq!(
15959                registry
15960                    .get_candidate("vault")
15961                    .unwrap()
15962                    .unwrap()
15963                    .connection_id,
15964                ConnectionId::new(2)
15965            );
15966        }
15967
15968        /// With no swap open the gate is inert: the incumbent's reserved gate
15969        /// and duplicate refusal behave exactly as before.
15970        #[test]
15971        fn without_an_open_swap_the_ordinary_gates_decide() {
15972            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15973            supervisor.close_swap("vault");
15974
15975            let candidate = handler
15976                .handle_control(
15977                    ConnectionId::new(2),
15978                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15979                )
15980                .unwrap();
15981            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15982            let duplicate = handler
15983                .handle_control(
15984                    ConnectionId::new(3),
15985                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15986                )
15987                .unwrap();
15988            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15989            assert!(registry.get_candidate("vault").unwrap().is_none());
15990        }
15991    }
15992
15993    /// `scope.sync` and `scope.describe` through the real control handler: who
15994    /// may sync is decided by the registration and launch nonce of the module
15995    /// connection, never by the request body.
15996    mod scopes {
15997        mod idle_tests {
15998            use super::*;
15999            include!("scope_idle_tests.rs");
16000        }
16001        mod route_open_timings {
16002            use super::*;
16003            include!("route_open_timing_tests.rs");
16004        }
16005        mod route_open_refusal_log {
16006            use super::*;
16007            include!("route_open_refusal_log_tests.rs");
16008        }
16009        mod activity {
16010            include!("activity_scope_tests.rs");
16011        }
16012        mod delta {
16013            include!("scope_handler_delta_tests.rs");
16014        }
16015        mod call_modes {
16016            include!("scope_call_mode_tests.rs");
16017        }
16018        #[tokio::test]
16019        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16020            let rig = rig().await;
16021            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16022                "s".into(),
16023                1,
16024                "tool".into(),
16025            ))
16026            .unwrap();
16027            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16028            let (direct, _rx) = wide_ctx(999);
16029            for (ctx, expected) in [
16030                (&rig.owner, FrameType::Response),
16031                (&direct, FrameType::Error),
16032            ] {
16033                let frame =
16034                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16035                        .unwrap();
16036                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16037                assert_eq!(replies[0].header.ty, expected);
16038                if expected == FrameType::Response {
16039                    let reply: subc_control::ScopeServedByReply =
16040                        serde_json::from_slice(&replies[0].body).unwrap();
16041                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16042                } else {
16043                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16044                }
16045            }
16046            let reply = call(
16047                &rig.handler,
16048                &rig.owner,
16049                &ModuleControlRequestFromModule::ScopeDescribe {
16050                    owner: Principal::Reserved {
16051                        module_id: OWNER.into(),
16052                    },
16053                    scope_ref: "s".into(),
16054                },
16055            )
16056            .await;
16057            assert!(matches!(
16058                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16059                ModuleControlResponseToModule::ScopeDescribe { .. }
16060            ));
16061        }
16062
16063        use subc_protocol::scope::{
16064            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16065            ScopeStatus,
16066        };
16067
16068        use super::*;
16069
16070        const OWNER: &str = "prefrontal-core";
16071
16072        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16073            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16074        }
16075
16076        async fn call(
16077            handler: &ControlHandler,
16078            ctx: &RouteCtx,
16079            request: &ModuleControlRequestFromModule,
16080        ) -> Frame {
16081            let body = serde_json::to_vec(request).unwrap();
16082            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16083            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16084            assert_eq!(replies.len(), 1, "{replies:?}");
16085            replies.pop().unwrap()
16086        }
16087
16088        async fn sync(
16089            handler: &ControlHandler,
16090            ctx: &RouteCtx,
16091            generation: u64,
16092            scopes: Vec<ScopeRecord>,
16093        ) -> Result<ModuleControlResponseToModule, String> {
16094            let reply = call(
16095                handler,
16096                ctx,
16097                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16098            )
16099            .await;
16100            match reply.header.ty {
16101                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16102                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16103            }
16104        }
16105
16106        async fn describe(
16107            handler: &ControlHandler,
16108            ctx: &RouteCtx,
16109            owner: &str,
16110            scope_ref: &str,
16111        ) -> ModuleControlResponseToModule {
16112            let reply = call(
16113                handler,
16114                ctx,
16115                &ModuleControlRequestFromModule::ScopeDescribe {
16116                    owner: Principal::Reserved {
16117                        module_id: owner.to_string(),
16118                    },
16119                    scope_ref: scope_ref.to_string(),
16120                },
16121            )
16122            .await;
16123            assert_eq!(
16124                reply.header.ty,
16125                FrameType::Response,
16126                "{:?}",
16127                parse_error(&reply)
16128            );
16129            serde_json::from_slice(&reply.body).unwrap()
16130        }
16131
16132        #[tokio::test]
16133        async fn scope_describe_never_echoes_bind_proof_or_hash() {
16134            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16135            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16136            let mut rig = rig().await;
16137            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16138                .await;
16139            let _bound = rig
16140                .bound(
16141                    Some(AFT),
16142                    PLEXUS,
16143                    Some(
16144                        rig.selector("s", Some(1))
16145                            .with_bind_proof(Some(PROOF.into())),
16146                    ),
16147                )
16148                .await;
16149            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16150            let output = serde_json::to_string(&reply).unwrap();
16151            assert!(output.contains("\"status\":\"live\""));
16152            assert!(!output.contains(PROOF));
16153            assert!(!output.contains(HASH));
16154        }
16155
16156        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
16157        async fn module(
16158            handler: &ControlHandler,
16159            connection: u64,
16160            module_id: &str,
16161            nonce: Option<&str>,
16162        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16163            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16164            hello_via_sink(
16165                handler,
16166                &ctx,
16167                &mut rx,
16168                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16169            )
16170            .await;
16171            (ctx, rx)
16172        }
16173
16174        /// `direct` and every other client connection has no registration, so
16175        /// it can neither sync nor own a scope.
16176        #[tokio::test]
16177        async fn a_client_connection_cannot_sync_or_describe() {
16178            let handler = ControlHandler::new(Arc::new(Registry::default()));
16179            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16180            for request in [
16181                ModuleControlRequestFromModule::ScopeSync {
16182                    generation: 1,
16183                    scopes: vec![head("s", 1)],
16184                },
16185                ModuleControlRequestFromModule::ScopeDescribe {
16186                    owner: Principal::Direct,
16187                    scope_ref: "s".to_string(),
16188                },
16189            ] {
16190                let reply = call(&handler, &ctx, &request).await;
16191                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16192            }
16193            assert!(
16194                !handler
16195                    .scopes
16196                    .read()
16197                    .unwrap()
16198                    .describe(
16199                        &Principal::Reserved {
16200                            module_id: OWNER.to_string()
16201                        },
16202                        "s"
16203                    )
16204                    .owner_synced
16205            );
16206        }
16207
16208        /// A module the supervisor did not spawn registers without a launch
16209        /// nonce, so it is never an owner's current launch.
16210        #[tokio::test]
16211        async fn a_module_without_a_supervised_launch_cannot_sync() {
16212            let handler = ControlHandler::new(Arc::new(Registry::default()));
16213            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16214            assert_eq!(
16215                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16216                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16217            );
16218        }
16219
16220        #[tokio::test]
16221        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16222            let supervisor = SupervisorHandle::new();
16223            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16224            let handler = ControlHandler::new(Arc::new(Registry::default()))
16225                .with_supervisor(supervisor.clone());
16226            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16227            sync(&handler, &incumbent, 1, vec![head("s", 1)])
16228                .await
16229                .expect("the current launch syncs");
16230
16231            // A swap candidate registers with the swap token and is refused
16232            // while the incumbent keeps syncing.
16233            supervisor.open_swap(OWNER, "n2".to_string());
16234            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16235            assert_eq!(
16236                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16237                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16238            );
16239            sync(&handler, &incumbent, 2, vec![head("s", 1)])
16240                .await
16241                .expect("the serving owner syncs during the swap");
16242
16243            // The swap fails and is rolled back. The candidate never held sync
16244            // authority, and still cannot sync.
16245            supervisor.close_swap(OWNER);
16246            assert_eq!(
16247                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16248                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16249            );
16250            sync(&handler, &incumbent, 3, vec![head("s", 1)])
16251                .await
16252                .expect("the serving owner syncs after the rollback");
16253            handler.cleanup_connection(candidate.connection_id).unwrap();
16254
16255            // A swap that cuts over. Promotion records the candidate's nonce as
16256            // the module's spawn nonce, which is what `set_spawn_nonce` does
16257            // here; the promoted connection then takes authority at any
16258            // generation and the superseded incumbent is refused.
16259            supervisor.open_swap(OWNER, "n3".to_string());
16260            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16261            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16262            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16263                .await
16264                .expect("the promoted launch takes authority");
16265            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16266                panic!("unexpected reply {reply:?}");
16267            };
16268            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16269            assert_eq!(
16270                sync(&handler, &incumbent, 4, Vec::new()).await,
16271                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16272            );
16273        }
16274
16275        /// Authority dies with its connection: the cleanup path releases it,
16276        /// so the owner's next connection takes it at any generation.
16277        #[tokio::test]
16278        async fn closing_the_authority_connection_frees_sync_authority() {
16279            let supervisor = SupervisorHandle::new();
16280            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16281            let handler = ControlHandler::new(Arc::new(Registry::default()))
16282                .with_supervisor(supervisor.clone());
16283            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16284            sync(&handler, &first, 10, vec![head("s", 1)])
16285                .await
16286                .unwrap();
16287            handler.cleanup_connection(first.connection_id).unwrap();
16288
16289            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16290            sync(&handler, &second, 1, vec![head("s", 1)])
16291                .await
16292                .expect("the next connection takes the released authority");
16293        }
16294
16295        #[tokio::test]
16296        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16297            let supervisor = SupervisorHandle::new();
16298            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16299            let handler =
16300                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16301            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16302            sync(&handler, &first, 10, vec![head("s", 1)])
16303                .await
16304                .unwrap();
16305            handler
16306                .handle_control_frame(
16307                    &first,
16308                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16309                )
16310                .await
16311                .unwrap();
16312            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16313            sync(&handler, &second, 1, vec![head("s", 1)])
16314                .await
16315                .expect("GOODBYE releases authority even if the old socket remains open");
16316        }
16317
16318        #[tokio::test]
16319        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16320            let registry = Arc::new(Registry::default());
16321            let supervisor_handle = SupervisorHandle::new();
16322            let supervisor =
16323                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16324                    .with_handle(supervisor_handle.clone())
16325                    .with_daemon_incarnation("incarnation-7".to_string());
16326            // Configured with enabled: false, so the supervisor lists the
16327            // module without spawning a process for it.
16328            supervisor
16329                .supervise_configured(
16330                    ModuleSpec {
16331                        module_id: OWNER.to_string(),
16332                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16333                        args: Vec::new(),
16334                        env: Vec::new(),
16335                        reserved: false,
16336                        reserved_prefixes: Vec::new(),
16337                        protocol: ModuleProtocol::Subc,
16338                        overlap: Default::default(),
16339                    },
16340                    false,
16341                )
16342                .unwrap();
16343            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16344            let handler =
16345                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
16346            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16347
16348            // Configured but not yet synced: a reader waits for the owner.
16349            let ModuleControlResponseToModule::ScopeDescribe {
16350                status,
16351                daemon_incarnation,
16352                owner_synced,
16353                owner_configured,
16354                scope,
16355                ..
16356            } = describe(&handler, &reader, OWNER, "s").await
16357            else {
16358                panic!("not a describe reply");
16359            };
16360            assert_eq!(status, ScopeStatus::NotLive);
16361            assert_eq!(daemon_incarnation, "incarnation-7");
16362            assert!(!owner_synced);
16363            assert!(owner_configured);
16364            assert!(scope.is_none());
16365
16366            // Not a supervised module: the owner will never sync, and a reader
16367            // refuses rather than waits.
16368            let ModuleControlResponseToModule::ScopeDescribe {
16369                status,
16370                owner_configured,
16371                ..
16372            } = describe(&handler, &reader, "ghost", "s").await
16373            else {
16374                panic!("not a describe reply");
16375            };
16376            assert_eq!(status, ScopeStatus::NotLive);
16377            assert!(!owner_configured);
16378
16379            // Live, with the stamp fields and the computed owner_authorized.
16380            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16381            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16382            let ModuleControlResponseToModule::ScopeDescribe {
16383                status,
16384                scope_epoch,
16385                owner_synced,
16386                scope,
16387                ..
16388            } = describe(&handler, &reader, OWNER, "s").await
16389            else {
16390                panic!("not a describe reply");
16391            };
16392            assert_eq!(status, ScopeStatus::Live);
16393            assert_eq!(scope_epoch, Some(4));
16394            assert!(owner_synced);
16395            let stamp = scope.expect("a live scope carries its stamp");
16396            assert!(
16397                stamp.owner_authorized,
16398                "prefrontal-core is the default authority"
16399            );
16400            assert_eq!(stamp.kind, ScopeKind::Head);
16401        }
16402
16403        #[tokio::test]
16404        async fn scope_authority_owners_decides_owner_authorized() {
16405            let supervisor = SupervisorHandle::new();
16406            supervisor.set_spawn_nonce("broca", "b1".to_string());
16407            let handler = ControlHandler::new(Arc::new(Registry::default()))
16408                .with_supervisor(supervisor)
16409                .with_scope_authority_owners(vec!["broca".to_string()]);
16410            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16411            let mut gated = head("s", 1);
16412            gated.attributes.agent_id = Some("agent".to_string());
16413            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16414            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16415                describe(&handler, &broca, "broca", "s").await
16416            else {
16417                panic!("not a describe reply");
16418            };
16419            assert!(scope.unwrap().owner_authorized);
16420        }
16421
16422        /// With route admission, the stamp, the commit re-check and drains in
16423        /// place, the feature is advertised: the module ops in HELLO_ACK, and
16424        /// `scopes/v1` in HELLO_ACK and `server.describe`.
16425        #[tokio::test]
16426        async fn scope_ops_and_the_scopes_capability_are_advertised() {
16427            let handler = ControlHandler::new(Arc::new(Registry::default()));
16428            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16429            let ack = hello_via_sink(
16430                &handler,
16431                &ctx,
16432                &mut rx,
16433                hello_frame("m", PROTOCOL_VERSION, 1),
16434            )
16435            .await;
16436            let ack = parse_ack(&ack);
16437            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16438                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16439            }
16440            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16441
16442            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16443            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16444            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16445            let reply = handler
16446                .handle_control_frame(&client, frame)
16447                .await
16448                .unwrap()
16449                .pop()
16450                .unwrap();
16451            let ClientControlResponse::ServerDescribe { capabilities, .. } =
16452                serde_json::from_slice(&reply.body).unwrap()
16453            else {
16454                panic!("not a server.describe reply");
16455            };
16456            assert!(
16457                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16458                "{capabilities:?}"
16459            );
16460        }
16461
16462        // ---- route admission, stamps, commit re-check and drains ----------
16463
16464        const PLEXUS: &str = "plexus";
16465        const OTHER: &str = "other";
16466        const AFT: &str = "aft";
16467        const BROCA: &str = "broca";
16468        const MAGIC: &str = "magic-context";
16469
16470        fn nonce(module_id: &str) -> String {
16471            format!("nonce-{module_id}")
16472        }
16473
16474        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16475            let (tx, rx) = mpsc::channel(64);
16476            (
16477                RouteCtx {
16478                    connection_id: ConnectionId::new(connection),
16479                    egress: FrameSink::new(tx),
16480                },
16481                rx,
16482            )
16483        }
16484
16485        /// A daemon with a configured owner (prefrontal-core) registered on its
16486        /// own module connection, two routable targets (plexus, other), and
16487        /// launch nonces minted for the modules that open routes as carriers.
16488        struct Rig {
16489            handler: ControlHandler,
16490            forwarding: Arc<ForwardingTable>,
16491            owner: RouteCtx,
16492            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16493            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16494            generation: u64,
16495            next_connection: u64,
16496            _supervisor: Supervisor,
16497        }
16498
16499        async fn rig() -> Rig {
16500            rig_with_flow_support(true).await
16501        }
16502
16503        async fn rig_with_flow_support(flow_support: bool) -> Rig {
16504            rig_with_scope_capabilities(flow_support, false).await
16505        }
16506
16507        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16508            let mut provides = Vec::new();
16509            if flow_support {
16510                provides.push("flow-scopes/v1");
16511            }
16512            if run_support {
16513                provides.push("agent-run-scopes/v1");
16514            }
16515            rig_with_target_capabilities(&provides).await
16516        }
16517
16518        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16519            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16520            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16521            // Decoder versions alone cannot admit scoped calls: only a capability
16522            // declaration promises the required run or flow behaviour.
16523            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16524            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16525            Frame::build(
16526                FrameType::Hello,
16527                control_flags(),
16528                0,
16529                0,
16530                connection,
16531                serde_json::to_vec(&body).unwrap(),
16532            )
16533            .unwrap()
16534        }
16535
16536        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16537            let registry = Arc::new(Registry::default());
16538            let forwarding = Arc::new(ForwardingTable::default());
16539            let supervisor_handle = SupervisorHandle::new();
16540            let supervisor =
16541                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16542                    .with_handle(supervisor_handle.clone());
16543            supervisor
16544                .supervise_configured(
16545                    ModuleSpec {
16546                        module_id: OWNER.to_string(),
16547                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16548                        args: Vec::new(),
16549                        env: Vec::new(),
16550                        reserved: false,
16551                        reserved_prefixes: Vec::new(),
16552                        protocol: ModuleProtocol::Subc,
16553                        overlap: Default::default(),
16554                    },
16555                    false,
16556                )
16557                .unwrap();
16558            for module_id in [OWNER, AFT, BROCA, MAGIC] {
16559                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16560            }
16561            let handler =
16562                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16563                    .with_supervisor(supervisor_handle);
16564            let (owner, mut owner_rx) = wide_ctx(1);
16565            hello_via_sink(
16566                &handler,
16567                &owner,
16568                &mut owner_rx,
16569                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16570            )
16571            .await;
16572            let mut modules = BTreeMap::new();
16573            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16574                let (ctx, mut rx) = wide_ctx(connection);
16575                let hello = hello_with_capabilities(module_id, connection, provides);
16576                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16577                modules.insert(module_id.to_string(), (ctx, rx));
16578            }
16579            Rig {
16580                handler,
16581                forwarding,
16582                owner,
16583                _owner_rx: owner_rx,
16584                modules,
16585                generation: 0,
16586                next_connection: 100,
16587                _supervisor: supervisor,
16588            }
16589        }
16590
16591        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16592            ScopeCarrier::new(Principal::Reserved {
16593                module_id: module_id.to_string(),
16594            })
16595            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16596        }
16597
16598        /// The scope most tests open under: aft carries to any module, broca
16599        /// only to plexus and other, and the owner delegates as agent-1.
16600        fn session(scope_epoch: u64) -> ScopeRecord {
16601            let mut record = head("s", scope_epoch);
16602            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16603            record.attributes.agent_id = Some("agent-1".to_string());
16604            record.attributes.delegates = true;
16605            record
16606        }
16607
16608        impl Rig {
16609            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16610                self.generation += 1;
16611                sync(&self.handler, &self.owner, self.generation, scopes)
16612                    .await
16613                    .expect("the owner's sync is accepted");
16614            }
16615
16616            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16617                ScopeSelector::new(
16618                    Principal::Reserved {
16619                        module_id: OWNER.to_string(),
16620                    },
16621                    scope_ref.to_string(),
16622                    scope_epoch,
16623                )
16624            }
16625
16626            fn open_frame(
16627                &mut self,
16628                opener: Option<&str>,
16629                target: &str,
16630                scope: Option<ScopeSelector>,
16631            ) -> (
16632                RouteCtx,
16633                mpsc::Receiver<crate::router::OutboundFrame>,
16634                Frame,
16635            ) {
16636                self.next_connection += 1;
16637                let (ctx, rx) = wide_ctx(self.next_connection);
16638                let root = unique_project_root("scoped-open");
16639                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16640                    target: RouteTarget::ToolProvider {
16641                        module_id: target.to_string(),
16642                    },
16643                    identity: BindIdentity::new(
16644                        root.path().to_path_buf(),
16645                        "unit".to_string(),
16646                        "session".to_string(),
16647                    ),
16648                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
16649                        module_id: module_id.to_string(),
16650                        launch_nonce: nonce(module_id),
16651                    }),
16652                    consumer_capabilities: None,
16653                    role_versions: None,
16654                    admission_facts: None,
16655                    scope,
16656                })
16657                .unwrap();
16658                let frame = Frame::build(
16659                    FrameType::Request,
16660                    control_flags(),
16661                    0,
16662                    0,
16663                    self.next_connection,
16664                    body,
16665                )
16666                .unwrap();
16667                (ctx, rx, frame)
16668            }
16669
16670            /// Open and expect a refusal before anything is relayed.
16671            async fn refused(
16672                &mut self,
16673                opener: Option<&str>,
16674                target: &str,
16675                scope: Option<ScopeSelector>,
16676            ) -> String {
16677                self.refusal_body(opener, target, scope).await["code"]
16678                    .as_str()
16679                    .unwrap()
16680                    .to_string()
16681            }
16682
16683            async fn refusal_body(
16684                &mut self,
16685                opener: Option<&str>,
16686                target: &str,
16687                scope: Option<ScopeSelector>,
16688            ) -> Value {
16689                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16690                let replies = tokio::time::timeout(
16691                    Duration::from_secs(2),
16692                    self.handler.handle_control_frame(&ctx, frame),
16693                )
16694                .await
16695                .expect("the open must be refused before waiting for a bind ack")
16696                .unwrap();
16697                assert_eq!(replies.len(), 1, "{replies:?}");
16698                assert_eq!(replies[0].header.ty, FrameType::Error);
16699                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16700                assert!(
16701                    module_rx.try_recv().is_err(),
16702                    "a refused open relays nothing"
16703                );
16704                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16705                parse_error(&replies[0])
16706            }
16707
16708            /// Start an open and return its task and the bind the target got.
16709            async fn relayed(
16710                &mut self,
16711                opener: Option<&str>,
16712                target: &str,
16713                scope: Option<ScopeSelector>,
16714            ) -> Relayed {
16715                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16716                let handler = self.handler.clone();
16717                let task_ctx = ctx.clone();
16718                let task = tokio::spawn(async move {
16719                    handler
16720                        .handle_control_frame(&task_ctx, frame)
16721                        .await
16722                        .unwrap()
16723                });
16724                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16725                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16726                    .await
16727                    .expect("the target receives the relayed route.bind")
16728                    .unwrap()
16729                    .frame;
16730                Relayed {
16731                    target: target.to_string(),
16732                    client: ctx,
16733                    client_rx: rx,
16734                    task,
16735                    bind,
16736                }
16737            }
16738
16739            async fn ack(&self, relayed: &Relayed) {
16740                let (module, _) = &self.modules[&relayed.target];
16741                self.handler
16742                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16743                    .await
16744                    .unwrap();
16745            }
16746
16747            /// Open, ack and return the bound route.
16748            async fn bound(
16749                &mut self,
16750                opener: Option<&str>,
16751                target: &str,
16752                scope: Option<ScopeSelector>,
16753            ) -> Bound {
16754                let relayed = self.relayed(opener, target, scope).await;
16755                self.ack(&relayed).await;
16756                let Relayed {
16757                    target,
16758                    client,
16759                    mut client_rx,
16760                    task,
16761                    bind,
16762                } = relayed;
16763                assert!(
16764                    task.await.unwrap().is_empty(),
16765                    "the open is answered by commit"
16766                );
16767                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16768                Bound {
16769                    target,
16770                    client,
16771                    client_rx,
16772                    channel,
16773                    epoch,
16774                    bind,
16775                }
16776            }
16777
16778            fn live(&self, route: &Bound) -> bool {
16779                matches!(
16780                    self.forwarding
16781                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16782                        .unwrap(),
16783                    DataRoute::Client(DataRouteState::Bound(_))
16784                )
16785            }
16786        }
16787
16788        struct Relayed {
16789            target: String,
16790            client: RouteCtx,
16791            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16792            task: tokio::task::JoinHandle<Vec<Frame>>,
16793            bind: Frame,
16794        }
16795
16796        struct Bound {
16797            target: String,
16798            client: RouteCtx,
16799            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16800            channel: u16,
16801            epoch: u32,
16802            bind: Frame,
16803        }
16804
16805        impl Bound {
16806            /// The reason of the `route.closed` this client was sent, after
16807            /// checking it also got a GOODBYE on exactly this route.
16808            fn closed_reason(&mut self) -> RouteCloseReason {
16809                let mut reason = None;
16810                let mut goodbye = false;
16811                while let Ok(outbound) = self.client_rx.try_recv() {
16812                    let frame = outbound.frame;
16813                    match frame.header.ty {
16814                        FrameType::Goodbye => {
16815                            assert_eq!(
16816                                (frame.header.channel, frame.header.epoch),
16817                                (self.channel, self.epoch)
16818                            );
16819                            goodbye = true;
16820                        }
16821                        FrameType::Push => {
16822                            let ClientControlPush::RouteClosed {
16823                                reason: r,
16824                                module_id,
16825                                ..
16826                            } = serde_json::from_slice(&frame.body).unwrap()
16827                            else {
16828                                panic!("unexpected push");
16829                            };
16830                            assert_eq!(module_id, self.target);
16831                            reason = Some(r);
16832                        }
16833                        other => panic!("unexpected frame {other:?}"),
16834                    }
16835                }
16836                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16837                reason.expect("the client is told why the route closed")
16838            }
16839
16840            fn untouched(&mut self) -> bool {
16841                self.client_rx.try_recv().is_err()
16842            }
16843
16844            fn stamp(&self) -> Option<ScopeStamp> {
16845                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16846                    ModuleControlRequest::RouteBind { scope, .. } => scope,
16847                    other => panic!("expected a route.bind, got {other:?}"),
16848                }
16849            }
16850        }
16851
16852        #[tokio::test]
16853        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16854        ) {
16855            let mut rig = rig().await;
16856            let mut record = session(1);
16857            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16858            record.child_owners = vec![Principal::Reserved {
16859                module_id: MAGIC.to_string(),
16860            }];
16861            rig.sync(vec![record]).await;
16862            let scope = || Some(rig_selector("s", Some(1)));
16863
16864            // Admitted: the owner, a bare carrier to any module, a targeted
16865            // carrier to its listed module.
16866            rig.bound(Some(OWNER), PLEXUS, scope()).await;
16867            rig.bound(Some(AFT), OTHER, scope()).await;
16868            rig.bound(Some(BROCA), PLEXUS, scope()).await;
16869
16870            // Refused scope_not_carrier: a targeted carrier to an unlisted
16871            // module, a module that is not listed at all (a child owner is not
16872            // a carrier), and a direct key-holder.
16873            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16874                assert_eq!(
16875                    rig.refused(opener, target, scope()).await,
16876                    error_codes::SCOPE_NOT_CARRIER,
16877                    "{opener:?} -> {target}"
16878                );
16879            }
16880        }
16881
16882        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16883            ScopeSelector::new(
16884                Principal::Reserved {
16885                    module_id: OWNER.to_string(),
16886                },
16887                scope_ref.to_string(),
16888                scope_epoch,
16889            )
16890        }
16891
16892        #[tokio::test]
16893        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16894            let mut rig = rig().await;
16895            rig.sync(vec![session(1)]).await;
16896            for opener in [OWNER, AFT] {
16897                assert_eq!(
16898                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16899                        .await,
16900                    error_codes::SCOPE_EPOCH_REQUIRED,
16901                    "{opener}"
16902                );
16903            }
16904        }
16905
16906        #[tokio::test]
16907        async fn admission_separates_not_synced_not_live_and_ended() {
16908            let mut rig = rig().await;
16909            // Before the configured owner's first sync: retryable.
16910            let code = rig
16911                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16912                .await;
16913            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16914            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16915
16916            // An owner that is not configured will never sync: terminal.
16917            let ghost = ScopeSelector::new(
16918                Principal::Reserved {
16919                    module_id: "ghost".to_string(),
16920                },
16921                "s".to_string(),
16922                Some(1),
16923            );
16924            assert_eq!(
16925                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16926                error_codes::SCOPE_NOT_LIVE
16927            );
16928
16929            rig.sync(vec![session(2)]).await;
16930            assert_eq!(
16931                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16932                    .await,
16933                error_codes::SCOPE_NOT_LIVE
16934            );
16935            for epoch in [1, 3] {
16936                assert_eq!(
16937                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16938                        .await,
16939                    error_codes::SCOPE_ENDED,
16940                    "epoch {epoch}"
16941                );
16942            }
16943            // Control: the live epoch is admitted.
16944            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16945                .await;
16946        }
16947
16948        #[tokio::test]
16949        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16950            let mut rig = rig().await;
16951            rig.sync(vec![session(1)]).await;
16952            let route = rig
16953                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16954                .await;
16955            let stamp = route.stamp().expect("a scoped bind carries the stamp");
16956            assert_eq!(stamp.scope_ref, "s");
16957            assert_eq!(stamp.scope_epoch, 1);
16958            assert_eq!(stamp.kind, ScopeKind::Head);
16959            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16960            assert!(stamp.attributes.delegates);
16961            assert!(stamp.owner_authorized);
16962            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16963            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16964
16965            // broca owns a scope of its own on its own module connection; it is
16966            // not in scope_authority_owners, so its stamp is not authorized.
16967            let (broca, mut broca_rx) = wide_ctx(50);
16968            hello_via_sink(
16969                &rig.handler,
16970                &broca,
16971                &mut broca_rx,
16972                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16973            )
16974            .await;
16975            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16976                .await
16977                .unwrap();
16978            let own = ScopeSelector::new(
16979                Principal::Reserved {
16980                    module_id: BROCA.to_string(),
16981                },
16982                "b".to_string(),
16983                Some(1),
16984            );
16985            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16986            assert!(!route.stamp().unwrap().owner_authorized);
16987        }
16988
16989        #[tokio::test]
16990        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16991            let mut rig = rig().await;
16992            let mut record = head("s", 1);
16993            record.carriers = vec![carrier(AFT, None)];
16994            let flow_id = "Flow:run-7/step_2!~";
16995            record.attributes.flow_id = Some(flow_id.to_string());
16996            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16997                .await
16998                .unwrap();
16999            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17000                panic!("not a sync reply");
17001            };
17002            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17003            let route = rig
17004                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17005                .await;
17006            assert!(rig.live(&route), "the stamped bind committed");
17007            let stamp = route.stamp().expect("a flow scope carries a stamp");
17008            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17009            assert_eq!(stamp.attributes.agent_id, None);
17010            assert!(!stamp.attributes.delegates);
17011            assert!(stamp.owner_authorized);
17012            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17013            assert_eq!(unscoped.stamp(), None);
17014        }
17015
17016        #[tokio::test]
17017        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17018            let mut rig = rig_with_flow_support(false).await;
17019            let mut record = session(1);
17020            record.attributes.flow_id = Some("flow:7".to_string());
17021            rig.sync(vec![record]).await;
17022            for opener in [OWNER, AFT] {
17023                let body = rig
17024                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17025                    .await;
17026                assert_eq!(body["code"], "target_flow_unsupported");
17027                let message = body["message"].as_str().unwrap();
17028                for required in [PLEXUS, "flow-scopes/v1"] {
17029                    assert!(message.contains(required), "{message}");
17030                }
17031            }
17032        }
17033
17034        #[tokio::test]
17035        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17036            let mut rig = rig_with_flow_support(true).await;
17037            let mut record = session(1);
17038            record.attributes.flow_id = Some("flow:7".to_string());
17039            rig.sync(vec![record]).await;
17040            for opener in [OWNER, AFT] {
17041                let route = rig
17042                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17043                    .await;
17044                assert!(rig.live(&route));
17045                assert_eq!(
17046                    route.stamp().unwrap().attributes.flow_id.as_deref(),
17047                    Some("flow:7")
17048                );
17049            }
17050        }
17051
17052        #[tokio::test]
17053        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17054            use std::future::Future;
17055
17056            let mut rig = rig_with_flow_support(true).await;
17057            let mut record = session(1);
17058            record.attributes.flow_id = Some("flow:7".to_string());
17059            rig.sync(vec![record]).await;
17060            let (client, mut client_rx, frame) =
17061                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17062            // Hold the route-open response permit so admission sees the first
17063            // target but relay reservation cannot capture an endpoint yet.
17064            for _ in 0..64 {
17065                client.egress.try_send(route_bind_ack(1)).unwrap();
17066            }
17067            let handler = rig.handler.clone();
17068            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17069            std::future::poll_fn(|cx| {
17070                assert!(open.as_mut().poll(cx).is_pending());
17071                std::task::Poll::Ready(())
17072            })
17073            .await;
17074            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17075
17076            let old_connection = rig.modules[PLEXUS].0.connection_id;
17077            rig.handler.cleanup_connection(old_connection).unwrap();
17078            let (replacement, mut replacement_rx) = wide_ctx(200);
17079            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17080            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17081            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17082            let hello = Frame::build(
17083                FrameType::Hello,
17084                control_flags(),
17085                0,
17086                0,
17087                200,
17088                serde_json::to_vec(&body).unwrap(),
17089            )
17090            .unwrap();
17091            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17092
17093            client_rx.try_recv().unwrap();
17094            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17095                .await
17096                .expect("the replacement is refused without waiting for a bind ack")
17097                .unwrap();
17098            assert_eq!(replies.len(), 1);
17099            let body = parse_error(&replies[0]);
17100            assert_eq!(body["code"], "target_flow_unsupported");
17101            for required in [PLEXUS, "flow-scopes/v1"] {
17102                assert!(body["message"].as_str().unwrap().contains(required));
17103            }
17104            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17105            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17106            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17107            assert!(rig
17108                .handler
17109                .registry
17110                .get_module_by_connection(replacement.connection_id)
17111                .unwrap()
17112                .is_some());
17113        }
17114
17115        #[tokio::test]
17116        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17117            use std::future::Future;
17118            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17119            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17120            const NEXT_HASH: &str =
17121                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17122            let mut rig = rig().await;
17123            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17124            record.carriers.push(ScopeCarrier::new(Principal::Direct));
17125            rig.sync(vec![record.clone()]).await;
17126            let (client, mut rx, frame) = rig.open_frame(
17127                None,
17128                PLEXUS,
17129                Some(
17130                    rig.selector("s", Some(1))
17131                        .with_bind_proof(Some(PROOF.into())),
17132                ),
17133            );
17134            // Hold egress reservation while the scope owner changes its bind-proof
17135            // hash, so a proof valid at admission must be checked again at relay.
17136            for _ in 0..64 {
17137                client.egress.try_send(route_bind_ack(1)).unwrap();
17138            }
17139            let handler = rig.handler.clone();
17140            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17141            std::future::poll_fn(|cx| {
17142                assert!(open.as_mut().poll(cx).is_pending());
17143                std::task::Poll::Ready(())
17144            })
17145            .await;
17146            record.bind_proof_sha256 = Some(NEXT_HASH.into());
17147            rig.sync(vec![record]).await;
17148            rx.try_recv().unwrap();
17149            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17150                .await
17151                .unwrap()
17152                .unwrap();
17153            assert_eq!(
17154                parse_error(&replies[0])["code"],
17155                error_codes::SCOPE_BIND_PROOF_MISMATCH
17156            );
17157            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17158            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17159            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17160        }
17161
17162        #[tokio::test]
17163        async fn scope_bind_proof_reserved_openers_ignore_proof() {
17164            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17165            let mut rig = rig().await;
17166            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17167                .await;
17168            for opener in [OWNER, AFT] {
17169                for proof in [None, Some("malformed+proof=")] {
17170                    let route = rig
17171                        .bound(
17172                            Some(opener),
17173                            PLEXUS,
17174                            Some(
17175                                rig.selector("s", Some(1))
17176                                    .with_bind_proof(proof.map(str::to_string)),
17177                            ),
17178                        )
17179                        .await;
17180                    assert!(rig.live(&route));
17181                }
17182            }
17183        }
17184
17185        #[tokio::test]
17186        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17187            use std::future::Future;
17188            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17189            const NEXT_HASH: &str =
17190                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17191            let mut rig = rig().await;
17192            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17193                .await;
17194            let (client, mut rx, frame) =
17195                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17196            for _ in 0..64 {
17197                client.egress.try_send(route_bind_ack(1)).unwrap();
17198            }
17199            let handler = rig.handler.clone();
17200            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17201            std::future::poll_fn(|cx| {
17202                assert!(open.as_mut().poll(cx).is_pending());
17203                std::task::Poll::Ready(())
17204            })
17205            .await;
17206            rig.sync(vec![
17207                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17208            ])
17209            .await;
17210            rx.try_recv().unwrap();
17211            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17212            let bind = std::future::poll_fn(|cx| {
17213                assert!(open.as_mut().poll(cx).is_pending());
17214                module_rx.poll_recv(cx)
17215            })
17216            .await
17217            .unwrap()
17218            .frame;
17219            handler
17220                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17221                .await
17222                .unwrap();
17223            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17224                .await
17225                .unwrap()
17226                .unwrap();
17227            assert_eq!(replies.len(), 1);
17228            // Proof rechecking allowed the relay. Independent tag settlement
17229            // still asks the opener to retry after any same-epoch content change.
17230            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17231            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17232            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17233            let route = rig
17234                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17235                .await;
17236            assert!(rig.live(&route));
17237        }
17238
17239        #[tokio::test]
17240        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17241        {
17242            let mut rig = rig_with_flow_support(false).await;
17243            rig.sync(vec![session(1)]).await;
17244            let route = rig
17245                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17246                .await;
17247            assert!(rig.live(&route));
17248            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17249            let mut record = session(1);
17250            record.attributes.flow_id = Some("flow:7".to_string());
17251            rig.sync(vec![record]).await;
17252            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17253            assert!(rig.live(&unscoped));
17254            assert_eq!(unscoped.stamp(), None);
17255        }
17256
17257        #[tokio::test]
17258        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17259            let mut rig = rig().await;
17260            let mut record = session(1);
17261            record.attributes.flow_id = Some("flow:7".to_string());
17262            rig.sync(vec![record.clone()]).await;
17263            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17264            let mut owner_route = rig
17265                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17266                .await;
17267            let mut carrier_route = rig
17268                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17269                .await;
17270            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17271            rig.sync(vec![record.clone()]).await;
17272            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17273            assert!(rig.live(&owner_route) && owner_route.untouched());
17274            assert!(rig.live(&carrier_route) && carrier_route.untouched());
17275
17276            record.attributes.flow_id = Some("flow:8".to_string());
17277            rig.sync(vec![record]).await;
17278            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17279            let before = before.unwrap();
17280            assert_eq!(after.scope_epoch, before.scope_epoch);
17281            assert!(after.version > before.version);
17282            for route in [&mut owner_route, &mut carrier_route] {
17283                assert!(!rig.live(route));
17284                assert_eq!(
17285                    route.closed_reason(),
17286                    RouteCloseReason::ScopeDelegationChanged
17287                );
17288            }
17289            assert!(rig.live(&unscoped) && unscoped.untouched());
17290            // Each provider also receives a GOODBYE for its drained route;
17291            // consume it before expecting the next route.bind on that sink.
17292            for target in [PLEXUS, OTHER] {
17293                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17294                let goodbye = module_rx
17295                    .try_recv()
17296                    .expect("the provider sees the drain")
17297                    .frame;
17298                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17299                assert!(module_rx.try_recv().is_err());
17300            }
17301            let rebound = rig
17302                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17303                .await;
17304            assert_eq!(
17305                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17306                Some("flow:8")
17307            );
17308        }
17309
17310        /// The owner's sync lands between admission and the module's ack. The
17311        /// open is refused by name, the module's other routes stay up, and the
17312        /// reserved pair is released. Changed content is retryable; an ended
17313        /// scope is not.
17314        #[tokio::test]
17315        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17316            let mut rig = rig().await;
17317            rig.sync(vec![session(1)]).await;
17318            let mut cotenant = rig
17319                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17320                .await;
17321
17322            let mut changed = session(1);
17323            changed.child_owners.push(Principal::Reserved {
17324                module_id: MAGIC.to_string(),
17325            });
17326            let mut ended = None;
17327            for (code, next) in [
17328                (error_codes::SCOPE_CHANGED, vec![changed]),
17329                (error_codes::SCOPE_ENDED, Vec::new()),
17330            ] {
17331                let relayed = rig
17332                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17333                    .await;
17334                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17335                ended = Some(next.is_empty());
17336                rig.sync(next).await;
17337                rig.ack(&relayed).await;
17338                let replies = relayed.task.await.unwrap();
17339                assert_eq!(replies.len(), 1, "{replies:?}");
17340                assert_eq!(parse_error(&replies[0])["code"], code);
17341                assert_eq!(
17342                    subc_protocol::error_codes::is_retryable_route_open(code),
17343                    code == error_codes::SCOPE_CHANGED
17344                );
17345                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17346                // The module is told to drop just the binding it created.
17347                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17348                // Collected, because ending the scope also closes the co-tenant
17349                // route, whose GOODBYE comes first.
17350                let mut goodbyes = Vec::new();
17351                while let Ok(outbound) = plexus_rx.try_recv() {
17352                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17353                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17354                }
17355                assert!(
17356                    goodbyes.contains(&(bind_channel, bind_epoch)),
17357                    "{goodbyes:?}"
17358                );
17359                assert!(rig
17360                    .handler
17361                    .registry
17362                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17363                    .unwrap()
17364                    .is_some());
17365            }
17366            assert_eq!(ended, Some(true));
17367            // The co-tenant stayed up through the change, and closed only when
17368            // the scope ended, by the drain rule rather than by the commit.
17369            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17370        }
17371
17372        /// Each row of the drain table on one set of routes: the owner's, a
17373        /// bare carrier's, and a targeted carrier's to each of its targets.
17374        #[tokio::test]
17375        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17376            struct Case {
17377                name: &'static str,
17378                change: fn(&mut ScopeRecord),
17379                /// Closed routes by index: owner->plexus, aft->plexus,
17380                /// broca->plexus, broca->other.
17381                closed: [Option<RouteCloseReason>; 4],
17382            }
17383            use RouteCloseReason::*;
17384            let cases = [
17385                Case {
17386                    name: "a carrier entry removed",
17387                    change: |r| {
17388                        r.carriers.retain(|c| {
17389                            c.principal
17390                                != Principal::Reserved {
17391                                    module_id: AFT.to_string(),
17392                                }
17393                        })
17394                    },
17395                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17396                },
17397                Case {
17398                    name: "a target removed from a carrier",
17399                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17400                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
17401                },
17402                Case {
17403                    name: "a bare carrier narrowed to targets",
17404                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17405                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17406                },
17407                Case {
17408                    name: "delegates turned off",
17409                    change: |r| r.attributes.delegates = false,
17410                    closed: [Some(ScopeDelegationChanged); 4],
17411                },
17412                Case {
17413                    name: "agent_id changed",
17414                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17415                    closed: [Some(ScopeDelegationChanged); 4],
17416                },
17417                Case {
17418                    name: "a carrier added, child owners changed, the record re-sent",
17419                    change: |r| {
17420                        r.carriers.push(carrier(MAGIC, None));
17421                        r.child_owners.push(Principal::Reserved {
17422                            module_id: MAGIC.to_string(),
17423                        });
17424                    },
17425                    closed: [None; 4],
17426                },
17427                Case {
17428                    name: "a target added",
17429                    change: |r| {
17430                        r.carriers[1]
17431                            .targets
17432                            .as_mut()
17433                            .unwrap()
17434                            .push("third".to_string())
17435                    },
17436                    closed: [None; 4],
17437                },
17438                Case {
17439                    name: "delegates turned on",
17440                    change: |r| r.attributes.delegates = true,
17441                    closed: [None; 4],
17442                },
17443            ];
17444            for case in cases {
17445                let mut rig = rig().await;
17446                rig.sync(vec![session(1)]).await;
17447                let scope = || Some(rig_selector("s", Some(1)));
17448                let mut routes = [
17449                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
17450                    rig.bound(Some(AFT), PLEXUS, scope()).await,
17451                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
17452                    rig.bound(Some(BROCA), OTHER, scope()).await,
17453                ];
17454                let mut record = session(1);
17455                (case.change)(&mut record);
17456                rig.sync(vec![record]).await;
17457                for (index, expected) in case.closed.iter().enumerate() {
17458                    let route = &mut routes[index];
17459                    match expected {
17460                        Some(reason) => {
17461                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
17462                            assert_eq!(
17463                                route.closed_reason(),
17464                                *reason,
17465                                "{}: route {index}",
17466                                case.name
17467                            );
17468                        }
17469                        None => {
17470                            assert!(rig.live(route), "{}: route {index} closed", case.name);
17471                            assert!(
17472                                route.untouched(),
17473                                "{}: route {index} was told something",
17474                                case.name
17475                            );
17476                        }
17477                    }
17478                }
17479            }
17480        }
17481
17482        #[tokio::test]
17483        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17484            // Removed, and replaced by a higher epoch.
17485            for next in [Vec::new(), vec![session(2)]] {
17486                let mut rig = rig().await;
17487                rig.sync(vec![session(1)]).await;
17488                let mut route = rig
17489                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17490                    .await;
17491                rig.sync(next).await;
17492                assert!(!rig.live(&route));
17493                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17494            }
17495
17496            // A child whose parent ends: its routes close as parent-ended, the
17497            // child stays live, and routes under the parent close as ended.
17498            let mut rig = rig().await;
17499            let mut child = session(1);
17500            child.scope_ref = "child".to_string();
17501            child.kind = ScopeKind::Worker;
17502            child.parent = Some(ScopeParent::new(
17503                Principal::Reserved {
17504                    module_id: OWNER.to_string(),
17505                },
17506                "s".to_string(),
17507                1,
17508            ));
17509            rig.sync(vec![session(1), child.clone()]).await;
17510            let mut child_route = rig
17511                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17512                .await;
17513            assert_eq!(
17514                child_route.stamp().unwrap().parent_state,
17515                Some(ParentState::Linked)
17516            );
17517            rig.sync(vec![child]).await;
17518            assert!(!rig.live(&child_route));
17519            assert_eq!(
17520                child_route.closed_reason(),
17521                RouteCloseReason::ScopeParentEnded
17522            );
17523        }
17524
17525        #[tokio::test]
17526        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17527        ) {
17528            let mut rig = rig().await;
17529            rig.sync(vec![session(1)]).await;
17530            let mut route = rig
17531                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17532                .await;
17533            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17534            rig.sync(vec![session(1)]).await;
17535            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17536            assert!(rig.live(&route) && route.untouched());
17537
17538            // A call in flight on the route when another carrier is added. A
17539            // forwarded REQUEST holds one credit on the route's flow until the
17540            // module answers; the router takes it exactly like this.
17541            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17542                .forwarding
17543                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17544                .unwrap()
17545            else {
17546                panic!("the route is bound");
17547            };
17548            binding.flow.acquire_tagged(9, false).await.unwrap();
17549            let mut widened = session(1);
17550            widened.carriers.push(carrier(MAGIC, None));
17551            rig.sync(vec![widened]).await;
17552            assert!(rig.live(&route) && route.untouched());
17553            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17554            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17555            // The call's credit is still held on an open flow, so its answer
17556            // will be delivered: closing the route would have closed the flow.
17557            assert_eq!(binding.flow.in_flight(), 1);
17558            binding
17559                .flow
17560                .acquire_tagged(10, false)
17561                .await
17562                .expect("the flow is still open");
17563        }
17564
17565        /// A swap's superseded endpoint keeps its routes until drained; ending
17566        /// the scope closes them there too.
17567        #[tokio::test]
17568        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17569            let mut rig = rig().await;
17570            rig.sync(vec![session(1)]).await;
17571            let mut on_incumbent = rig
17572                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17573                .await;
17574
17575            // Swap plexus: register a candidate and cut over, leaving the
17576            // incumbent superseded with the route still on it.
17577            let (candidate, _candidate_rx) = wide_ctx(9);
17578            let registration = rig
17579                .handler
17580                .registry
17581                .register_candidate_with_control_ops(
17582                    manifest(PLEXUS, PROTOCOL_VERSION),
17583                    PROTOCOL_VERSION,
17584                    candidate.connection_id,
17585                    module_baseline_control_ops(),
17586                )
17587                .unwrap();
17588            rig.forwarding
17589                .register_candidate_module_connection(
17590                    candidate.connection_id,
17591                    PLEXUS.to_string(),
17592                    PROTOCOL_VERSION,
17593                    manifest_concurrency(&registration.manifest),
17594                    candidate.egress.clone(),
17595                )
17596                .unwrap();
17597            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17598            rig.handler
17599                .registry
17600                .promote_candidate(PLEXUS)
17601                .unwrap()
17602                .unwrap();
17603            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17604
17605            rig.sync(Vec::new()).await;
17606            assert!(!rig.live(&on_incumbent));
17607            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17608            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17609            let goodbye = incumbent_rx
17610                .try_recv()
17611                .expect("the superseded endpoint is told")
17612                .frame;
17613            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17614        }
17615    }
17616}
17617
17618#[cfg(test)]
17619mod concurrency_default_exposure_tests {
17620    use super::*;
17621
17622    fn hello_body(role_json: &str) -> Vec<u8> {
17623        format!(
17624            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17625        )
17626        .into_bytes()
17627    }
17628
17629    fn manifest_from(body: &[u8]) -> ModuleManifest {
17630        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17631        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17632            .expect("manifest parses")
17633    }
17634
17635    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17636
17637    #[test]
17638    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17639        let body = hello_body(&format!(
17640            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17641        ));
17642        let manifest = manifest_from(&body);
17643        // Precondition: serde really resolved it to the default, so the typed
17644        // manifest alone cannot answer the question this probe exists for.
17645        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17646        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17647    }
17648
17649    #[test]
17650    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17651        let body = hello_body(&format!(
17652            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17653        ));
17654        let manifest = manifest_from(&body);
17655        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17656        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17657    }
17658
17659    #[test]
17660    fn non_management_roles_are_never_reported() {
17661        let body = hello_body(
17662            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17663        );
17664        let manifest = manifest_from(&body);
17665        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17666    }
17667}