Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
26        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
27        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
28    },
29    scope::{
30        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32        SCOPE_SYNC_OP,
33    },
34    session::{
35        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38        ROLE_VERSIONS_FIELD,
39    },
40    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41    Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47    caller_identity::CallerIdentity,
48    capability_requirements::{
49        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51        RuntimeModule,
52    },
53    child_roster::DaemonShutdownFlag,
54    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55    forwarding::{
56        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59    },
60    observability::{
61        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62    },
63    privileged_prompt::{prompt_text, PrivilegedOperation},
64    provenance::{
65        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66    },
67    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68    router::{RouteCtx, RouterError},
69    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71    stderr_tail::{CaptureState, TailEntry},
72    supervise::{
73        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74        SupervisorHandle, SwapHelloAdmission,
75    },
76    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79/// Lowest envelope version this subc build will negotiate.
80///
81/// Module HELLO negotiation is exact: peers must use the daemon's locked
82/// protocol version. Older and newer peers receive `version_unsupported` and
83/// are not registered.
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93    ops::ACTIVITY_RECORD,
94    ops::ACTIVITY_QUERY,
95    ops::ACTIVITY_SUBSCRIBE,
96    ops::SERVER_DESCRIBE,
97    ops::CATALOG_LIST,
98    ops::ROUTE_OPEN,
99    ops::ROUTE_POLL,
100    ops::ROUTE_CLOSING,
101    ops::ROUTE_CLOSED,
102    ops::SUPERVISOR_LIST,
103    ops::SUPERVISOR_RESTART,
104    ops::SUPERVISOR_SWAP,
105    ops::SUPERVISOR_RELOAD,
106    ops::SUPERVISOR_RESCAN,
107    ops::SUPERVISOR_RELEASE_RESERVED,
108    ops::SUPERVISOR_SET_ENABLED,
109    ops::SUPERVISOR_HEALTH_PROBE,
110    ops::SUPERVISOR_HEALTH,
111    ops::SUPERVISOR_STDERR_TAIL,
112    ops::SUPERVISOR_TERMINALS,
113    ops::SUPERVISOR_ROUTES,
114    ops::SUPERVISOR_PROVENANCE,
115    ops::SUPERVISOR_SPAWN_SNAPSHOT,
116    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
120/// announced by a capability in `server.describe` instead, and clients check
121/// that capability before sending it. `ck daemon --json` prints `subc_ops`
122/// verbatim and scripts read that output, so a new name there would change it.
123/// A malformed body for one of these is still refused `invalid_control_body`
124/// rather than as an unknown op.
125const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132    "supervisor.live_roots",
133    SCOPE_SYNC_OP,
134    SCOPE_APPLY_OP,
135    SCOPE_DESCRIBE_OP,
136    "operator.confirm",
137];
138
139/// Module-originated ops the daemon answers but does not advertise in
140/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
141/// belongs to is incomplete, so no module is told it works before it does.
142const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146/// How long subc waits for a module to ack a relayed route.bind before returning
147/// `module_timeout`. The ack waits on the module's own configure, which for AFT
148/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
149/// and DB-open work — on a cold page cache or a large repo that legitimately
150/// exceeds a couple of seconds. The default is generous because rejecting a VALID
151/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
152/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
153pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155/// How many CONSECUTIVE full-budget relay timeouts against one target module
156/// open that module's bind-relay breaker.
157///
158/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
159/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
160/// so three consecutive full-budget timeouts take ~36s to observe: every client
161/// whose open contributed to opening the breaker had already given up on its
162/// own. That is what makes opening the breaker unable to turn a call that would
163/// have succeeded into a refusal — it can only make an already-failing module
164/// fail faster.
165///
166/// Two would be reachable inside one default deadline. One would convict a
167/// module on a single cold-cache bind, which is exactly the valid-but-slow case
168/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
169pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171/// How long a module's bind-relay breaker stays open before exactly one
172/// `route.open` is let through as a probe.
173///
174/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
175/// re-pays a full-budget stall almost continuously, and the breaker stops being
176/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
177/// deadline: a client that starts retrying after the module recovers has to get
178/// a probe opportunity inside its own deadline, or the breaker converts a
179/// recovered module into a failed call — the failure it exists to prevent,
180/// pointed the other way.
181///
182/// 20s sits between those with room on both sides, and it caps what a wedged
183/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
184/// rather than one per `route.open` per connection. The stall that motivated
185/// this, with its measurements, is written up in
186/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
187/// waited the whole budget out.
188pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194    configured: &Path,
195    spawned_from: Option<&Path>,
196    image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198    let path = match spawned_from {
199        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200            ReloadPathAgreement::Match
201        }
202        Some(spawned_from) => ReloadPathAgreement::Mismatch {
203            configured: configured.to_path_buf(),
204            spawned_from: spawned_from.to_path_buf(),
205        },
206        None => ReloadPathAgreement::Unavailable {
207            reason: if matches!(
208                image,
209                subc_control::RunningImageAgreement::Unavailable {
210                    reason: subc_control::RunningImageUnavailableReason::NotRunning
211                }
212            ) {
213                ReloadPathUnavailableReason::NotRunning
214            } else {
215                ReloadPathUnavailableReason::SpawnedPathUnavailable
216            },
217        },
218    };
219    PendingReloadVerdict {
220        path,
221        image,
222        spec: None,
223    }
224}
225
226fn launch_spec_agreement(
227    configured: &crate::supervise::ModuleSpec,
228    running: bool,
229    spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232    if !running {
233        return Agreement::Unavailable {
234            reason: Reason::NotRunning,
235        };
236    }
237    match spawned {
238        None => Agreement::Unavailable {
239            reason: Reason::SpawnedSpecUnavailable,
240        },
241        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242            Agreement::Match
243        }
244        Some(_) => Agreement::Mismatch,
245    }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249    if configured == spawned_from {
250        return true;
251    }
252    #[cfg(windows)]
253    {
254        // Windows spawn records the resolved verbatim path (\\?\...), while
255        // configuration keeps its original spelling. Resolve both with the OS
256        // instead of stripping prefixes or folding case, which could conflate
257        // distinct names in case-sensitive directories. Different resolved
258        // paths still differ even if their files have identical content.
259        match (
260            std::fs::canonicalize(configured),
261            std::fs::canonicalize(spawned_from),
262        ) {
263            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264            _ => false,
265        }
266    }
267    #[cfg(not(windows))]
268    {
269        false
270    }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275    build: DaemonBuildProvenance,
276    pid: Option<u32>,
277    started_at_ms: Option<u64>,
278    start_clock: Option<crate::clock::StartClock>,
279    executable_path: Option<PathBuf>,
280    executable_identity: Option<SpawnedFileIdentity>,
281    process_start_time: Option<u64>,
282    probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286    fn default() -> Self {
287        Self {
288            build: DaemonBuildProvenance {
289                build_git_sha: None,
290                build_lock_digest: None,
291            },
292            pid: None,
293            started_at_ms: None,
294            start_clock: None,
295            executable_path: None,
296            executable_identity: None,
297            process_start_time: None,
298            probe: ExecutableIdentityProbe::default(),
299        }
300    }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305    supervisor: Supervisor,
306    config_path: PathBuf,
307    configured_port: Option<u16>,
308    storage_config: Option<crate::daemon_config::StorageConfig>,
309    admission_facts_carrier_module_id: Option<String>,
310    admission_facts_targets: Option<Vec<String>>,
311    scope_authority_owners: Vec<String>,
312    privileged_operator_prompt: PrivilegedOperatorPrompt,
313    /// Counts executed rescans that set out to change the supervised modules'
314    /// staged or applied configuration, counted before the first change so a
315    /// rescan that fails partway still counts. With the daemon incarnation it
316    /// names the state a `supervisor.rescan_checked` preview was computed
317    /// against.
318    generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321/// Refusal labels passed to `observe_route_open_refusal` that mean the target
322/// module is not serving right now, and so open or extend an outage in the
323/// route outage tracker. Every one of them is only reachable after the target
324/// was found in the registry, which is what keeps an arbitrary client-chosen
325/// id from ever creating tracker state.
326///
327/// Deliberately absent: `not_registered` and `removed` (the id may be
328/// anything a client sent, and a removed module is gone on purpose),
329/// `protocol_none` (such a module never serves routes, so nothing is out),
330/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
331/// capability and admission-facts refusals (they refuse the caller, not a
332/// module outage), and `relay_reservation_failed` (its code ranges over
333/// capacity limits as well as a vanished connection). Capacity, breaker,
334/// relay-timeout and module-rejection refusals do not pass through that
335/// function at all; the breaker logs its own transitions.
336///
337/// The two not-serving refusals that bypass that function record themselves
338/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
339/// `required_capability_unprovided` is not tracked: the module itself is up,
340/// and the outage belongs to the missing provider.
341const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342    "reloading",
343    "supervisor_not_live",
344    "registration_not_active",
345    "no_forwarding_connection",
346    "relay_send_failed",
347];
348
349/// Real channel-0 control handler for subc itself.
350#[derive(Clone)]
351pub struct ControlHandler {
352    registry: Arc<Registry>,
353    activity: Option<crate::activity::ActivityLedger>,
354    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355    forwarding: Arc<ForwardingTable>,
356    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357    supervisor: SupervisorHandle,
358    subc_capabilities: Arc<[String]>,
359    /// Daemon-wide route.bind relay budget. Used as the fallback when the
360    /// target module has no per-module override in
361    /// `route_bind_relay_timeouts`.
362    route_bind_relay_timeout: Duration,
363    /// Per-module route.bind relay budget overrides, keyed by module id. When
364    /// `handle_route_open` resolves the deadline for a target module, a
365    /// per-module entry wins over the daemon-wide value above.
366    route_bind_relay_timeouts: BTreeMap<String, Duration>,
367    /// Per-target-module bind-relay breaker state. Shared with the forwarding
368    /// table, which is where a new module connection resets it.
369    route_bind_breakers: RouteBindBreakers,
370    /// Live relay admissions keyed by target module. Shared through the
371    /// forwarding table so cloned or separately built handlers enforce one cap.
372    route_bind_concurrency: RouteBindConcurrency,
373    /// Start and end of each module's not-serving period as seen by
374    /// `route.open`, so an outage gets one line at each edge instead of only
375    /// the per-refusal INFO lines. Taken from the forwarding table, so every
376    /// handler built over one table shares it.
377    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378    /// Consecutive relay timeouts that open a module's breaker.
379    route_bind_breaker_threshold: u32,
380    /// How long a breaker stays open before one probe is admitted.
381    route_bind_breaker_cooldown: Duration,
382    health_probe_timeout: Duration,
383    /// Central storage policy. When set, each registering module receives its
384    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
385    storage_config: Option<crate::daemon_config::StorageConfig>,
386    /// The machine id established at boot, served on every HELLO_ACK and on
387    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
388    /// changes the file, never this value. `None` serves no id.
389    machine_id: Option<crate::machine_id::MachineId>,
390    admission_facts_carrier_module_id: Option<String>,
391    admission_facts_targets: Option<Vec<String>>,
392    /// Scope records with their sync authorities and tombstones; see
393    /// `crate::scopes`. Shared by clones of this handler, so every connection
394    /// reads and writes one table.
395    scopes: Arc<RwLock<ScopeTable>>,
396    /// Wakes the expiry loop when an accepted scope change may have added a
397    /// deadline, so the loop can wait without ticking while no live scope has
398    /// one. A permit is stored when nothing waits, so no wake is lost.
399    scope_deadline_added: Arc<tokio::sync::Notify>,
400    /// The configured `scope_authority_owners`, kept so a rescan can report a
401    /// changed value as needing a daemon restart; rescan never applies it.
402    scope_authority_owners: Vec<String>,
403    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
404    privileged_operator_prompt: PrivilegedOperatorPrompt,
405    /// The local and peer address of each accepted TCP client connection, read
406    /// from the daemon's own socket, so a privileged request can name its caller.
407    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408    #[cfg(test)]
409    caller_lookup: Option<TestCallerLookup>,
410    /// The launch nonce each module connection presented at HELLO, which is how
411    /// a `scope.sync` is matched to the owner's current launch.
412    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413    rescan: Option<SupervisorRescanContext>,
414    connected_clients: ConnectedClients,
415    counters: DaemonCounters,
416    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
418    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
419    /// their usual severity (see [`log_requirement_events`]). A handler built
420    /// without one has a flag that is never set.
421    daemon_shutdown: DaemonShutdownFlag,
422    daemon_provenance: DaemonProvenanceFacts,
423    #[cfg(test)]
424    control_dispatch_delay: Option<Duration>,
425    #[cfg(test)]
426    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427    /// Holds `route.open` after the route.bind is queued to the module and
428    /// before it is marked enqueued, so a test can make the module's reply (or
429    /// a client teardown) land exactly inside that window.
430    #[cfg(test)]
431    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432    /// Holds a HELLO after its registry entry is reserved and before its
433    /// forwarding endpoint is installed, so a test can read the catalog and
434    /// open routes while the module is registered but not yet routable. A
435    /// blocking barrier because the HELLO handler is synchronous; the test
436    /// runs the HELLO on its own thread.
437    #[cfg(test)]
438    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443        f.debug_struct("ControlHandler")
444            .field("registry", &self.registry)
445            .field("forwarding", &self.forwarding)
446            .field("process_liveness", &self.process_liveness.is_some())
447            .field("supervisor", &self.supervisor)
448            .field("subc_capabilities", &self.subc_capabilities)
449            .finish()
450    }
451}
452
453struct RouteOpenRequest {
454    target: RouteTarget,
455    identity: BindIdentity,
456    consumer_identity: Option<ConsumerIdentity>,
457    consumer_capabilities: Option<Vec<String>>,
458    role_versions: Option<BTreeMap<String, String>>,
459    admission_facts: Option<serde_json::Value>,
460    scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465    Admission,
466    ProjectRoot,
467    EgressReservation,
468    BindSend,
469    BindAckWait,
470}
471
472/// Store elapsed time for each route-open phase, including attempts that end
473/// in an early refusal or backend error. Admission also includes rechecks and
474/// commit bookkeeping performed between the other timed phases.
475struct RouteOpenTimings {
476    module_id: String,
477    principal: String,
478    admission: Duration,
479    project_root: Duration,
480    egress_reservation: Duration,
481    bind_send: Duration,
482    bind_ack_wait: Duration,
483    phase: Option<RouteOpenPhase>,
484    phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488    fn new() -> Self {
489        Self {
490            module_id: String::new(),
491            // Refusals before attestation must not claim a verified principal.
492            principal: "unresolved".to_string(),
493            admission: Duration::ZERO,
494            project_root: Duration::ZERO,
495            egress_reservation: Duration::ZERO,
496            bind_send: Duration::ZERO,
497            bind_ack_wait: Duration::ZERO,
498            phase: Some(RouteOpenPhase::Admission),
499            phase_started_at: Instant::now(),
500        }
501    }
502
503    fn finish(&mut self) {
504        let elapsed = self.phase_started_at.elapsed();
505        match self.phase.take() {
506            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511            None => {}
512        }
513    }
514
515    fn enter(&mut self, phase: RouteOpenPhase) {
516        self.finish();
517        self.phase = Some(phase);
518        self.phase_started_at = Instant::now();
519    }
520}
521
522struct RouteBindReservationGuard {
523    forwarding: Arc<ForwardingTable>,
524    endpoint: ModuleEndpointId,
525    relay_corr: u64,
526    armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530    forwarding: Arc<ForwardingTable>,
531    endpoint: ModuleEndpointId,
532    corr: u64,
533    armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538        Self {
539            forwarding,
540            endpoint,
541            corr,
542            armed: true,
543        }
544    }
545
546    fn disarm(&mut self) {
547        self.armed = false;
548    }
549}
550
551impl Drop for ModuleControlRpcGuard {
552    fn drop(&mut self) {
553        if self.armed {
554            let _ = self
555                .forwarding
556                .cancel_module_control_rpc(self.endpoint, self.corr);
557        }
558    }
559}
560
561impl RouteBindReservationGuard {
562    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563        Self {
564            forwarding,
565            endpoint,
566            relay_corr,
567            armed: true,
568        }
569    }
570
571    fn release_and_disarm(&mut self) {
572        if !self.armed {
573            return;
574        }
575        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576            self.endpoint,
577            self.relay_corr,
578            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579        ) {
580            send_goodbye_target_best_effort(
581                &self.forwarding.counters(),
582                &target,
583                "canceled route.bind",
584            );
585        }
586        self.armed = false;
587    }
588
589    fn disarm(&mut self) {
590        self.armed = false;
591    }
592}
593
594impl Drop for RouteBindReservationGuard {
595    fn drop(&mut self) {
596        self.release_and_disarm();
597    }
598}
599
600/// Per-target-module circuit breaker around the `route.bind` relay.
601///
602/// The connection reader is serial per connection, so a module whose `on_bind`
603/// sits on the ack blocks every LATER frame on the connections that call it,
604/// including calls to unrelated modules. This does not make any module's bind
605/// fast; it stops the daemon paying the full budget again and again for a
606/// condition it has already observed.
607///
608/// State is keyed by TARGET MODULE and shared by every connection: a wedged
609/// module wedges everyone, so what one connection learned should protect the
610/// rest.
611///
612/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
613/// relay to that module has actually timed out, and is removed again when a
614/// relay is accepted or the module reconnects, so it cannot grow with traffic
615/// or with modules that behave.
616///
617/// # Why a `std` mutex here is not the head-of-line defect again
618///
619/// Acquisition never awaits. The critical section is a hash lookup plus a few
620/// integer updates, with no I/O and no `.await` inside it, so a reader task
621/// cannot be descheduled behind it the way it can behind
622/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
623/// primitive, held for the same kind of work, as the refusal counter this very
624/// path already increments.
625///
626/// It is also NOT on the data-plane splice path: only `route.open` and module
627/// registration touch it, so bound-route frames gain no state check and no
628/// contention.
629#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636    modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640    concurrency: RouteBindConcurrency,
641    module_id: String,
642}
643
644impl RouteBindConcurrency {
645    /// Admit without waiting. Waiting here would move the bind stall from the
646    /// module reply to a semaphore and restore reader head-of-line blocking.
647    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648        let mut modules = self
649            .modules
650            .lock()
651            .expect("route.bind concurrency mutex poisoned");
652        let in_flight = modules.entry(module_id.to_string()).or_default();
653        if *in_flight >= limit {
654            return Err(*in_flight);
655        }
656        *in_flight += 1;
657        Ok(RouteBindConcurrencyGuard {
658            concurrency: self.clone(),
659            module_id: module_id.to_string(),
660        })
661    }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665    fn drop(&mut self) {
666        let mut modules = self
667            .concurrency
668            .modules
669            .lock()
670            .expect("route.bind concurrency mutex poisoned");
671        let remove = {
672            let in_flight = modules
673                .get_mut(&self.module_id)
674                .expect("admitted route.bind has a concurrency entry");
675            *in_flight -= 1;
676            *in_flight == 0
677        };
678        if remove {
679            modules.remove(&self.module_id);
680        }
681    }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686    /// Relay timeouts observed with no accepted relay in between.
687    consecutive_timeouts: u32,
688    /// `Some` while the breaker is open: the instant the cooldown expires and
689    /// the next arrival may probe. `None` means closed.
690    cooldown_until: Option<Instant>,
691    /// A half-open probe has been admitted and has not settled yet. This is
692    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
693    /// that read the cooldown, so concurrent opens arriving at the moment the
694    /// cooldown expires cannot all decide that they are the probe.
695    probe_in_flight: Option<Arc<()>>,
696}
697
698/// What the breaker decided for one `route.open`, before any relay work.
699enum RouteBindAdmission<'a> {
700    Admitted {
701        guard: RouteBindBreakerGuard<'a>,
702        /// This open is the single half-open probe, so the transition is worth
703        /// one log line.
704        probe: bool,
705    },
706    Refused {
707        consecutive_timeouts: u32,
708        /// What is left of the cooldown. Zero when the refusal is because the
709        /// one probe is already in flight rather than because the cooldown has
710        /// not elapsed.
711        retry_in: Duration,
712        probe_in_flight: bool,
713    },
714}
715
716/// An outstanding admission, which must be told how its relay settled.
717///
718/// `Drop` settles it as inconclusive, so an early return between admission and
719/// the relay -- or the whole handler being cancelled when the client
720/// disconnects -- releases a half-open probe slot instead of leaving the
721/// breaker wedged half-open with no further probes.
722struct RouteBindBreakerGuard<'a> {
723    breakers: RouteBindBreakers,
724    module_id: &'a str,
725    probe_token: Option<Arc<()>>,
726    settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730    /// The module answered within the budget and took the bind. THE ONLY
731    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
732    /// breaker, which is a transition worth logging.
733    fn record_accepted(&mut self) -> bool {
734        self.settled = true;
735        self.breakers.record_accepted(self.module_id)
736    }
737
738    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
739    /// COUNTS TOWARD OPENING.
740    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741        self.settled = true;
742        self.breakers.record_timeout(
743            self.module_id,
744            self.probe_token.as_ref(),
745            threshold,
746            cooldown,
747        )
748    }
749
750    /// Everything else: the module REJECTED the bind, its connection went away
751    /// mid-relay, or the waiter was cancelled.
752    ///
753    /// None of these is evidence that a module is slow, and each already has
754    /// its own refusal with its own code. A module that rejects a bind in
755    /// microseconds is healthy and must never be convicted for it; a module
756    /// that died has said nothing about the module that replaces it. So these
757    /// neither increment nor reset the count -- they only release a probe slot.
758    fn record_inconclusive(&mut self) {
759        self.settled = true;
760        self.breakers
761            .record_inconclusive(self.module_id, self.probe_token.as_ref());
762    }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766    fn drop(&mut self) {
767        if !self.settled {
768            self.breakers
769                .record_inconclusive(self.module_id, self.probe_token.as_ref());
770        }
771    }
772}
773
774/// The breaker moved to open, reported so the caller can log it outside the
775/// lock. Opening is rare and load-bearing; the refusals that follow are
776/// frequent and are counted rather than logged.
777struct BreakerOpened {
778    consecutive_timeouts: u32,
779    /// True when a failed probe re-opened an already-open breaker, which reads
780    /// very differently in a log from a first opening.
781    reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786        self.modules
787            .lock()
788            .expect("route.bind breaker mutex poisoned")
789    }
790
791    /// Decide whether this `route.open` may attempt its relay. Takes the map
792    /// lock and nothing else, and never awaits.
793    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795            probe: probe_token.is_some(),
796            guard: RouteBindBreakerGuard {
797                breakers: self.clone(),
798                module_id,
799                probe_token,
800                settled: false,
801            },
802        };
803
804        let mut modules = self.lock();
805        let Some(state) = modules.get_mut(module_id) else {
806            return admitted(None);
807        };
808        let Some(cooldown_until) = state.cooldown_until else {
809            return admitted(None);
810        };
811        if state.probe_in_flight.is_some() {
812            return RouteBindAdmission::Refused {
813                consecutive_timeouts: state.consecutive_timeouts,
814                retry_in: Duration::ZERO,
815                probe_in_flight: true,
816            };
817        }
818        let now = Instant::now();
819        if now < cooldown_until {
820            return RouteBindAdmission::Refused {
821                consecutive_timeouts: state.consecutive_timeouts,
822                retry_in: cooldown_until - now,
823                probe_in_flight: false,
824            };
825        }
826        let token = Arc::new(());
827        state.probe_in_flight = Some(Arc::clone(&token));
828        admitted(Some(token))
829    }
830
831    fn record_accepted(&self, module_id: &str) -> bool {
832        self.lock()
833            .remove(module_id)
834            .is_some_and(|state| state.cooldown_until.is_some())
835    }
836
837    fn record_timeout(
838        &self,
839        module_id: &str,
840        probe_token: Option<&Arc<()>>,
841        threshold: u32,
842        cooldown: Duration,
843    ) -> Option<BreakerOpened> {
844        let mut modules = self.lock();
845        let state = modules.entry(module_id.to_string()).or_default();
846        let was_open = state.cooldown_until.is_some();
847        let was_probe = Self::owns_probe(state, probe_token);
848        if was_probe {
849            state.probe_in_flight = None;
850        }
851        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852        if state.consecutive_timeouts < threshold {
853            return None;
854        }
855        state.cooldown_until = Some(Instant::now() + cooldown);
856        Some(BreakerOpened {
857            consecutive_timeouts: state.consecutive_timeouts,
858            reopened_after_probe: was_open && was_probe,
859        })
860    }
861
862    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863        // A relay can finish after the breaker was reset or after it opened
864        // again and started a new probe. Only the guard whose token matches the
865        // active probe may release it, so a late relay never frees a newer probe.
866        state
867            .probe_in_flight
868            .as_ref()
869            .zip(token)
870            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871    }
872
873    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874        if let Some(state) = self.lock().get_mut(module_id) {
875            if Self::owns_probe(state, probe_token) {
876                state.probe_in_flight = None;
877            }
878        }
879    }
880
881    /// Discard what was learned about a module, because the process it was
882    /// learned about is gone. Returns the discarded count when it was non-zero.
883    ///
884    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
885    /// `module_id` is a configuration identity that outlives any particular
886    /// child; what the breaker observed was the process behind the module
887    /// connection of the moment. When a new connection registers under that id
888    /// the verdict's subject no longer exists, so the verdict is stale by
889    /// construction rather than merely likely to be wrong. Keeping it would
890    /// apply a dead process's record to a live one, which is the same defect
891    /// class this breaker exists to stop the daemon committing.
892    ///
893    /// A half-open probe in flight is discarded with the rest: it was a
894    /// question about the old process.
895    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896        self.lock()
897            .remove(module_id)
898            .map(|state| state.consecutive_timeouts)
899            .filter(|discarded| *discarded > 0)
900    }
901
902    /// Open breakers, for the `server.describe` counters object. `None` when
903    /// none is open, so the key stays absent rather than present-and-empty.
904    ///
905    /// This is the operator's answer to "is this module refusing instantly or
906    /// is it fine?", which look identical from a client that retries and then
907    /// succeeds.
908    fn open_snapshot(&self) -> Option<serde_json::Value> {
909        let now = Instant::now();
910        let modules = self.lock();
911        let open = modules
912            .iter()
913            .filter_map(|(module_id, state)| {
914                let cooldown_until = state.cooldown_until?;
915                Some((
916                    module_id.clone(),
917                    serde_json::json!({
918                        "consecutive_timeouts": state.consecutive_timeouts,
919                        "cooldown_remaining_ms":
920                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
921                        "probe_in_flight": state.probe_in_flight.is_some(),
922                    }),
923                ))
924            })
925            .collect::<serde_json::Map<String, serde_json::Value>>();
926        (!open.is_empty()).then_some(serde_json::Value::Object(open))
927    }
928}
929
930impl ControlHandler {
931    pub fn new(registry: Arc<Registry>) -> Self {
932        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933    }
934
935    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936        let counters = forwarding.counters();
937        // Taken from the forwarding table rather than created here, so that the
938        // breaker a `route.open` consults is the same one a module's
939        // registration resets, however many handlers are built over one table.
940        let route_bind_breakers = forwarding.route_bind_breakers();
941        let route_bind_concurrency = forwarding.route_bind_concurrency();
942        let route_outages = forwarding.route_outages();
943        Self {
944            registry,
945            activity: None,
946            activity_streams: Arc::new(Mutex::new(HashMap::new())),
947            forwarding,
948            process_liveness: None,
949            supervisor: SupervisorHandle::new(),
950            subc_capabilities: Arc::from([
951                CAP_MANIFEST_REGISTRATION.to_string(),
952                CAP_CHANNEL_LIFECYCLE.to_string(),
953                CAP_PING_PONG.to_string(),
954                CAP_SESSION_ATTACH.to_string(),
955                CAP_ADMISSION_FACTS_RELAY.to_string(),
956                CAP_SCOPES_V1.to_string(),
957                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966            ]),
967            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968            route_bind_relay_timeouts: BTreeMap::new(),
969            route_bind_breakers,
970            route_bind_concurrency,
971            route_outages,
972            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975            storage_config: None,
976            machine_id: None,
977            admission_facts_carrier_module_id: None,
978            admission_facts_targets: None,
979            scopes: Arc::new(RwLock::new(ScopeTable::new(
980                crate::daemon_config::default_scope_authority_owners(),
981            ))),
982            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986            #[cfg(test)]
987            caller_lookup: None,
988            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989            rescan: None,
990            connected_clients: ConnectedClients::new(),
991            counters,
992            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993            daemon_shutdown: DaemonShutdownFlag::default(),
994            daemon_provenance: DaemonProvenanceFacts::default(),
995            #[cfg(test)]
996            control_dispatch_delay: None,
997            #[cfg(test)]
998            provenance_probe_override: None,
999            #[cfg(test)]
1000            route_bind_relay_sent_hold: None,
1001            #[cfg(test)]
1002            hello_registry_to_forwarding_hold: None,
1003        }
1004    }
1005
1006    /// Set the central storage policy: registering modules then receive their
1007    /// resolved storage descriptor in HELLO_ACK.
1008    pub fn with_storage_config(
1009        mut self,
1010        storage_config: Option<crate::daemon_config::StorageConfig>,
1011    ) -> Self {
1012        self.storage_config = storage_config;
1013        self
1014    }
1015
1016    /// Open the ledger on its own writer thread; an unavailable sink does not
1017    /// prevent the daemon from serving routing or supervisor requests.
1018    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019        self.activity = Some(crate::activity::ActivityLedger::open(
1020            run_dir.join("activity.jsonl"),
1021            incarnation,
1022        ));
1023        self.supervisor.set_activity(self.activity.clone());
1024        self
1025    }
1026
1027    /// Set the machine id served to every registering module (HELLO_ACK) and on
1028    /// `server.describe`.
1029    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030        self.machine_id = machine_id;
1031        self
1032    }
1033
1034    /// Configure the exact reserved module and target ids permitted to relay
1035    /// opaque admission facts. Config-file loading validates this authority;
1036    /// this builder keeps the same policy available to embedded test daemons.
1037    pub fn with_admission_facts_config(
1038        mut self,
1039        carrier_module_id: Option<String>,
1040        targets: Option<Vec<String>>,
1041    ) -> Self {
1042        self.admission_facts_carrier_module_id = carrier_module_id;
1043        self.admission_facts_targets = targets;
1044        self
1045    }
1046
1047    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1048    /// Replaces the scope table with an empty one under the new list, so call it
1049    /// while building the handler, before any module can sync.
1050    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052        self.scope_authority_owners = owners;
1053        self
1054    }
1055
1056    /// Set the startup policy before installing the rescan context or serving requests.
1057    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058        self.privileged_operator_prompt = mode;
1059        self
1060    }
1061
1062    /// Override the route.bind relay timeout. Used by tests that assert the
1063    /// timeout path so they don't block on the production-safe default.
1064    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065        self.route_bind_relay_timeout = timeout;
1066        self
1067    }
1068
1069    /// Install per-module route.bind relay budget overrides. A module id
1070    /// listed here wins over the daemon-wide default set via
1071    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1072    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1073    /// the same `Duration` the bind path will use.
1074    pub fn with_route_bind_relay_timeouts(
1075        mut self,
1076        timeouts: impl IntoIterator<Item = (String, Duration)>,
1077    ) -> Self {
1078        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079        self
1080    }
1081
1082    /// Resolve the route.bind relay budget for a specific target module id.
1083    /// Per-module overrides win; the daemon-wide value (set via
1084    /// `with_route_bind_relay_timeout` or the built-in default) is the
1085    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1086    /// the same resolution `handle_route_open` will use.
1087    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088        self.route_bind_relay_timeouts
1089            .get(module_id)
1090            .copied()
1091            .unwrap_or(self.route_bind_relay_timeout)
1092    }
1093
1094    /// Override the per-module bind-relay breaker policy.
1095    ///
1096    /// Used by tests, which cannot spend three production budgets opening a
1097    /// breaker or twenty seconds waiting for its cooldown. The production
1098    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1099    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1100    /// reasoning for the numbers.
1101    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102        self.route_bind_breaker_threshold = threshold.max(1);
1103        self.route_bind_breaker_cooldown = cooldown;
1104        self
1105    }
1106
1107    #[cfg(test)]
1108    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109        self.health_probe_timeout = timeout;
1110        self
1111    }
1112
1113    #[cfg(test)]
1114    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115        self.control_dispatch_delay = Some(delay);
1116        self
1117    }
1118
1119    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1120    /// announce it is parked, once to be released. Build the barrier for two
1121    /// parties, the test being the other one.
1122    #[cfg(test)]
1123    pub(crate) fn with_route_bind_relay_sent_hold(
1124        mut self,
1125        hold: Arc<tokio::sync::Barrier>,
1126    ) -> Self {
1127        self.route_bind_relay_sent_hold = Some(hold);
1128        self
1129    }
1130
1131    /// A HELLO waits on `hold` twice between reserving its registry entry and
1132    /// installing its forwarding endpoint: once to announce it is parked, once
1133    /// to be released. Build the barrier for two parties, the test being the
1134    /// other one, and run the HELLO on a thread of its own.
1135    #[cfg(test)]
1136    pub(crate) fn with_hello_registry_to_forwarding_hold(
1137        mut self,
1138        hold: Arc<std::sync::Barrier>,
1139    ) -> Self {
1140        self.hello_registry_to_forwarding_hold = Some(hold);
1141        self
1142    }
1143
1144    #[cfg(test)]
1145    fn hold_between_registry_and_forwarding(&self) {
1146        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147            hold.wait();
1148            hold.wait();
1149        }
1150    }
1151
1152    pub fn with_process_liveness(
1153        mut self,
1154        process_liveness: Arc<dyn ModuleProcessLiveness>,
1155    ) -> Self {
1156        self.process_liveness = Some(process_liveness);
1157        self
1158    }
1159
1160    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161        self.supervisor = supervisor;
1162        self.supervisor.set_activity(self.activity.clone());
1163        self
1164    }
1165
1166    pub fn with_daemon_provenance(
1167        mut self,
1168        pid: u32,
1169        started_at_ms: u64,
1170        executable_path: Option<PathBuf>,
1171        build_git_sha: Option<String>,
1172        build_lock_digest: Option<String>,
1173    ) -> Self {
1174        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175        let process_start_time = process_start_time(pid);
1176        self.daemon_provenance = DaemonProvenanceFacts {
1177            build: DaemonBuildProvenance {
1178                build_git_sha,
1179                build_lock_digest,
1180            },
1181            pid: Some(pid),
1182            started_at_ms: Some(started_at_ms),
1183            start_clock: None,
1184            executable_path,
1185            executable_identity,
1186            process_start_time,
1187            probe: ExecutableIdentityProbe::default(),
1188        };
1189        self
1190    }
1191
1192    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193        self.daemon_provenance.start_clock = Some(clock);
1194        self
1195    }
1196
1197    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1198    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199        self.scopes
1200            .write()
1201            .unwrap_or_else(|poisoned| poisoned.into_inner())
1202            .set_wall_clock(Arc::new(clock));
1203        self
1204    }
1205
1206    #[cfg(test)]
1207    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208        self.provenance_probe_override = Some(result);
1209        self
1210    }
1211
1212    /// Install the configured module set and its reserved capability bindings.
1213    /// Bindings are configuration-scoped and may point at a provider that has not
1214    /// been installed yet, so this does not require the bound module to exist.
1215    pub fn with_capability_config(
1216        self,
1217        modules: impl IntoIterator<Item = (String, bool)>,
1218        reserved_capabilities: BTreeMap<String, String>,
1219    ) -> Self {
1220        self.capability_evaluator
1221            .configure(modules, reserved_capabilities);
1222        self
1223    }
1224
1225    /// Share the supervisor's daemon-shutdown flag with this handler, so
1226    /// requirement changes after the daemon has begun its announced shutdown
1227    /// are not reported as faults.
1228    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229        self.daemon_shutdown = flag;
1230        self
1231    }
1232
1233    pub fn with_supervisor_rescan(
1234        mut self,
1235        supervisor: Supervisor,
1236        config_path: impl Into<PathBuf>,
1237        configured_port: Option<u16>,
1238    ) -> Self {
1239        self.rescan = Some(SupervisorRescanContext {
1240            supervisor,
1241            config_path: config_path.into(),
1242            configured_port,
1243            storage_config: self.storage_config.clone(),
1244            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245            admission_facts_targets: self.admission_facts_targets.clone(),
1246            scope_authority_owners: self.scope_authority_owners.clone(),
1247            privileged_operator_prompt: self.privileged_operator_prompt,
1248            generation: Arc::default(),
1249        });
1250        self
1251    }
1252
1253    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254        self.connected_clients = connected_clients;
1255        self
1256    }
1257
1258    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259        Arc::clone(&self.forwarding)
1260    }
1261
1262    pub(crate) fn counters(&self) -> DaemonCounters {
1263        self.counters.clone()
1264    }
1265
1266    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1267    /// requirement event without depending on an operator polling a status command.
1268    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269        tokio::spawn(async move {
1270            loop {
1271                self.capability_evaluator
1272                    .wait_for_change_or_deadline()
1273                    .await;
1274                self.refresh_capability_requirements();
1275            }
1276        });
1277    }
1278
1279    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1280    /// deadline passed during sleep is enforced on the first tick after wake.
1281    /// While no live scope carries a deadline the loop waits for an accepted
1282    /// scope change instead of ticking, so a daemon without deadlines never
1283    /// wakes for expiry.
1284    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285        tokio::spawn(async move {
1286            loop {
1287                let has_deadlines = self
1288                    .scopes
1289                    .read()
1290                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1291                    .has_deadlines();
1292                if !has_deadlines {
1293                    self.scope_deadline_added.notified().await;
1294                    continue;
1295                }
1296                tokio::time::sleep(Duration::from_secs(1)).await;
1297                if let Err(error) = self.sweep_expired_scopes() {
1298                    warn!(%error, "scope expiry sweep failed");
1299                }
1300            }
1301        });
1302    }
1303
1304    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305        let mut table = self
1306            .scopes
1307            .write()
1308            .unwrap_or_else(|poisoned| poisoned.into_inner());
1309        let swept = table.sweep_expired();
1310        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311        drop(table);
1312        self.close_scope_drained_routes(drained);
1313        Ok(())
1314    }
1315
1316    fn publish_scope_changes(
1317        &self,
1318        changes: &[crate::scopes::ScopeTagChange],
1319        expired: &[crate::scopes::ScopeExpired],
1320    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321        if changes.is_empty() {
1322            return Ok(Vec::new());
1323        }
1324        let drained = self
1325            .forwarding
1326            .publish_scope_changes(changes)
1327            .map_err(RouterError::Forwarding)?;
1328        let mut counts = HashMap::new();
1329        for route in &drained {
1330            *counts
1331                .entry((
1332                    route.scope.owner.as_str(),
1333                    route.scope.scope_ref.as_str(),
1334                    route.scope.tag.scope_epoch,
1335                ))
1336                .or_insert(0usize) += 1;
1337        }
1338        for scope in expired {
1339            let routes_closed = counts
1340                .get(&(
1341                    scope.owner.as_str(),
1342                    scope.scope_ref.as_str(),
1343                    scope.scope_epoch,
1344                ))
1345                .copied()
1346                .unwrap_or(0);
1347            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349                routes_closed, "scope expired");
1350        }
1351        Ok(drained)
1352    }
1353
1354    fn runtime_capability_snapshot(
1355        &self,
1356    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357        let runtime = self
1358            .supervisor
1359            .list()
1360            .into_iter()
1361            .map(|module| {
1362                let status = module.status().map_err(|err| {
1363                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364                })?;
1365                Ok(RuntimeModule {
1366                    module_id: status.module_id,
1367                    state: status.state,
1368                    enabled: status.enabled,
1369                })
1370            })
1371            .collect::<Result<Vec<_>, RouterError>>()?;
1372        // Include every registration, published or not. The capability
1373        // evaluator counts a module as providing its capabilities as soon as it
1374        // has registered, even before `catalog.list` shows it, so a HELLO in
1375        // progress does not change other modules' required-capability
1376        // verdicts. See `Registry::list_registered_modules`.
1377        let registrations = self.registry.list_registered_modules().map_err(|err| {
1378            RouterError::backend(
1379                0,
1380                0,
1381                format!("failed to list capability registrations: {err}"),
1382            )
1383        })?;
1384        let registrations = registrations
1385            .into_iter()
1386            .map(|registration| RegisteredModule {
1387                module_id: registration.manifest.module_id,
1388                module_version: registration.manifest.module_version,
1389                capabilities: registration.manifest.capabilities,
1390            })
1391            .collect();
1392        Ok((runtime, registrations))
1393    }
1394
1395    /// The capability side effects of a module becoming the active registration
1396    /// for its id: cache its manifest (warning if its claims drifted), run the
1397    /// deny census when its declarations call for one, and recompute the
1398    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1399    /// candidate's does not, and the supervisor does it at promotion instead,
1400    /// through [`crate::supervise::SwapPromotionObserver`].
1401    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402        let cached_registration = RegisteredModule {
1403            module_id: registration.manifest.module_id.clone(),
1404            module_version: registration.manifest.module_version.clone(),
1405            capabilities: registration.manifest.capabilities.clone(),
1406        };
1407        if self.capability_evaluator.record_hello(&cached_registration) {
1408            warn!(
1409                module_id = %cached_registration.module_id,
1410                "capability claims drifted from the cached manifest"
1411            );
1412        }
1413        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414            self.enforce_capability_denies();
1415        }
1416        self.refresh_capability_requirements();
1417    }
1418
1419    /// Point the shared supervisor handle at this handler for swap promotions.
1420    /// Called wherever a handler is put behind the `Arc` the router serves, so
1421    /// it can be held weakly.
1422    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425        self.supervisor.set_swap_promotion_observer(observer);
1426    }
1427
1428    pub fn refresh_capability_requirements(&self) {
1429        match self.runtime_capability_snapshot() {
1430            Ok((runtime, registrations)) => {
1431                log_requirement_events(
1432                    self.capability_evaluator
1433                        .evaluate_now(&runtime, &registrations),
1434                    &self.daemon_shutdown,
1435                );
1436            }
1437            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438        }
1439    }
1440
1441    /// Reconcile only live, attested route bindings after a capability deny edge
1442    /// or target claim was added. This is deliberately a control-plane census:
1443    /// the opaque forwarding hot path must not grow a per-frame capability check.
1444    fn enforce_capability_denies(&self) {
1445        let registrations = match self.registry.list_registered_modules() {
1446            Ok(snapshot) => snapshot,
1447            Err(err) => {
1448                warn!(error = %err, "failed to read registrations for capability deny census");
1449                return;
1450            }
1451        };
1452        let manifests = registrations
1453            .into_iter()
1454            .map(|registration| {
1455                (
1456                    registration.manifest.module_id.clone(),
1457                    registration.manifest,
1458                )
1459            })
1460            .collect::<BTreeMap<_, _>>();
1461        let census = match self.forwarding.route_census(None) {
1462            Ok(census) => census,
1463            Err(err) => {
1464                warn!(error = %err, "failed to read route census for capability deny enforcement");
1465                return;
1466            }
1467        };
1468
1469        for (target_module_id, routes) in census {
1470            let Some(target_manifest) = manifests.get(&target_module_id) else {
1471                continue;
1472            };
1473            let mut closed_routes = Vec::new();
1474            let mut module_goodbyes = Vec::new();
1475            for route in routes {
1476                let Principal::Reserved {
1477                    module_id: opening_module_id,
1478                } = &route.principal
1479                else {
1480                    continue;
1481                };
1482                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483                    continue;
1484                };
1485                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486                    continue;
1487                };
1488
1489                match self.forwarding.release_client_route(
1490                    route.goodbye_target.connection_id,
1491                    route.goodbye_target.channel,
1492                    route.goodbye_target.epoch,
1493                ) {
1494                    Ok(RouteRelease::Removed(module_goodbye)) => {
1495                        warn!(
1496                            opening_module_id,
1497                            target_module_id,
1498                            capability,
1499                            "force-closing route because an attested capability deny edge now matches"
1500                        );
1501                        closed_routes.push(route);
1502                        module_goodbyes.push(module_goodbye);
1503                    }
1504                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505                    Err(err) => warn!(
1506                        opening_module_id,
1507                        target_module_id,
1508                        capability,
1509                        error = %err,
1510                        "failed to force-close capability-denied route"
1511                    ),
1512                }
1513            }
1514
1515            if closed_routes.is_empty() {
1516                continue;
1517            }
1518            send_route_control_pushes(
1519                &self.forwarding,
1520                closed_routes,
1521                ClientControlPush::RouteClosed {
1522                    module_id: target_module_id,
1523                    channels: Vec::new(),
1524                    reason: RouteCloseReason::CapabilityDenied,
1525                    drained: false,
1526                    abandoned: 0,
1527                    excluded_subscriptions: 0,
1528                    terminal: Some(false),
1529                },
1530            );
1531            self.emit_route_goodbyes(module_goodbyes);
1532        }
1533    }
1534
1535    /// Why a registered module is not accepting new route binds, or `None` when
1536    /// it is. This is the module's effective readiness: its declared readiness
1537    /// first, then every `need: required` capability it declares evaluating to
1538    /// `provided`. `route.open` and `catalog.list` both read it here so the
1539    /// catalog never reports a module routable that `route.open` would refuse.
1540    fn not_ready_reason(
1541        &self,
1542        registration: &crate::registry::ModuleRegistration,
1543    ) -> Option<NotReadyReason> {
1544        if !registration.ready {
1545            return Some(NotReadyReason {
1546                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1547                capability: None,
1548            });
1549        }
1550        self.first_unprovided_required_capability(registration)
1551            .map(|capability| NotReadyReason {
1552                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1553                capability: Some(capability),
1554            })
1555    }
1556
1557    /// The lexicographically first capability this registration declares
1558    /// `need: required` whose evaluator verdict is not `provided`.
1559    ///
1560    /// The verdicts are the capability evaluator's own; nothing here decides
1561    /// what "provided" means. The evaluator counts a capability provided as
1562    /// soon as a module claiming it has REGISTERED, not once that module is
1563    /// ready. That distinction is what keeps two modules that require each
1564    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1565    /// is ready", each would wait for the other to become ready first and
1566    /// neither ever would. Do not tighten it to readiness.
1567    ///
1568    /// A required capability with no verdict at all means this registration's
1569    /// HELLO or catalog.update landed after the last recompute; recompute once
1570    /// rather than let a missing verdict read as either answer. If it is still
1571    /// missing (the recompute itself failed) the capability counts as
1572    /// unprovided: the refusal is retryable, and routing a module whose
1573    /// required provider is unknown is the outcome this check exists to stop.
1574    fn first_unprovided_required_capability(
1575        &self,
1576        registration: &crate::registry::ModuleRegistration,
1577    ) -> Option<String> {
1578        let required = registration
1579            .manifest
1580            .capabilities
1581            .iter()
1582            .flat_map(|declarations| declarations.requires.iter())
1583            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1584            .map(|requirement| requirement.capability.as_str())
1585            .collect::<BTreeSet<_>>();
1586        if required.is_empty() {
1587            return None;
1588        }
1589        let module_id = registration.manifest.module_id.as_str();
1590        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1591        if required
1592            .iter()
1593            .any(|capability| verdict(capability).is_none())
1594        {
1595            self.refresh_capability_requirements();
1596        }
1597        required
1598            .into_iter()
1599            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1600            .map(str::to_string)
1601    }
1602
1603    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1604        self.capability_evaluator
1605            .statuses()
1606            .into_iter()
1607            .map(capability_requirement_status)
1608            .collect()
1609    }
1610
1611    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1612    /// registration-release watch. The signal is what the supervisor waits on
1613    /// before spawning a replacement, so it must only fire once forwarding
1614    /// teardown is also done (see [`Self::cleanup_connection`] /
1615    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1616    /// admitted but its forwarding endpoint could not be installed.
1617    fn deregister_connection(
1618        &self,
1619        connection_id: ConnectionId,
1620        reason: RegistrationEndReason,
1621    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1622        self.registry
1623            .deregister_connection_with_reason(connection_id, reason)
1624    }
1625
1626    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1627        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1628            return None;
1629        }
1630        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1631            parse_client_control_request(&frame.body)
1632        else {
1633            return None;
1634        };
1635        Some(target_module_id(&target).to_string())
1636    }
1637
1638    pub(crate) fn route_open_capacity_refusal(
1639        &self,
1640        ctx: &RouteCtx,
1641        frame: &Frame,
1642        target_module_id: &str,
1643        in_flight: usize,
1644        limit: usize,
1645    ) -> Result<Frame, RouterError> {
1646        // Sent before the request reaches `handle_route_open`, so the caller
1647        // fields come from the raw frame here.
1648        let caller = self.route_open_caller_from_frame(frame);
1649        self.route_open_admission_refusal_frame(
1650            ctx,
1651            &caller,
1652            frame,
1653            target_module_id,
1654            "open_admission_full",
1655            (in_flight, limit),
1656            format!(
1657                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1658            ),
1659        )
1660    }
1661
1662    fn route_open_target_capacity_refusal(
1663        &self,
1664        ctx: &RouteCtx,
1665        caller: &RouteOpenCaller,
1666        frame: &Frame,
1667        target_module_id: &str,
1668        in_flight: usize,
1669    ) -> Result<Frame, RouterError> {
1670        self.route_open_admission_refusal_frame(
1671            ctx,
1672            caller,
1673            frame,
1674            target_module_id,
1675            "target_binds_full",
1676            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1677            format!(
1678                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1679            ),
1680        )
1681    }
1682
1683    /// Admission pressure clears as existing binds settle, so its refusal must
1684    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1685    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1686    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1687    /// cannot currently reach its target; `module_timeout` would falsely claim
1688    /// that a wait expired. A new, cleaner code would be terminal to deployed
1689    /// clients, so it requires a client-tolerance rollout before daemon emission.
1690    #[allow(clippy::too_many_arguments)]
1691    fn route_open_admission_refusal_frame(
1692        &self,
1693        ctx: &RouteCtx,
1694        caller: &RouteOpenCaller,
1695        frame: &Frame,
1696        target_module_id: &str,
1697        reason: &'static str,
1698        (in_flight, limit): (usize, usize),
1699        message: impl Into<String>,
1700    ) -> Result<Frame, RouterError> {
1701        let code = error_codes::TARGET_UNAVAILABLE;
1702        self.counters.increment_route_open_refused(code);
1703        info!(
1704            target: "control",
1705            code,
1706            reason,
1707            module_id = ?target_module_id,
1708            connection_id = ctx.connection_id.get(),
1709            principal = %caller.principal,
1710            scope = %caller.scope,
1711            harness = %caller.harness,
1712            in_flight,
1713            limit,
1714            "route.open refused"
1715        );
1716        control_error_frame(frame, code, message.into())
1717    }
1718
1719    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1720    ///
1721    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1722    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1723    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1724    #[cfg(test)]
1725    pub fn handle_control(
1726        &self,
1727        connection_id: ConnectionId,
1728        frame: Frame,
1729    ) -> Result<Vec<Frame>, RouterError> {
1730        match frame.header.ty {
1731            FrameType::Ping => Ok(vec![pong(&frame)?]),
1732            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1733            FrameType::Goodbye => self.handle_goodbye(connection_id),
1734            ty => Ok(vec![control_error_frame(
1735                &frame,
1736                "unsupported_control_frame",
1737                format!("unsupported channel-0 frame {ty:?}"),
1738            )?]),
1739        }
1740    }
1741
1742    pub async fn handle_control_frame(
1743        &self,
1744        ctx: &RouteCtx,
1745        frame: Frame,
1746    ) -> Result<Vec<Frame>, RouterError> {
1747        self.handle_control_frame_timed(ctx, frame, None).await
1748    }
1749
1750    pub(crate) fn record_caller_endpoints(
1751        &self,
1752        connection: ConnectionId,
1753        endpoints: Option<(SocketAddr, SocketAddr)>,
1754    ) {
1755        let mut callers = self
1756            .caller_endpoints
1757            .lock()
1758            .unwrap_or_else(|p| p.into_inner());
1759        callers.remove(&connection);
1760        if let Some(endpoints) = endpoints {
1761            callers.insert(connection, endpoints);
1762        }
1763    }
1764
1765    #[cfg(test)]
1766    pub(crate) fn with_caller_lookup_for_test(
1767        mut self,
1768        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1769    ) -> Self {
1770        self.caller_lookup = Some(Arc::new(lookup));
1771        self
1772    }
1773
1774    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1775        let endpoints = self
1776            .caller_endpoints
1777            .lock()
1778            .unwrap_or_else(|p| p.into_inner())
1779            .get(&connection)
1780            .copied();
1781        #[cfg(test)]
1782        let lookup = self.caller_lookup.clone();
1783        tokio::task::spawn_blocking(move || {
1784            #[cfg(test)]
1785            if let Some(lookup) = lookup {
1786                return lookup(endpoints);
1787            }
1788            endpoints
1789                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1790                .unwrap_or_default()
1791        })
1792        .await
1793        .unwrap_or_default()
1794    }
1795
1796    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1797        frame.header.channel == 0
1798            && frame.header.ty == FrameType::Request
1799            && self
1800                .forwarding
1801                .module_endpoint_for_connection(ctx.connection_id)
1802                .is_ok_and(|endpoint| endpoint.is_none())
1803            && parse_client_control_request(&frame.body)
1804                .ok()
1805                .is_some_and(|request| privileged_operation(&request).is_some())
1806    }
1807
1808    pub(crate) async fn handle_control_frame_timed(
1809        &self,
1810        ctx: &RouteCtx,
1811        frame: Frame,
1812        dispatch_started_at: Option<StdInstant>,
1813    ) -> Result<Vec<Frame>, RouterError> {
1814        match frame.header.ty {
1815            FrameType::Ping => Ok(vec![pong(&frame)?]),
1816            FrameType::Hello => {
1817                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1818            }
1819            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1820            FrameType::Cancel => {
1821                // A Cancel on channel 0 names a waiting operator.confirm or a
1822                // subscription. Successful cancellation has no ordinary reply.
1823                if self
1824                    .forwarding
1825                    .operator_confirms()
1826                    .cancel(ctx.connection_id, frame.header.corr)
1827                    || self
1828                        .supervisor
1829                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1830                    || self.cancel_activity_subscription(ctx, &frame)?
1831                {
1832                    Ok(Vec::new())
1833                } else {
1834                    Ok(vec![control_error_frame(
1835                        &frame,
1836                        "unknown_subscription",
1837                        "no subscription has this correlation id",
1838                    )?])
1839                }
1840            }
1841            FrameType::Request => {
1842                // This additive operation is not part of the existing exhaustive
1843                // module-control enum. Probe the op before decoding that enum.
1844                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1845                if op
1846                    .as_ref()
1847                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1848                {
1849                    return self.handle_activity_record(ctx, frame).await;
1850                }
1851                if op.as_ref().is_some_and(|probe| {
1852                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1853                }) {
1854                    return self.handle_activity_read(ctx, frame).await;
1855                }
1856                if op
1857                    .as_ref()
1858                    .is_some_and(|probe| probe.op == "operator.confirm")
1859                {
1860                    return self.handle_operator_confirm(ctx, frame);
1861                }
1862                if op
1863                    .as_ref()
1864                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1865                {
1866                    return self.handle_scope_served_by(ctx.connection_id, frame);
1867                }
1868                if self
1869                    .forwarding
1870                    .module_endpoint_for_connection(ctx.connection_id)
1871                    .map_err(RouterError::Forwarding)?
1872                    .is_some()
1873                {
1874                    if !is_known_module_request_op(&frame.body) {
1875                        return Ok(vec![control_error_frame(
1876                            &frame,
1877                            "unsupported_control_frame",
1878                            "module-originated channel-0 REQUEST is not supported",
1879                        )?]);
1880                    }
1881                    let request = match parse_module_control_request_from_module(&frame.body) {
1882                        Ok(request) => request,
1883                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1884                            return Ok(vec![control_error_frame(
1885                                &frame,
1886                                "unsupported_control_frame",
1887                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1888                            )?])
1889                        }
1890                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1891                            return Ok(vec![control_error_frame(
1892                                &frame,
1893                                "invalid_control_body",
1894                                format!("malformed module control body: {err}"),
1895                            )?])
1896                        }
1897                    };
1898                    let op = module_control_request_op(&request);
1899                    let corr = frame.header.corr;
1900                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1901                    let result =
1902                        self.handle_module_control_request(ctx.connection_id, frame, request);
1903                    log_slow_control_dispatch(
1904                        dispatch_started_at,
1905                        op,
1906                        ctx.connection_id,
1907                        corr,
1908                        None,
1909                    );
1910                    return result;
1911                }
1912
1913                if is_known_module_request_op(&frame.body) {
1914                    return Ok(vec![control_error_frame(
1915                        &frame,
1916                        "not_registered",
1917                        "catalog.update requires an active module registration owned by this connection",
1918                    )?]);
1919                }
1920
1921                let request = match parse_client_control_request(&frame.body) {
1922                    Ok(request) => request,
1923                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1924                        return Ok(vec![control_error_frame(
1925                            &frame,
1926                            "unknown_control_op",
1927                            format!("unknown client control op: {err}"),
1928                        )?])
1929                    }
1930                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1931                        return Ok(vec![control_error_frame(
1932                            &frame,
1933                            "invalid_control_body",
1934                            format!("malformed client control body: {err}"),
1935                        )?])
1936                    }
1937                };
1938                let op = client_control_request_op(&request);
1939                let corr = frame.header.corr;
1940                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1941                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1942                #[cfg(test)]
1943                if let Some(delay) = self.control_dispatch_delay {
1944                    tokio::time::sleep(delay).await;
1945                }
1946                let result = self
1947                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1948                    .await;
1949                if let Some(timings) = route_open_timings.as_mut() {
1950                    timings.finish();
1951                }
1952                log_slow_control_dispatch(
1953                    dispatch_started_at,
1954                    op,
1955                    ctx.connection_id,
1956                    corr,
1957                    route_open_timings.as_ref(),
1958                );
1959                result
1960            }
1961            FrameType::Push => {
1962                let Some(endpoint) = self
1963                    .forwarding
1964                    .module_endpoint_for_connection(ctx.connection_id)
1965                    .map_err(RouterError::Forwarding)?
1966                else {
1967                    return Ok(vec![control_error_frame(
1968                        &frame,
1969                        "unsupported_control_frame",
1970                        "client-originated channel-0 PUSH is not supported",
1971                    )?]);
1972                };
1973                self.handle_status_update(endpoint, frame)
1974            }
1975            FrameType::Response | FrameType::Error
1976                if self
1977                    .forwarding
1978                    .module_endpoint_for_connection(ctx.connection_id)
1979                    .map_err(RouterError::Forwarding)?
1980                    .is_some() =>
1981            {
1982                self.handle_module_relay_response(ctx.connection_id, frame)
1983            }
1984            ty => Ok(vec![control_error_frame(
1985                &frame,
1986                "unsupported_control_frame",
1987                format!("unsupported channel-0 frame {ty:?}"),
1988            )?]),
1989        }
1990    }
1991
1992    pub fn cleanup_connection(
1993        &self,
1994        connection_id: ConnectionId,
1995    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1996        self.caller_endpoints
1997            .lock()
1998            .unwrap_or_else(|p| p.into_inner())
1999            .remove(&connection_id);
2000        self.forwarding
2001            .operator_confirms()
2002            .connection_closed(connection_id);
2003        self.cleanup_connection_with_end_reason(
2004            connection_id,
2005            RegistrationEndReason::ConnectionClosed,
2006        )
2007    }
2008
2009    fn cleanup_connection_with_end_reason(
2010        &self,
2011        connection_id: ConnectionId,
2012        requested_reason: RegistrationEndReason,
2013    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2014        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2015            self.registry
2016                .get_module_by_connection(connection_id)?
2017                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2018                .and_then(|module| module.registration_end_reason().ok().flatten())
2019                .unwrap_or(requested_reason)
2020        } else {
2021            requested_reason
2022        };
2023        let crash_closed = self
2024            .registry
2025            .get_module_by_connection(connection_id)?
2026            .and_then(|registration| {
2027                self.forwarding
2028                    .module_endpoint_for_connection(connection_id)
2029                    .ok()
2030                    .flatten()
2031                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2032                    .map(|routes| (registration.manifest.module_id, routes))
2033            });
2034        let crash_closed = crash_closed.map(|(module_id, routes)| {
2035            let terminal = match self.supervisor.get(&module_id) {
2036                None => false,
2037                Some(module) => match module.will_recover_after_connection_loss() {
2038                    Ok(will_recover) => !will_recover,
2039                    Err(err) => {
2040                        warn!(
2041                            %module_id,
2042                            error = %err,
2043                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2044                        );
2045                        false
2046                    }
2047                },
2048            };
2049            // The forwarding table gates all providers at the start of daemon
2050            // shutdown, before their connections are closed. An ordinary
2051            // module disconnect still reports crash if that gate is not set.
2052            let (reason, drained) = match self
2053                .forwarding
2054                .daemon_drain_result_for_connection(connection_id)
2055            {
2056                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2057                Ok(None) => (RouteCloseReason::Crash, false),
2058                Err(err) => {
2059                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2060                    (RouteCloseReason::Crash, false)
2061                }
2062            };
2063            (module_id, routes, reason, drained, terminal)
2064        });
2065        let registrations = self.deregister_connection(connection_id, end_reason);
2066        let cleanup = if crash_closed.is_some() {
2067            self.forwarding.cleanup_connection_counted(connection_id)
2068        } else {
2069            // A module connection's teardown needs the count of abandoned
2070            // route.bind relays for its route.closed notice below. Any other
2071            // connection, such as a client's, sends no such notice and needs
2072            // only its routes released, so it uses the route-only wrapper and
2073            // reports zero.
2074            self.forwarding
2075                .cleanup_connection(connection_id)
2076                .map(|released| crate::forwarding::ConnectionCleanup {
2077                    released,
2078                    abandoned_relays: 0,
2079                })
2080        };
2081        // The route.closed push waits for forwarding teardown because only
2082        // teardown knows how many pending route.bind relays it aborted. It still
2083        // goes out before the GOODBYEs for the released routes, and its targets
2084        // were captured above, before teardown removed those routes.
2085        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2086            let abandoned = cleanup
2087                .as_ref()
2088                .map_or(0, |cleanup| cleanup.abandoned_relays);
2089            send_route_control_pushes(
2090                &self.forwarding,
2091                routes,
2092                ClientControlPush::RouteClosed {
2093                    module_id,
2094                    channels: Vec::new(),
2095                    reason,
2096                    drained,
2097                    abandoned,
2098                    excluded_subscriptions: 0,
2099                    terminal: Some(terminal),
2100                },
2101            );
2102        }
2103        if let Ok(cleanup) = cleanup {
2104            self.emit_route_goodbyes(cleanup.released);
2105        }
2106        // Signal the registration-release watch only now that BOTH registry and
2107        // forwarding teardown are done, so a supervisor waiting to spawn a
2108        // replacement never observes release while old routes still exist.
2109        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2110            crate::supervise::notify_registration_release();
2111            self.capability_evaluator.wake_deadline_loop();
2112            self.refresh_capability_requirements();
2113        }
2114        self.supervisor.remove_spawn_subscribers(connection_id);
2115        self.activity_streams
2116            .lock()
2117            .unwrap_or_else(|p| p.into_inner())
2118            .retain(|(id, _), task| {
2119                if *id == connection_id {
2120                    task.abort();
2121                    false
2122                } else {
2123                    true
2124                }
2125            });
2126        // Sync authority dies with its connection, so the owner's next
2127        // connection can take it; the owner's scopes stay as they are.
2128        self.hello_launch_nonces
2129            .lock()
2130            .unwrap_or_else(|poisoned| poisoned.into_inner())
2131            .forget(connection_id);
2132        self.scopes
2133            .write()
2134            .unwrap_or_else(|poisoned| poisoned.into_inner())
2135            .release_connection(connection_id);
2136        registrations
2137    }
2138
2139    pub(crate) fn handle_route_goodbye(
2140        &self,
2141        connection_id: ConnectionId,
2142        route_channel: u16,
2143        route_epoch: u32,
2144    ) -> Result<bool, RouterError> {
2145        debug!(
2146            connection_id = connection_id.get(),
2147            route_channel, route_epoch, "handling route GOODBYE"
2148        );
2149        let RouteRelease::Removed(released_route) = self
2150            .forwarding
2151            .release_client_route(connection_id, route_channel, route_epoch)
2152            .map_err(RouterError::Forwarding)?
2153        else {
2154            return Ok(false);
2155        };
2156        self.emit_route_goodbyes(vec![released_route]);
2157        Ok(true)
2158    }
2159
2160    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2161        for released in released_routes {
2162            let frame = match Frame::build_with_version(
2163                released.negotiated_ver,
2164                FrameType::Goodbye,
2165                control_flags(),
2166                released.channel,
2167                released.epoch,
2168                0,
2169                Vec::new(),
2170            ) {
2171                Ok(frame) => frame,
2172                Err(err) => {
2173                    warn!(
2174                        route_channel = released.channel,
2175                        error = %err,
2176                        "failed to build route GOODBYE frame"
2177                    );
2178                    continue;
2179                }
2180            };
2181            if !released.close_on_delivery_failure() {
2182                crate::forwarding::send_module_route_goodbye(
2183                    &self.counters,
2184                    &released.sink,
2185                    frame,
2186                    released.module_id.as_deref(),
2187                    "client route released",
2188                );
2189                continue;
2190            }
2191            if let Err(err) = released.sink.try_send(frame) {
2192                warn!(
2193                    target_connection_id = released.connection_id.get(),
2194                    route_channel = released.channel,
2195                    error = %err,
2196                    "route GOODBYE was not delivered to client; closing target connection"
2197                );
2198                if self
2199                    .forwarding
2200                    .escalate_client_delivery_failure(
2201                        released.connection_id,
2202                        released.channel,
2203                        released.epoch,
2204                        CloseReason::new(
2205                            "route_goodbye_delivery_failed",
2206                            format!(
2207                                "failed to enqueue route GOODBYE for channel {}: {err}",
2208                                released.channel
2209                            ),
2210                        ),
2211                        crate::forwarding::UndeliveredFrame {
2212                            module_id: released.module_id.as_deref(),
2213                            sink: &released.sink,
2214                        },
2215                    )
2216                    .unwrap_or(false)
2217                {
2218                    self.counters.increment_goodbye_relay_client_failed();
2219                }
2220            }
2221        }
2222    }
2223
2224    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2225    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2226    /// own commit failed after the module had already accepted). Without this, a
2227    /// module that accepts late keeps a binding subc has torn down, so a later
2228    /// frame on that module channel could misdeliver if the channel is reused.
2229    ///
2230    /// Never closes the shared module connection on failure: a dropped notification
2231    /// only wastes a bounded amount of warm module-side state, which the module's
2232    /// own idle reaper reclaims. Only call this once the route.bind relay was
2233    /// actually enqueued to the module — if the relay send itself failed, the
2234    /// module never created a binding and there is nothing to tear down.
2235    fn send_abandoned_route_bind_goodbye(
2236        &self,
2237        module_sink: &crate::FrameSink,
2238        negotiated_ver: u8,
2239        module_channel: u16,
2240        module_epoch: u32,
2241    ) {
2242        let frame = match Frame::build_with_version(
2243            negotiated_ver,
2244            FrameType::Goodbye,
2245            control_flags(),
2246            module_channel,
2247            module_epoch,
2248            0,
2249            Vec::new(),
2250        ) {
2251            Ok(frame) => frame,
2252            Err(err) => {
2253                warn!(
2254                    route_channel = module_channel,
2255                    error = %err,
2256                    "failed to build GOODBYE for abandoned route.bind"
2257                );
2258                return;
2259            }
2260        };
2261        crate::forwarding::send_module_route_goodbye(
2262            &self.counters,
2263            module_sink,
2264            frame,
2265            None,
2266            "abandoned route.bind",
2267        );
2268    }
2269
2270    fn handle_hello(
2271        &self,
2272        connection_id: ConnectionId,
2273        sink: Option<crate::FrameSink>,
2274        frame: Frame,
2275    ) -> Result<Vec<Frame>, RouterError> {
2276        debug!(
2277            connection_id = connection_id.get(),
2278            corr = frame.header.corr,
2279            "handling HELLO"
2280        );
2281        // A module connection has one identity for its entire lifetime. A second
2282        // registration would leave the old registry owner behind while replacing
2283        // its forwarding endpoint and launch nonce.
2284        if self
2285            .registry
2286            .get_module_by_connection(connection_id)
2287            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2288            .is_some()
2289        {
2290            return Ok(vec![control_error_frame(
2291                &frame,
2292                "invalid_hello",
2293                "connection is already registered as a module",
2294            )?]);
2295        }
2296        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2297            Ok(value) => value,
2298            Err(err) => {
2299                return Ok(vec![control_error_frame(
2300                    &frame,
2301                    "invalid_hello",
2302                    format!("malformed HELLO body: {err}"),
2303                )?])
2304            }
2305        };
2306        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2307            return Ok(vec![control_error_frame(
2308                &frame,
2309                "invalid_capability_grammar",
2310                err.to_string(),
2311            )?]);
2312        }
2313        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2314            return Ok(vec![control_error_frame(
2315                &frame,
2316                "invalid_manifest",
2317                err.to_string(),
2318            )?]);
2319        }
2320        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2321            return Ok(vec![control_error_frame(
2322                &frame,
2323                "invalid_event_declaration",
2324                err.to_string(),
2325            )?]);
2326        }
2327        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2328            return Ok(vec![control_error_frame(
2329                &frame,
2330                "invalid_manifest",
2331                err.to_string(),
2332            )?]);
2333        }
2334        if let Some(provenance) = hello_value
2335            .get("manifest")
2336            .and_then(|manifest| manifest.get("provenance"))
2337        {
2338            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2339                return Ok(vec![control_error_frame(
2340                    &frame,
2341                    "invalid_manifest",
2342                    format!("malformed manifest provenance: {err}"),
2343                )?]);
2344            }
2345        }
2346        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2347            let module_id = hello_value["manifest"]["module_id"]
2348                .as_str()
2349                .unwrap_or("<unknown>");
2350            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2351        }
2352        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2353            Ok(hello) => hello,
2354            Err(err) => {
2355                return Ok(vec![control_error_frame(
2356                    &frame,
2357                    "invalid_hello",
2358                    format!("malformed HELLO body: {err}"),
2359                )?])
2360            }
2361        };
2362
2363        if hello.protocol_ver != hello.manifest.protocol_ver {
2364            return Ok(vec![control_error_frame(
2365                &frame,
2366                "invalid_manifest",
2367                format!(
2368                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2369                    hello.protocol_ver, hello.manifest.protocol_ver
2370                ),
2371            )?]);
2372        }
2373
2374        if hello.manifest.module_id.trim().is_empty() {
2375            return Ok(vec![control_error_frame(
2376                &frame,
2377                "invalid_manifest",
2378                "manifest module_id must not be empty",
2379            )?]);
2380        }
2381
2382        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2383            Ok(negotiated_ver) => negotiated_ver,
2384            Err(message) => {
2385                return Ok(vec![control_error_frame(
2386                    &frame,
2387                    "version_unsupported",
2388                    message,
2389                )?])
2390            }
2391        };
2392
2393        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2394        // swap is open for this id, the only HELLO admitted as a second process
2395        // is the one carrying the candidate's launch nonce (the swap token), and
2396        // it registers into the candidate slot rather than being refused as a
2397        // duplicate. Run after the reserved gate, a reserved module's candidate
2398        // would be refused `reserved_module` for presenting a nonce that gate
2399        // does not know. See `SupervisorHandle::swap_hello_admission`.
2400        let swap_admission = self
2401            .supervisor
2402            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2403        if swap_admission == SwapHelloAdmission::Refused {
2404            warn!(
2405                module_id = %hello.manifest.module_id,
2406                connection_id = connection_id.get(),
2407                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2408            );
2409            return Ok(vec![control_error_frame(
2410                &frame,
2411                "swap_token_invalid",
2412                format!(
2413                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2414                    hello.manifest.module_id
2415                ),
2416            )?]);
2417        }
2418        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2419
2420        // A swap candidate proves its identity with the candidate nonce above.
2421        // Other registrations receive this catalog fact only when reserved-module
2422        // policy required and accepted a nonce; unreserved registrations require
2423        // no nonce check.
2424        let nonce_verified = if swap_candidate {
2425            true
2426        } else {
2427            match self.supervisor.reserved_hello_verification(
2428                &hello.manifest.module_id,
2429                hello.launch_nonce.as_deref(),
2430            ) {
2431                Ok(verified) => verified,
2432                Err(rejection) => {
2433                    let message = match rejection {
2434                        ReservedHelloRejection::Exact { module_id } => format!(
2435                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2436                        ),
2437                        ReservedHelloRejection::Prefix {
2438                            prefix,
2439                            owner_module_id,
2440                        } => format!(
2441                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2442                            hello.manifest.module_id
2443                        ),
2444                    };
2445                    return Ok(vec![control_error_frame(
2446                        &frame,
2447                        "reserved_module",
2448                        message,
2449                    )?]);
2450                }
2451            }
2452        };
2453
2454        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2455            &hello.manifest.module_id,
2456            hello.manifest.capabilities.as_ref(),
2457        );
2458        if let Some(refusal) = reserved_capability_refusals.first() {
2459            let capability = refusal.capability.clone();
2460            let bound_module = refusal.claimants[0].clone();
2461            log_duplicate_claim_events(reserved_capability_refusals);
2462            return Ok(vec![control_error_frame(
2463                &frame,
2464                "reserved_capability",
2465                format!(
2466                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2467                    capability, bound_module, hello.manifest.module_id
2468                ),
2469            )?]);
2470        }
2471
2472        // A connection that already opened client routes must not also register as
2473        // a module: cleanup would then release only one side and leak the other.
2474        if self
2475            .forwarding
2476            .connection_has_client_routes(connection_id)
2477            .map_err(RouterError::Forwarding)?
2478        {
2479            return Ok(vec![control_error_frame(
2480                &frame,
2481                "invalid_hello",
2482                "connection has open client routes and cannot also register as a module",
2483            )?]);
2484        }
2485
2486        // Kept for scope sync authority, which goes only to the connection that
2487        // presented the module's current launch nonce. Recorded before the
2488        // registration is attempted: a connection whose registration then fails
2489        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2490        self.hello_launch_nonces
2491            .lock()
2492            .unwrap_or_else(|poisoned| poisoned.into_inner())
2493            .record(connection_id, hello.launch_nonce.as_deref());
2494        let control_ops = effective_module_control_ops(hello.control_ops);
2495        // Built before anything is registered so an encoding failure leaves no
2496        // registry or forwarding state behind.
2497        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2498        if swap_candidate {
2499            return self.register_swap_candidate(
2500                connection_id,
2501                sink,
2502                &frame,
2503                hello.manifest,
2504                negotiated_ver,
2505                control_ops,
2506                hello_ack,
2507            );
2508        }
2509        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2510        // is refused, but `catalog.list` does not show the module and the
2511        // catalog generation does not move until it is published below, once
2512        // its forwarding endpoint exists. Listing it now would advertise a
2513        // module that `route.open` still refuses for having no forwarding
2514        // connection.
2515        let registration = match self
2516            .registry
2517            .register_unpublished_verified_with_control_ops(
2518                hello.manifest,
2519                negotiated_ver,
2520                connection_id,
2521                control_ops,
2522                nonce_verified,
2523            ) {
2524            Ok(registration) => registration,
2525            Err(RegistryError::DuplicateModuleId { module_id }) => {
2526                return Ok(vec![control_error_frame(
2527                    &frame,
2528                    "duplicate_module_id",
2529                    format!(
2530                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2531                    ),
2532                )?])
2533            }
2534            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2535                return Ok(vec![control_error_frame(
2536                    &frame,
2537                    "invalid_module_id",
2538                    err.to_string(),
2539                )?])
2540            }
2541            Err(err) => {
2542                return Ok(vec![control_error_frame(
2543                    &frame,
2544                    "registry_error",
2545                    err.to_string(),
2546                )?])
2547            }
2548        };
2549        #[cfg(test)]
2550        self.hold_between_registry_and_forwarding();
2551
2552        let reply = if let Some(sink) = sink {
2553            // The forwarding table's module store is also the daemon-to-module
2554            // control-RPC lane, so every HELLO gets a live endpoint even when the
2555            // manifest has no routable provider role. Non-routable modules still
2556            // cannot receive route.bind in production: `handle_route_open` checks
2557            // the registry manifest with `target_has_required_role` before the
2558            // only production call to `begin_route_bind_relay_for` below that
2559            // route.open path. The remaining direct relay callers are unit tests
2560            // and benchmark harnesses that construct forwarding state explicitly.
2561            //
2562            // The HELLO_ACK is queued by the forwarding table itself, before the
2563            // endpoint becomes visible, and is NOT returned as a reply. A module
2564            // reads HELLO_ACK first and exits on anything else; a reply is only
2565            // written after this handler returns, by which time a route.open on
2566            // another connection could already have queued a route.bind request
2567            // for this module ahead of it.
2568            let concurrency = manifest_concurrency(&registration.manifest);
2569            if let Err(err) = self.forwarding.register_module_connection_acked(
2570                connection_id,
2571                registration.manifest.module_id.clone(),
2572                negotiated_ver,
2573                concurrency,
2574                sink,
2575                hello_ack,
2576            ) {
2577                // Forwarding registration failed, so there is no forwarding
2578                // state to tear down. Remove the registry entry, which was never
2579                // published and so never listed or counted in the catalog
2580                // generation, and signal the release watch directly.
2581                if matches!(
2582                    self.deregister_connection(
2583                        connection_id,
2584                        RegistrationEndReason::RegistrationFailed,
2585                    ),
2586                    Ok(r) if !r.is_empty()
2587                ) {
2588                    crate::supervise::notify_registration_release();
2589                }
2590                return Ok(vec![control_error_frame(
2591                    &frame,
2592                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2593                        "invalid_hello"
2594                    } else {
2595                        forwarding_error_code(&err)
2596                    },
2597                    err.to_string(),
2598                )?]);
2599            }
2600            Vec::new()
2601        } else {
2602            // No sink means no forwarding endpoint, so nothing can be routed
2603            // ahead of the ack; it goes out as the reply.
2604            vec![hello_ack]
2605        };
2606
2607        // The module is routable now (or, without a sink, as routable as it
2608        // will ever be), so the catalog may list it. Publishing bumps the
2609        // catalog generation in the same registry critical section that makes
2610        // the module listed.
2611        if !self
2612            .registry
2613            .publish_registration(connection_id)
2614            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2615        {
2616            debug!(
2617                module_id = %registration.manifest.module_id,
2618                connection_id = connection_id.get(),
2619                "registration was removed before it could be published"
2620            );
2621        }
2622
2623        // Exposure over assumption: Concurrency's serde default is pinned to the
2624        // pre-field behavior (ModuleManaged), so a management surface that is
2625        // genuinely Serial and just never declared it inherits concurrent
2626        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2627        // turns "no module has been bitten yet" into the checkable claim "no
2628        // module is exposed" -- one read of the boot log instead of a fleet
2629        // audit. Detected from the raw HELLO bytes because the serde default
2630        // deliberately erases the absent/declared distinction from the type.
2631        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2632            info!(
2633                module_id = %registration.manifest.module_id,
2634                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2635            );
2636        }
2637
2638        self.apply_registration_capabilities(&registration);
2639
2640        self.supervisor
2641            .record_activity_registration(&registration.manifest.module_id);
2642
2643        info!(
2644            module_id = %registration.manifest.module_id,
2645            module_version = %registration.manifest.module_version,
2646            negotiated_ver,
2647            routable_provider = manifest_provides_routable_role(&registration.manifest),
2648            connection_id = connection_id.get(),
2649            "module registered"
2650        );
2651
2652        Ok(reply)
2653    }
2654
2655    /// Register a HELLO the swap gate admitted into the candidate slot of the
2656    /// registry and of forwarding, where it is reachable over its own
2657    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2658    /// nothing routes to it until the supervisor cuts over.
2659    ///
2660    /// Registry first (unpublished), then forwarding, then publication, the
2661    /// same order as an ordinary HELLO; a forwarding failure removes the
2662    /// registry entry again. The swap supervisor treats a published, ready
2663    /// candidate as registered and promotes it, and promotion needs the
2664    /// candidate's forwarding endpoint, so the candidate is published only
2665    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2666    /// recompute capability verdicts or run the capability deny census
2667    /// (`enforce_capability_denies`): both are about the module in the active
2668    /// slot, which `route.open` can target. A candidate sits in the candidate
2669    /// slot, so it neither provides capabilities to other modules nor can be
2670    /// routed to until promotion moves it into the active slot.
2671    #[allow(clippy::too_many_arguments)]
2672    fn register_swap_candidate(
2673        &self,
2674        connection_id: ConnectionId,
2675        sink: Option<crate::FrameSink>,
2676        frame: &Frame,
2677        manifest: ModuleManifest,
2678        negotiated_ver: u8,
2679        control_ops: Vec<String>,
2680        hello_ack: Frame,
2681    ) -> Result<Vec<Frame>, RouterError> {
2682        let module_id = manifest.module_id.clone();
2683        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2684            manifest,
2685            negotiated_ver,
2686            connection_id,
2687            control_ops,
2688        ) {
2689            Ok(registration) => registration,
2690            Err(RegistryError::DuplicateModuleId { module_id }) => {
2691                return Ok(vec![control_error_frame(
2692                    frame,
2693                    "duplicate_module_id",
2694                    format!(
2695                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2696                    ),
2697                )?])
2698            }
2699            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2700                return Ok(vec![control_error_frame(
2701                    frame,
2702                    "invalid_module_id",
2703                    err.to_string(),
2704                )?])
2705            }
2706            Err(err) => {
2707                return Ok(vec![control_error_frame(
2708                    frame,
2709                    "registry_error",
2710                    err.to_string(),
2711                )?])
2712            }
2713        };
2714        #[cfg(test)]
2715        self.hold_between_registry_and_forwarding();
2716        let reply = if let Some(sink) = sink {
2717            // Same ordering as an ordinary HELLO: the forwarding table queues
2718            // the HELLO_ACK before the candidate endpoint is inserted, because
2719            // a module exits if its first frame after HELLO is anything else.
2720            let concurrency = manifest_concurrency(&registration.manifest);
2721            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2722                connection_id,
2723                module_id.clone(),
2724                negotiated_ver,
2725                concurrency,
2726                sink,
2727                hello_ack,
2728            ) {
2729                if matches!(
2730                    self.deregister_connection(
2731                        connection_id,
2732                        RegistrationEndReason::RegistrationFailed,
2733                    ),
2734                    Ok(r) if !r.is_empty()
2735                ) {
2736                    crate::supervise::notify_registration_release();
2737                }
2738                return Ok(vec![control_error_frame(
2739                    frame,
2740                    forwarding_error_code(&err),
2741                    err.to_string(),
2742                )?]);
2743            }
2744            Vec::new()
2745        } else {
2746            vec![hello_ack]
2747        };
2748        if !self
2749            .registry
2750            .publish_registration(connection_id)
2751            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2752        {
2753            debug!(
2754                module_id = %module_id,
2755                connection_id = connection_id.get(),
2756                "swap candidate registration was removed before it could be published"
2757            );
2758        }
2759        self.supervisor.mark_swap_candidate_admitted(&module_id);
2760        info!(
2761            module_id = %module_id,
2762            module_version = %registration.manifest.module_version,
2763            negotiated_ver,
2764            ready = registration.ready,
2765            connection_id = connection_id.get(),
2766            "swap candidate registered; not routable until cutover"
2767        );
2768        Ok(reply)
2769    }
2770
2771    fn build_hello_ack(
2772        &self,
2773        frame: &Frame,
2774        negotiated_ver: u8,
2775        module_id: &str,
2776    ) -> Result<Frame, RouterError> {
2777        let ack = ModuleHelloAckBody {
2778            negotiated_ver,
2779            subc_ops: module_subc_ops(),
2780            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2781            storage: self
2782                .storage_config
2783                .as_ref()
2784                .map(|cfg| cfg.descriptor_for(module_id)),
2785            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2786        };
2787        let body = serde_json::to_vec(&ack).map_err(|err| {
2788            RouterError::backend(
2789                0,
2790                frame.header.corr,
2791                format!("failed to encode HELLO_ACK: {err}"),
2792            )
2793        })?;
2794
2795        Frame::build_with_version(
2796            negotiated_ver,
2797            FrameType::HelloAck,
2798            control_flags(),
2799            0,
2800            0,
2801            frame.header.corr,
2802            body,
2803        )
2804        .map_err(RouterError::FrameBuild)
2805    }
2806
2807    async fn handle_client_control_request(
2808        &self,
2809        ctx: &RouteCtx,
2810        frame: Frame,
2811        request: ClientControlRequest,
2812        route_open_timings: &mut Option<RouteOpenTimings>,
2813    ) -> Result<Vec<Frame>, RouterError> {
2814        if let Some(operation) = privileged_operation(&request) {
2815            let caller = self.resolve_caller(ctx.connection_id).await;
2816            let op = client_control_request_op(&request);
2817            let text = prompt_text(operation, &caller);
2818            match self.privileged_operator_prompt {
2819                PrivilegedOperatorPrompt::Log => {
2820                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2821                        target = ?operation, caller = %caller.description(), summary = %text,
2822                        "privileged request would prompt the operator");
2823                }
2824                PrivilegedOperatorPrompt::Enforce => {
2825                    let key =
2826                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2827                    let outcome = self
2828                        .forwarding
2829                        .operator_confirms()
2830                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2831                        .await;
2832                    if outcome.code != "confirmed" {
2833                        return Ok(vec![outcome.frame(&frame)]);
2834                    }
2835                }
2836            }
2837        }
2838        match request {
2839            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2840            ClientControlRequest::CatalogList { module_id } => {
2841                self.handle_catalog_list(frame, module_id)
2842            }
2843            ClientControlRequest::RouteOpen {
2844                target,
2845                identity,
2846                consumer_identity,
2847                consumer_capabilities,
2848                role_versions,
2849                admission_facts,
2850                scope,
2851            } => {
2852                self.handle_route_open(
2853                    ctx,
2854                    frame,
2855                    RouteOpenRequest {
2856                        target,
2857                        identity,
2858                        consumer_identity,
2859                        consumer_capabilities,
2860                        role_versions,
2861                        admission_facts,
2862                        scope,
2863                    },
2864                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2865                )
2866                .await
2867            }
2868            ClientControlRequest::RoutePoll {
2869                route_channel,
2870                route_epoch,
2871                kind,
2872            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2873            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2874            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2875                self.handle_supervisor_spawn_snapshot(frame)
2876            }
2877            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2878                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2879            }
2880            ClientControlRequest::SupervisorRestart {
2881                module_id,
2882                drain_timeout_ms,
2883            } => {
2884                self.log_supervisor_request_received(
2885                    ctx,
2886                    frame.header.corr,
2887                    ops::SUPERVISOR_RESTART,
2888                    Some(&module_id),
2889                    None,
2890                )?;
2891                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2892                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2893                    .await
2894            }
2895            ClientControlRequest::SupervisorSwap {
2896                module_id,
2897                ready_timeout_ms,
2898            } => {
2899                self.log_supervisor_request_received(
2900                    ctx,
2901                    frame.header.corr,
2902                    ops::SUPERVISOR_SWAP,
2903                    Some(&module_id),
2904                    None,
2905                )?;
2906                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2907                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2908                    .await
2909            }
2910            ClientControlRequest::SupervisorReload { module_id } => {
2911                self.log_supervisor_request_received(
2912                    ctx,
2913                    frame.header.corr,
2914                    ops::SUPERVISOR_RELOAD,
2915                    Some(&module_id),
2916                    None,
2917                )?;
2918                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2919                self.handle_supervisor_reload(frame, module_id, requester)
2920                    .await
2921            }
2922            ClientControlRequest::SupervisorRescan { preview } => {
2923                if !preview {
2924                    self.log_supervisor_request_received(
2925                        ctx,
2926                        frame.header.corr,
2927                        ops::SUPERVISOR_RESCAN,
2928                        None,
2929                        None,
2930                    )?;
2931                }
2932                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2933                self.handle_supervisor_rescan(
2934                    frame,
2935                    if preview {
2936                        RescanMode::Preview
2937                    } else {
2938                        RescanMode::Execute
2939                    },
2940                    requester,
2941                )
2942                .await
2943            }
2944            ClientControlRequest::SupervisorRescanChecked {
2945                preview,
2946                candidate,
2947                expect,
2948            } => {
2949                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2950                    .await
2951            }
2952            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2953                self.log_supervisor_request_received(
2954                    ctx,
2955                    frame.header.corr,
2956                    ops::SUPERVISOR_RELEASE_RESERVED,
2957                    Some(&module_id),
2958                    None,
2959                )?;
2960                self.handle_supervisor_release_reserved(frame, module_id)
2961                    .await
2962            }
2963            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2964                self.log_supervisor_request_received(
2965                    ctx,
2966                    frame.header.corr,
2967                    ops::SUPERVISOR_SET_ENABLED,
2968                    Some(&module_id),
2969                    Some(enabled),
2970                )?;
2971                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2972                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2973                    .await
2974            }
2975            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2976                self.handle_supervisor_health_probe(frame, module_id).await
2977            }
2978            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2979            ClientControlRequest::SupervisorRoutes { module_id } => {
2980                self.handle_supervisor_routes(frame, module_id)
2981            }
2982            ClientControlRequest::SupervisorProvenance { module_id } => {
2983                self.handle_supervisor_provenance(frame, module_id).await
2984            }
2985            ClientControlRequest::SupervisorStderrTail {
2986                module_id,
2987                max_lines,
2988                max_bytes,
2989            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2990            ClientControlRequest::SupervisorTerminals { module_id } => {
2991                self.handle_supervisor_terminals(frame, module_id).await
2992            }
2993        }
2994    }
2995
2996    fn log_supervisor_request_received(
2997        &self,
2998        ctx: &RouteCtx,
2999        corr: u64,
3000        op: &'static str,
3001        module_id: Option<&str>,
3002        enabled: Option<bool>,
3003    ) -> Result<(), RouterError> {
3004        let caller = self.supervisor_requester(ctx, corr)?;
3005        let caller = principal_label(&caller);
3006        // `Option` fields are recorded only when present, so a request with no
3007        // module id (rescan) or no enabled flag simply omits that field.
3008        info!(
3009            target: "control",
3010            op,
3011            module_id,
3012            enabled,
3013            connection_id = ctx.connection_id.get(),
3014            caller = %caller,
3015            "supervisor request received"
3016        );
3017        Ok(())
3018    }
3019
3020    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3021        Ok(self
3022            .registry
3023            .get_module_by_connection(ctx.connection_id)
3024            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3025            .map(|registration| Principal::Reserved {
3026                module_id: registration.manifest.module_id,
3027            })
3028            .unwrap_or(Principal::Direct))
3029    }
3030
3031    fn handle_module_control_request(
3032        &self,
3033        connection_id: ConnectionId,
3034        frame: Frame,
3035        request: ModuleControlRequestFromModule,
3036    ) -> Result<Vec<Frame>, RouterError> {
3037        match request {
3038            ModuleControlRequestFromModule::CatalogUpdate {
3039                provides,
3040                capabilities,
3041                ready,
3042            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3043            ModuleControlRequestFromModule::LiveRoots {} => {
3044                let registered = self
3045                    .registry
3046                    .get_module_by_connection(connection_id)
3047                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3048                let Some(registration) = registered else {
3049                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3050                };
3051                let response = self
3052                    .forwarding
3053                    .live_roots(&registration.manifest.module_id)
3054                    .map_err(RouterError::Forwarding)?;
3055                Ok(vec![control_response_body_frame(
3056                    &frame,
3057                    &response,
3058                    "ModuleControlResponseToModule::LiveRoots",
3059                )?])
3060            }
3061            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3062                self.handle_scope_sync(connection_id, frame, generation, scopes)
3063            }
3064            ModuleControlRequestFromModule::ScopeApply {
3065                generation,
3066                upsert,
3067                end,
3068            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3069            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3070                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3071            }
3072        }
3073    }
3074
3075    fn handle_scope_served_by(
3076        &self,
3077        connection_id: ConnectionId,
3078        frame: Frame,
3079    ) -> Result<Vec<Frame>, RouterError> {
3080        let registered = self
3081            .registry
3082            .get_module_by_connection(connection_id)
3083            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3084        let Some(registered) = registered else {
3085            return Ok(vec![control_error_frame(
3086                &frame,
3087                "not_registered",
3088                "scope.served_by requires an active module registration owned by this connection",
3089            )?]);
3090        };
3091        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3092        {
3093            Ok(request) => request,
3094            Err(err) => {
3095                return Ok(vec![control_error_frame(
3096                    &frame,
3097                    "invalid_control_body",
3098                    format!("malformed scope.served_by body: {err}"),
3099                )?])
3100            }
3101        };
3102        let answer = self
3103            .scopes
3104            .read()
3105            .unwrap_or_else(|poisoned| poisoned.into_inner())
3106            .served_by(
3107                &registered.manifest.module_id,
3108                &request.scope_ref,
3109                request.scope_epoch,
3110                &request.module_id,
3111            );
3112        let reply = subc_control::ScopeServedByReply::new(
3113            answer,
3114            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3115        );
3116        Ok(vec![control_response_body_frame(
3117            &frame,
3118            &reply,
3119            "ScopeServedByReply",
3120        )?])
3121    }
3122
3123    fn handle_operator_confirm(
3124        &self,
3125        ctx: &RouteCtx,
3126        frame: Frame,
3127    ) -> Result<Vec<Frame>, RouterError> {
3128        use crate::operator_confirm::{audit, Outcome};
3129        let registration = self
3130            .registry
3131            .get_module_by_connection(ctx.connection_id)
3132            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3133        let Some(registration) = registration else {
3134            let outcome = Outcome::refusal("not_registered");
3135            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3136            return Ok(vec![outcome.frame(&frame)]);
3137        };
3138        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3139            Ok(request) => request,
3140            Err(_) => {
3141                let outcome = Outcome::refusal("invalid_control_body");
3142                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3143                return Ok(vec![outcome.frame(&frame)]);
3144            }
3145        };
3146        let module_id = registration.manifest.module_id;
3147        // Read the launch nonce (under its own lock) before taking the forwarding
3148        // table's lock below: holding forwarding while waiting on another daemon
3149        // lock risks a lock-order deadlock with paths that take them the other way.
3150        let nonce = self
3151            .hello_launch_nonces
3152            .lock()
3153            .unwrap_or_else(|p| p.into_inner())
3154            .nonce(ctx.connection_id)
3155            .map(str::to_owned);
3156        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3157            self.supervisor
3158                .spawned_consumer_authorized(&module_id, nonce)
3159        });
3160        let confirms = self.forwarding.operator_confirms();
3161        self.forwarding
3162            .with_operator_route(
3163                ctx.connection_id,
3164                request.route_channel,
3165                request.route_epoch,
3166                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3167            )
3168            .map_err(RouterError::Forwarding)
3169    }
3170
3171    /// Resolve folders before taking the scope-table lock: a slow filesystem
3172    /// must not hold up admissions for unrelated scoped routes.
3173    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3174        let mut scopes = ScopeRecords::from(scopes);
3175        for (index, record) in scopes.records.iter_mut().enumerate() {
3176            if let Some(refusal) = Self::canonicalize_workdir(record) {
3177                scopes.refusals.insert(index, refusal);
3178            }
3179        }
3180        scopes
3181    }
3182
3183    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3184        let workdir = record.workdir.as_mut()?;
3185        let path = std::path::Path::new(workdir);
3186        if !path.is_absolute() {
3187            return Some((
3188                error_codes::INVALID_WORKDIR,
3189                "workdir must be absolute".to_string(),
3190            ));
3191        }
3192        match ProjectRootId::from_path_allowing_missing(path) {
3193            Ok(root) => match root.as_path().to_str() {
3194                Some(canonical) => {
3195                    *workdir = canonical.to_string();
3196                    None
3197                }
3198                None => Some((
3199                    error_codes::INVALID_WORKDIR,
3200                    "canonical workdir is not UTF-8".to_string(),
3201                )),
3202            },
3203            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3204        }
3205    }
3206
3207    /// `scope.sync`: the owner is the module registered on this connection.
3208    /// A connection with no registration (every client connection, `direct`
3209    /// included) is refused `not_registered` before the table is consulted.
3210    fn handle_scope_sync(
3211        &self,
3212        connection_id: ConnectionId,
3213        frame: Frame,
3214        generation: u64,
3215        scopes: Vec<ScopeRecord>,
3216    ) -> Result<Vec<Frame>, RouterError> {
3217        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3218    }
3219
3220    fn handle_scope_change(
3221        &self,
3222        connection_id: ConnectionId,
3223        frame: Frame,
3224        generation: u64,
3225        scopes: Vec<ScopeRecord>,
3226        end: Option<Vec<ScopeEnd>>,
3227    ) -> Result<Vec<Frame>, RouterError> {
3228        let is_apply = end.is_some();
3229        let Some(registration) = self
3230            .registry
3231            .get_module_by_connection(connection_id)
3232            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3233        else {
3234            return Ok(vec![control_error_frame(
3235                &frame,
3236                "not_registered",
3237                "scope.sync and scope.apply require an active module registration owned by this connection",
3238            )?]);
3239        };
3240        let owner = registration.manifest.module_id;
3241        let scopes = Self::prepare_scope_records(scopes);
3242        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3243        let is_current_launch = |connection: ConnectionId| {
3244            self.hello_launch_nonces
3245                .lock()
3246                .unwrap_or_else(|poisoned| poisoned.into_inner())
3247                .presented(connection, current_nonce.as_deref())
3248        };
3249        // Lock order is the scope table, then the forwarding table: the new
3250        // tags are published, and the routes the change closes are selected,
3251        // while the scope table is still write-locked, so no admission can read
3252        // a record whose tag is not yet published.
3253        let mut table = self
3254            .scopes
3255            .write()
3256            .unwrap_or_else(|poisoned| poisoned.into_inner());
3257        let outcome = match end {
3258            Some(end) => table.apply(
3259                &owner,
3260                connection_id,
3261                is_current_launch,
3262                generation,
3263                scopes,
3264                end,
3265            ),
3266            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3267        };
3268        let drained = match &outcome {
3269            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3270            Err(_) => Vec::new(),
3271        };
3272        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3273        drop(table);
3274        if has_deadlines {
3275            self.scope_deadline_added.notify_one();
3276        }
3277        match outcome {
3278            Ok(applied) => {
3279                let counts = ScopeOutcomeCounts::of(&applied.results);
3280                info!(
3281                    owner = %owner,
3282                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3283                    generation,
3284                    records = applied.results.len(),
3285                    created = counts.created,
3286                    replaced = counts.replaced,
3287                    updated = counts.updated,
3288                    unchanged = counts.unchanged,
3289                    refused = counts.refused,
3290                    ended = applied.ended.len(),
3291                    tag_changes = applied.tag_changes.len(),
3292                    routes_closed = drained.len(),
3293                    "scope change accepted"
3294                );
3295                // An accepted scope change can still refuse individual records, and the
3296                // owner is the only party that sees the reply. Name them here so
3297                // an operator can tell a refused session from a missing one
3298                // without the owner's logs. Capped so a call that refuses
3299                // thousands cannot flood the log; the count above is complete.
3300                for refused in applied
3301                    .results
3302                    .iter()
3303                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3304                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3305                {
3306                    warn!(
3307                        owner = %owner,
3308                        generation,
3309                        scope_ref = %refused.scope_ref,
3310                        scope_epoch = refused.scope_epoch,
3311                        code = refused.code.as_deref().unwrap_or(""),
3312                        "scope record refused"
3313                    );
3314                }
3315                self.close_scope_drained_routes(drained);
3316                let response = if is_apply {
3317                    ModuleControlResponseToModule::ScopeApply {
3318                        generation,
3319                        results: applied.results,
3320                        end_results: applied.end_results,
3321                        ended: applied.ended,
3322                    }
3323                } else {
3324                    ModuleControlResponseToModule::ScopeSync {
3325                        generation,
3326                        results: applied.results,
3327                        ended: applied.ended,
3328                    }
3329                };
3330                Ok(vec![control_response_body_frame(
3331                    &frame,
3332                    &response,
3333                    "scope change response",
3334                )?])
3335            }
3336            Err(refusal) => {
3337                info!(
3338                    owner = %owner,
3339                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3340                    generation,
3341                    code = refusal.code,
3342                    "scope change refused"
3343                );
3344                Ok(vec![control_error_frame(
3345                    &frame,
3346                    refusal.code,
3347                    refusal.message,
3348                )?])
3349            }
3350        }
3351    }
3352
3353    /// Tell both ends of each route a scope change closed. The module gets a
3354    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3355    /// the client's route handle. The client also gets `route.closed` with the
3356    /// scope reason, one push per module and reason, so it can tell a revoked
3357    /// route from an ordinary close and not reopen it.
3358    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3359        if drained.is_empty() {
3360            return;
3361        }
3362        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3363            BTreeMap::new();
3364        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3365        for route in drained {
3366            warn!(
3367                module_id = %route.module_id,
3368                reason = ?route.reason,
3369                client_connection_id = route.client.connection_id.get(),
3370                route_channel = route.client.channel,
3371                "closing route because its scope changed"
3372            );
3373            pushes
3374                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3375                .or_insert_with(|| (route.reason, Vec::new()))
3376                .1
3377                .push(EndpointRoute {
3378                    goodbye_target: route.client.clone(),
3379                    principal: Principal::Unverified,
3380                    bound_at: Instant::now(),
3381                    draining: false,
3382                    drain_reason: None,
3383                });
3384            goodbyes.push(route.module);
3385            goodbyes.push(route.client);
3386        }
3387        for ((module_id, _), (reason, routes)) in pushes {
3388            send_route_control_pushes(
3389                &self.forwarding,
3390                routes,
3391                ClientControlPush::RouteClosed {
3392                    module_id,
3393                    channels: Vec::new(),
3394                    reason,
3395                    drained: false,
3396                    abandoned: 0,
3397                    excluded_subscriptions: 0,
3398                    terminal: Some(false),
3399                },
3400            );
3401        }
3402        self.emit_route_goodbyes(goodbyes);
3403    }
3404
3405    /// `scope.describe`: any registered module may read any scope, because a
3406    /// provider must read the scope a route it serves is stamped with.
3407    fn handle_scope_describe(
3408        &self,
3409        connection_id: ConnectionId,
3410        frame: Frame,
3411        owner: Principal,
3412        scope_ref: String,
3413    ) -> Result<Vec<Frame>, RouterError> {
3414        let registered = self
3415            .registry
3416            .get_module_by_connection(connection_id)
3417            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3418        let Some(registered) = registered else {
3419            return Ok(vec![control_error_frame(
3420                &frame,
3421                "not_registered",
3422                "scope.describe requires an active module registration owned by this connection",
3423            )?]);
3424        };
3425        let mut description = self
3426            .scopes
3427            .read()
3428            .unwrap_or_else(|poisoned| poisoned.into_inner())
3429            .describe(&owner, &scope_ref);
3430        // Describe readers also decode strict ScopeAttributes. Attribution is
3431        // safe to omit unless the requesting module declares it understands it.
3432        if !registered
3433            .manifest
3434            .capabilities
3435            .as_ref()
3436            .is_some_and(|caps| {
3437                caps.provides
3438                    .iter()
3439                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3440            })
3441        {
3442            if let Some(stamp) = description.stamp.as_mut() {
3443                stamp.attributes.on_behalf_of_agent = None;
3444            }
3445        }
3446        let owner_configured = match &owner {
3447            // Ask whether the owner is configured (`is_configured`), not
3448            // whether it is on the roster (`get(..).is_some()`): a supervised
3449            // module's process can register and describe a scope before the
3450            // supervisor has put it on the roster.
3451            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3452            _ => false,
3453        };
3454        let response = ModuleControlResponseToModule::ScopeDescribe {
3455            status: description.status,
3456            scope_epoch: description.scope_epoch,
3457            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3458            owner_synced: description.owner_synced,
3459            owner_configured,
3460            scope: description.stamp,
3461        };
3462        Ok(vec![control_response_body_frame(
3463            &frame,
3464            &response,
3465            "ModuleControlResponseToModule::ScopeDescribe",
3466        )?])
3467    }
3468
3469    fn handle_catalog_update(
3470        &self,
3471        connection_id: ConnectionId,
3472        frame: Frame,
3473        provides: Vec<ProviderRole>,
3474        capabilities: Option<CapabilityDeclarations>,
3475        ready: Option<bool>,
3476    ) -> Result<Vec<Frame>, RouterError> {
3477        self.refresh_capability_requirements();
3478        let Some(registration) = self
3479            .registry
3480            .get_module_by_connection(connection_id)
3481            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3482        else {
3483            return Ok(vec![control_error_frame(
3484                &frame,
3485                "not_registered",
3486                "catalog.update requires an active module registration owned by this connection",
3487            )?]);
3488        };
3489
3490        if let Some(message) =
3491            catalog_update_frozen_field_message(&registration.manifest, &provides)
3492        {
3493            return Ok(vec![control_error_frame(
3494                &frame,
3495                "catalog_update_frozen_field",
3496                message,
3497            )?]);
3498        }
3499
3500        let mut candidate = registration.manifest.clone();
3501        candidate.provides = provides.clone();
3502        candidate.capabilities = capabilities
3503            .clone()
3504            .or_else(|| registration.manifest.capabilities.clone());
3505        if let Some(rich_ui) = &candidate.rich_ui {
3506            if let Err(err) = rich_ui.validate() {
3507                return Ok(vec![control_error_frame(
3508                    &frame,
3509                    "invalid_manifest",
3510                    err.to_string(),
3511                )?]);
3512            }
3513        }
3514        if let Err(err) = candidate.validate_capability_grammar() {
3515            return Ok(vec![control_error_frame(
3516                &frame,
3517                "invalid_capability_grammar",
3518                err.to_string(),
3519            )?]);
3520        }
3521
3522        // Updates must honor the same reserved owner as initial registration;
3523        // otherwise an empty HELLO could acquire the claim after admission.
3524        let mut conflicts = self
3525            .capability_evaluator
3526            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3527        if let Some(conflict) = conflicts.first() {
3528            let message = format!(
3529                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3530                conflict.capability, conflict.claimants[0], candidate.module_id
3531            );
3532            for conflict in &mut conflicts {
3533                conflict.source = DuplicateClaimSource::CatalogUpdate;
3534            }
3535            log_duplicate_claim_events(conflicts);
3536            return Ok(vec![control_error_frame(
3537                &frame,
3538                "reserved_capability",
3539                message,
3540            )?]);
3541        }
3542
3543        let updated = self
3544            .registry
3545            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3546            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3547        if updated.is_none() {
3548            return Ok(vec![control_error_frame(
3549                &frame,
3550                "not_registered",
3551                "catalog.update requires an active module registration owned by this connection",
3552            )?]);
3553        }
3554        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3555            log_duplicate_claim_events(
3556                self.capability_evaluator
3557                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3558            );
3559        }
3560        if capability_census_trigger(
3561            registration.manifest.capabilities.as_ref(),
3562            updated
3563                .as_ref()
3564                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3565        ) {
3566            self.enforce_capability_denies();
3567        }
3568        self.refresh_capability_requirements();
3569
3570        let response = ModuleControlResponseToModule::CatalogUpdate {};
3571        control_response_body_frame(
3572            &frame,
3573            &response,
3574            "ModuleControlResponseToModule::CatalogUpdate",
3575        )
3576        .map(|frame| vec![frame])
3577    }
3578
3579    pub(crate) fn with_activity_ledger_config(
3580        mut self,
3581        run_dir: &Path,
3582        incarnation: String,
3583        config: crate::activity::ActivityConfig,
3584    ) -> Self {
3585        self.activity = Some(crate::activity::ActivityLedger::open_config(
3586            run_dir.join("activity.jsonl"),
3587            incarnation,
3588            config,
3589        ));
3590        self.supervisor.set_activity(self.activity.clone());
3591        self
3592    }
3593
3594    pub(crate) async fn configure_activity(
3595        &self,
3596        config: crate::activity::ActivityConfig,
3597    ) -> Result<(), &'static str> {
3598        match &self.activity {
3599            Some(ledger) => ledger.configure(config).await,
3600            None => Ok(()),
3601        }
3602    }
3603
3604    fn cancel_activity_subscription(
3605        &self,
3606        ctx: &RouteCtx,
3607        frame: &Frame,
3608    ) -> Result<bool, RouterError> {
3609        let task = self
3610            .activity_streams
3611            .lock()
3612            .unwrap_or_else(|p| p.into_inner())
3613            .remove(&(ctx.connection_id, frame.header.corr));
3614        let Some(task) = task else {
3615            return Ok(false);
3616        };
3617        task.abort();
3618        let end = Frame::build_with_version(
3619            response_version(frame),
3620            FrameType::StreamEnd,
3621            control_flags(),
3622            0,
3623            0,
3624            frame.header.corr,
3625            Vec::new(),
3626        )
3627        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3628        let sink = ctx.egress.clone();
3629        tokio::spawn(async move {
3630            let _ = sink.send(end).await;
3631        });
3632        Ok(true)
3633    }
3634
3635    async fn handle_activity_read(
3636        &self,
3637        ctx: &RouteCtx,
3638        frame: Frame,
3639    ) -> Result<Vec<Frame>, RouterError> {
3640        use subc_control::{
3641            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3642            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3643        };
3644        #[derive(Deserialize)]
3645        struct Identity {
3646            #[serde(default)]
3647            consumer_identity: Option<ConsumerIdentity>,
3648        }
3649        // A registered module connection already identifies its caller. Ignore
3650        // consumer_identity there so a module cannot borrow a mirror's identity.
3651        // Only separate consumer connections use the Direct/nonce trust boundary.
3652        let principal = match self
3653            .forwarding
3654            .module_id_for_connection(ctx.connection_id)
3655            .map_err(RouterError::Forwarding)?
3656        {
3657            Some(module_id) => Ok(Principal::Reserved { module_id }),
3658            None => {
3659                let identity: Identity = match serde_json::from_slice(&frame.body) {
3660                    Ok(identity) => identity,
3661                    Err(_) => {
3662                        return Ok(vec![control_error_frame(
3663                            &frame,
3664                            ACTIVITY_MALFORMED,
3665                            "malformed activity read",
3666                        )?])
3667                    }
3668                };
3669                self.route_open_principal(&frame, identity.consumer_identity)?
3670            }
3671        };
3672        let admitted = matches!(principal, Ok(Principal::Direct))
3673            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3674        if !admitted {
3675            return Ok(vec![control_error_frame(
3676                &frame,
3677                ACTIVITY_READ_FORBIDDEN,
3678                "activity reads require an operator or trusted mirror principal",
3679            )?]);
3680        }
3681        let Some(ledger) = &self.activity else {
3682            return Ok(vec![control_error_frame(
3683                &frame,
3684                ACTIVITY_UNAVAILABLE,
3685                "activity ledger unavailable",
3686            )?]);
3687        };
3688        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3689            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3690        {
3691            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3692                Ok(request) => ledger.query(request).await,
3693                Err(_) => Err(ACTIVITY_MALFORMED),
3694            };
3695            return match result {
3696                Ok(reply) => Ok(vec![control_response_body_frame(
3697                    &frame,
3698                    &reply,
3699                    ops::ACTIVITY_QUERY,
3700                )?]),
3701                Err(code) => Ok(vec![control_error_frame(
3702                    &frame,
3703                    code,
3704                    "activity query refused",
3705                )?]),
3706            };
3707        }
3708        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3709            Ok(request) => request,
3710            Err(_) => {
3711                return Ok(vec![control_error_frame(
3712                    &frame,
3713                    ACTIVITY_MALFORMED,
3714                    "malformed activity subscription",
3715                )?])
3716            }
3717        };
3718        let mut subscription = match ledger.subscribe(request.since).await {
3719            Ok(subscription) => subscription,
3720            Err(code) => {
3721                return Ok(vec![control_error_frame(
3722                    &frame,
3723                    code,
3724                    "activity subscription refused",
3725                )?])
3726            }
3727        };
3728        let key = (ctx.connection_id, frame.header.corr);
3729        let sink = ctx.egress.clone();
3730        let mut streams = self
3731            .activity_streams
3732            .lock()
3733            .unwrap_or_else(|p| p.into_inner());
3734        streams.retain(|_, task| !task.is_finished());
3735        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3736            return Ok(vec![control_error_frame(
3737                &frame,
3738                "duplicate_subscription",
3739                "activity subscription correlation id already in use",
3740            )?]);
3741        }
3742        let task = tokio::spawn(async move {
3743            loop {
3744                let entry = if let Some(entry) = subscription.replay.pop_front() {
3745                    Some(entry)
3746                } else {
3747                    subscription.receiver.recv().await
3748                };
3749                let Some(entry) = entry else {
3750                    break;
3751                };
3752                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3753                    return;
3754                };
3755                let Ok(next) = Frame::build_with_version(
3756                    response_version(&frame),
3757                    FrameType::StreamData,
3758                    control_flags(),
3759                    0,
3760                    0,
3761                    frame.header.corr,
3762                    body,
3763                ) else {
3764                    return;
3765                };
3766                if sink.send(next).await.is_err() {
3767                    return;
3768                }
3769            }
3770            // Dropping a full subscriber's sender drains its accepted entries
3771            // before the terminal error, preserving their order on the wire.
3772            if let Some(code) = subscription.terminal_code() {
3773                if let Ok(end) =
3774                    control_error_frame(&frame, code, "activity subscriber fell behind")
3775                {
3776                    let _ = sink.send(end).await;
3777                }
3778            }
3779        });
3780        streams.insert(key, task.abort_handle());
3781        Ok(Vec::new())
3782    }
3783
3784    async fn handle_activity_record(
3785        &self,
3786        ctx: &RouteCtx,
3787        frame: Frame,
3788    ) -> Result<Vec<Frame>, RouterError> {
3789        use subc_control::{
3790            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3791            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3792        };
3793        let result = async {
3794            let module = self
3795                .forwarding
3796                .module_id_for_connection(ctx.connection_id)
3797                .ok()
3798                .flatten()
3799                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3800            let request: ActivityRecordRequest =
3801                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3802            crate::activity::validate(&module, &request)?;
3803            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3804            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3805            let at_ms = std::time::SystemTime::now()
3806                .duration_since(std::time::UNIX_EPOCH)
3807                .unwrap_or_default()
3808                .as_millis() as u64;
3809            ledger.record(module, request, actor, at_ms).await
3810        }
3811        .await;
3812        match result {
3813            Ok(entry) => Ok(vec![control_response_body_frame(
3814                &frame,
3815                &ActivityRecordResponse { seq: entry.seq },
3816                "activity.record",
3817            )?]),
3818            Err(code) => Ok(vec![control_error_frame(
3819                &frame,
3820                code,
3821                "activity recording refused",
3822            )?]),
3823        }
3824    }
3825
3826    fn activity_actor(
3827        &self,
3828        connection: ConnectionId,
3829        module: &str,
3830        on: Option<&subc_control::ActivityOn>,
3831    ) -> Result<subc_control::ActivityActor, &'static str> {
3832        use crate::forwarding::{DataRoute, DataRouteState};
3833        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3834        let principal = |id: &str| Principal::Reserved {
3835            module_id: id.to_owned(),
3836        };
3837        let mut actor = ActivityActor {
3838            principal: principal(module),
3839            scope_owner: None,
3840            agent_id: None,
3841            run_id: None,
3842            flow_id: None,
3843            project_root: None,
3844            workdir: None,
3845            project_id: None,
3846        };
3847        let scope = match on {
3848            None => return Ok(actor),
3849            Some(ActivityOn::Scope(scope)) => {
3850                let selector = subc_protocol::scope::ScopeSelector::new(
3851                    principal(module),
3852                    scope.scope_ref.clone(),
3853                    Some(scope.epoch),
3854                );
3855                self.scopes
3856                    .read()
3857                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3858                    .attribute(&principal(module), module, &selector, true)
3859                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3860            }
3861            Some(ActivityOn::Route(route)) => {
3862                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3863                    .forwarding
3864                    .lookup_data_route(connection, route.channel, route.epoch)
3865                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3866                else {
3867                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3868                };
3869                if binding.module_id != module {
3870                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3871                }
3872                actor.principal = binding.principal.clone();
3873                actor.project_root = binding
3874                    .project_root
3875                    .as_ref()
3876                    .map(|root| root.as_path().to_path_buf());
3877                let Some(bound) = &binding.scope else {
3878                    return Ok(actor);
3879                };
3880                let owner = principal(&bound.owner);
3881                let selector = subc_protocol::scope::ScopeSelector::new(
3882                    owner.clone(),
3883                    bound.scope_ref.clone(),
3884                    Some(bound.tag.scope_epoch),
3885                );
3886                let admitted = self
3887                    .scopes
3888                    .read()
3889                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3890                    .attribute(&owner, module, &selector, true)
3891                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3892                if admitted.tag != bound.tag {
3893                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3894                }
3895                admitted
3896            }
3897        };
3898        actor.scope_owner = Some(scope.stamp.owner);
3899        actor.agent_id = scope.stamp.attributes.agent_id;
3900        actor.run_id = scope.stamp.attributes.run_id;
3901        actor.flow_id = scope.stamp.attributes.flow_id;
3902        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3903        actor.project_id = scope.stamp.project_id;
3904        Ok(actor)
3905    }
3906
3907    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3908        self.refresh_capability_requirements();
3909        // A bare connection count is ambiguous between many clients holding a
3910        // route each and one client accumulating hundreds, so publish the
3911        // concentration alongside it. Route state is best-effort here: a
3912        // diagnostic endpoint must still answer if the forwarding lock is
3913        // contended.
3914        let mut counters = self.counters.snapshot();
3915        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3916            obj.insert(
3917                "activity_unparseable_lines".into(),
3918                ledger.read_errors().into(),
3919            );
3920            obj.insert(
3921                "activity_lifecycle_append_failures".into(),
3922                ledger.lifecycle_failures().into(),
3923            );
3924            obj.insert(
3925                "activity_started_digest_skips".into(),
3926                ledger.lifecycle_digest_skips().into(),
3927            );
3928        }
3929        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3930            self.forwarding.client_route_concentration(),
3931            counters.as_object_mut(),
3932        ) {
3933            obj.insert(
3934                "client_connections_with_routes".into(),
3935                connections_with_routes.into(),
3936            );
3937            obj.insert("max_routes_on_one_connection".into(), max.into());
3938        }
3939        // A module that is being fast-refused and a module that is fine look
3940        // identical from a client that retries and succeeds, so name the open
3941        // breakers here. This rides the existing free-form counters object
3942        // rather than a new wire field, so no sibling that deserializes
3943        // `ServerDescribe` has to be rebuilt to keep reading it.
3944        if let (Some(open_breakers), Some(obj)) = (
3945            self.route_bind_breakers.open_snapshot(),
3946            counters.as_object_mut(),
3947        ) {
3948            obj.insert("route_bind_breakers_open".into(), open_breakers);
3949        }
3950        let response = ClientControlResponse::ServerDescribe {
3951            protocol_ver: PROTOCOL_VERSION,
3952            subc_ops: subc_ops(),
3953            capabilities: self.describe_capabilities(),
3954            connected_clients: self.connected_clients.count(),
3955            counters: Some(counters),
3956            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3957            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3958            capability_requirements: self.capability_requirement_statuses(),
3959            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3960            config_path: self
3961                .rescan
3962                .as_ref()
3963                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3964        };
3965        Ok(vec![control_response_body_frame(
3966            &frame,
3967            &response,
3968            "ClientControlResponse::ServerDescribe",
3969        )?])
3970    }
3971
3972    fn handle_catalog_list(
3973        &self,
3974        frame: Frame,
3975        module_id: Option<String>,
3976    ) -> Result<Vec<Frame>, RouterError> {
3977        let (generation, modules) = self
3978            .registry
3979            .list_modules_with_nonce_verification()
3980            .map_err(|err| {
3981                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3982            })?;
3983        let entries = modules
3984            .into_iter()
3985            .filter(|(registration, _)| {
3986                module_id
3987                    .as_deref()
3988                    .map(|wanted| registration.manifest.module_id == wanted)
3989                    .unwrap_or(true)
3990            })
3991            .map(|(registration, nonce_verified)| {
3992                let not_ready = self.not_ready_reason(&registration);
3993                let roles = registration.manifest.provides;
3994                CatalogEntry::new(
3995                    registration.manifest.module_id,
3996                    roles,
3997                    registration.control_ops,
3998                )
3999                .with_ready(not_ready.is_none())
4000                .with_not_ready(not_ready)
4001                .with_module_version(Some(registration.manifest.module_version))
4002                .with_capabilities(registration.manifest.capabilities)
4003                .with_self_signals(registration.manifest.self_signals)
4004                .with_events(registration.manifest.events)
4005                .with_operation_delivery(registration.manifest.operation_delivery)
4006                .with_operation_payload(registration.manifest.operation_payload)
4007                .with_rich_ui(registration.manifest.rich_ui)
4008                .with_role_declarations(registration.manifest.role_declarations)
4009                .with_nonce_verified(nonce_verified)
4010            })
4011            .collect();
4012        let response = ClientControlResponse::CatalogList {
4013            generation,
4014            modules: entries,
4015            subc_ops: subc_ops(),
4016        };
4017        Ok(vec![control_response_body_frame(
4018            &frame,
4019            &response,
4020            "ClientControlResponse::CatalogList",
4021        )?])
4022    }
4023
4024    /// The principal a route.open's `consumer_identity` attests: `Direct`
4025    /// when it presents none, `Reserved` when its launch nonce verifies, and
4026    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4027    /// can run before any refusal check without changing which check refuses.
4028    fn attest_route_open_consumer(
4029        &self,
4030        consumer_identity: Option<&ConsumerIdentity>,
4031    ) -> Option<Principal> {
4032        let Some(consumer_identity) = consumer_identity else {
4033            return Some(Principal::Direct);
4034        };
4035        self.supervisor
4036            .spawned_consumer_authorized(
4037                &consumer_identity.module_id,
4038                &consumer_identity.launch_nonce,
4039            )
4040            .then(|| Principal::Reserved {
4041                module_id: consumer_identity.module_id.clone(),
4042            })
4043    }
4044
4045    /// The caller fields for a refusal that only has the raw request frame,
4046    /// such as the connection-level admission refusal sent before the request
4047    /// reaches [`Self::handle_route_open`].
4048    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4049        let Ok(ClientControlRequest::RouteOpen {
4050            identity,
4051            consumer_identity,
4052            scope,
4053            ..
4054        }) = parse_client_control_request(&frame.body)
4055        else {
4056            return RouteOpenCaller::unknown();
4057        };
4058        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4059        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4060    }
4061
4062    fn route_open_principal(
4063        &self,
4064        frame: &Frame,
4065        consumer_identity: Option<ConsumerIdentity>,
4066    ) -> Result<Result<Principal, Frame>, RouterError> {
4067        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4068        self.route_open_principal_attested(frame, consumer_identity, attested)
4069    }
4070
4071    /// [`Self::route_open_principal`] with `attested` already taken by
4072    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4073    /// open checks the launch nonce once.
4074    fn route_open_principal_attested(
4075        &self,
4076        frame: &Frame,
4077        consumer_identity: Option<ConsumerIdentity>,
4078        attested: Option<Principal>,
4079    ) -> Result<Result<Principal, Frame>, RouterError> {
4080        if let Some(principal) = attested {
4081            return Ok(Ok(principal));
4082        }
4083        let Some(consumer_identity) = consumer_identity else {
4084            return Ok(Ok(Principal::Direct));
4085        };
4086
4087        Ok(Err(control_error_frame(
4088            frame,
4089            "bad_consumer_identity",
4090            format!(
4091                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4092                consumer_identity.module_id
4093            ),
4094        )?))
4095    }
4096
4097    /// Ordinary `route.open` refusals go through here; admission and breaker
4098    /// refusals log separately with their capacity or breaker state. The daemon can
4099    /// attest which code it sent: without the event, a client's "the daemon
4100    /// refused me" and the daemon's own view could only be reconciled by
4101    /// argument. Malformed input (`invalid_project_root`) does not come here;
4102    /// rejecting a request that was never a valid open is not a refusal of one.
4103    #[allow(clippy::too_many_arguments)]
4104    fn route_open_refusal_frame(
4105        &self,
4106        ctx: &RouteCtx,
4107        caller: &RouteOpenCaller,
4108        frame: &Frame,
4109        module_id: &str,
4110        reason: &'static str,
4111        code: &'static str,
4112        message: impl Into<String>,
4113    ) -> Result<Frame, RouterError> {
4114        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4115        control_error_frame(frame, code, message.into())
4116    }
4117
4118    /// Refuse a `route.open` because the target module's bind-relay breaker is
4119    /// open, without attempting the relay.
4120    ///
4121    /// The wire code is `module_timeout`, which is the truth (the module has
4122    /// not been answering binds) and which both SDKs already classify as
4123    /// retryable with capped backoff. Reusing it is what keeps this change out
4124    /// of both SDKs; the daemon-side distinction lives in the counter key
4125    /// instead.
4126    ///
4127    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4128    /// While a breaker is open this fires on every open to that module, and the
4129    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4130    /// produced 261 lines about a single module inside 3000 lines of daemon
4131    /// log. The rare transitions are logged at warn/info instead and the volume
4132    /// is carried by the counter, so the evidence survives without the flood.
4133    /// The debug line keeps a per-refusal record reachable for whoever turns
4134    /// the level up.
4135    #[allow(clippy::too_many_arguments)]
4136    fn route_open_breaker_refusal_frame(
4137        &self,
4138        ctx: &RouteCtx,
4139        caller: &RouteOpenCaller,
4140        frame: &Frame,
4141        module_id: &str,
4142        consecutive_timeouts: u32,
4143        retry_in: Duration,
4144        probe_in_flight: bool,
4145    ) -> Result<Frame, RouterError> {
4146        self.counters
4147            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4148        debug!(
4149            target: "control",
4150            code = "module_timeout",
4151            module_id = ?module_id,
4152            connection_id = ctx.connection_id.get(),
4153            principal = %caller.principal,
4154            scope = %caller.scope,
4155            harness = %caller.harness,
4156            consecutive_timeouts,
4157            retry_in_ms = retry_in.as_millis() as u64,
4158            probe_in_flight,
4159            "route.open refused by open bind-relay breaker"
4160        );
4161        // Say what a caller can act on. An open bind-relay breaker means the
4162        // module timed out accepting several new routes in a row. The module
4163        // is still running and its established routes keep working; only new
4164        // route.open requests are refused until the cooldown ends and one
4165        // test route (the probe) gets through. A message that only counts
4166        // failed relays reads as "the module is down" to a worker that sees it.
4167        let detail = if probe_in_flight {
4168            "one test route is already being tried; retry once it settles".to_string()
4169        } else {
4170            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4171        };
4172        control_error_frame(
4173            frame,
4174            "module_timeout",
4175            format!(
4176                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4177                 timed out in a row); {detail}; its established routes are unaffected"
4178            ),
4179        )
4180    }
4181
4182    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4183    /// requester's bytes (an unknown target is whatever the client sent) and
4184    /// is Debug-formatted so control characters land in the log escaped
4185    /// rather than as terminal sequences for whoever tails it.
4186    ///
4187    /// `reason` names the check that refused, because one wire code has
4188    /// several senders: after a module registers, `target_unavailable` can
4189    /// come from a missing role, an inactive registration, a supervisor that
4190    /// has not marked the process live, a missing forwarding connection, or a
4191    /// failed relay, and a log that records only the code cannot say which of
4192    /// them fired. It is a static, daemon-chosen label per branch, so it is
4193    /// safe to print plainly and stays a closed set.
4194    fn observe_route_open_refusal(
4195        &self,
4196        ctx: &RouteCtx,
4197        caller: &RouteOpenCaller,
4198        module_id: &str,
4199        reason: &'static str,
4200        code: &'static str,
4201    ) {
4202        self.counters.increment_route_open_refused(code);
4203        info!(
4204            target: "control",
4205            code,
4206            reason,
4207            module_id = ?module_id,
4208            connection_id = ctx.connection_id.get(),
4209            principal = %caller.principal,
4210            scope = %caller.scope,
4211            harness = %caller.harness,
4212            "route.open refused"
4213        );
4214        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4215            self.route_outages.record_not_serving(module_id, reason);
4216        }
4217    }
4218
4219    /// Record an ACCEPTED route.open.
4220    ///
4221    /// Refusals have been logged and counted since the attestation work; accepts
4222    /// were invisible, so the daemon knew every principal it stamped and wrote
4223    /// none of them down. The party that attests the identity was the only party
4224    /// not recording it, which left a credential vault unable to name the sender
4225    /// of a call that reached it (claustrum #43) and left the launch-nonce
4226    /// concurrency question unanswerable from the outside.
4227    ///
4228    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4229    /// `code`/`module_id`/`connection_id` returns both directions of the same
4230    /// decision rather than two shapes a reader has to join by hand.
4231    ///
4232    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4233    /// and the difference carries information rather than being an
4234    /// inconsistency. This line is only reachable after a successful bind to a
4235    /// REGISTERED module, so the value has already passed HELLO validation
4236    /// including the path-hazard refusal and cannot contain control bytes. A
4237    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4238    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4239    ///
4240    /// Bare is also what every other daemon line already emits (`module
4241    /// registered`, `configured module supervised`). Shipping `?module_id` here
4242    /// made this instrument the only one in the file whose ids did not answer
4243    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4244    /// line whose whole purpose is being grepped beside its sibling.
4245    ///
4246    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4247    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4248    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4249    /// one are recorded identically. A test written against that harness passes
4250    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4251    /// green assertion that cannot fail. The same limit applies to the escaping
4252    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4253    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4254    /// Fencing either needs the real formatter, not the capture layer.
4255    ///
4256    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4257    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4258    /// to record. The ephemeral port would decay within minutes and answer only a
4259    /// live question. The identity question is instead answered by counting
4260    /// distinct live connections presenting one module's `consumer_identity` --
4261    /// "is anyone else holding this secret" rather than "is this the right
4262    /// process".
4263    fn observe_route_open_accept(
4264        &self,
4265        ctx: &RouteCtx,
4266        module_id: &str,
4267        principal: &str,
4268        scope: &str,
4269    ) {
4270        self.route_outages.record_accepted(module_id);
4271        self.counters.increment_route_open_accepted(principal);
4272        info!(
4273            target: "control",
4274            principal,
4275            module_id,
4276            connection_id = ctx.connection_id.get(),
4277            scope = %scope,
4278            "route.open accepted"
4279        );
4280    }
4281
4282    fn supervised_absent_route_open_refusal_frame(
4283        &self,
4284        ctx: &RouteCtx,
4285        caller: &RouteOpenCaller,
4286        frame: &Frame,
4287        module_id: &str,
4288        code: &'static str,
4289        status: &crate::supervise::ModuleStatus,
4290    ) -> Result<Frame, RouterError> {
4291        self.counters.increment_route_open_refused(code);
4292        info!(
4293            target: "control",
4294            code,
4295            reason = "supervised_not_registered",
4296            module_id = ?module_id,
4297            connection_id = ctx.connection_id.get(),
4298            principal = %caller.principal,
4299            scope = %caller.scope,
4300            harness = %caller.harness,
4301            state = %status.state,
4302            enabled = status.enabled,
4303            live = status.live,
4304            "route.open refused"
4305        );
4306        // A supervised module whose process has not registered is not
4307        // serving, whatever the reason; the supervisor knows this id, so it is
4308        // safe to track.
4309        self.route_outages
4310            .record_not_serving(module_id, "supervised_not_registered");
4311        let message = if let Some(pid) = status.waiting_for_previous_pid {
4312            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4313                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4314            })
4315        } else {
4316            format!(
4317                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4318                status.state, status.enabled, status.live
4319            )
4320        };
4321        control_error_frame(frame, code, message)
4322    }
4323
4324    async fn handle_route_open(
4325        &self,
4326        ctx: &RouteCtx,
4327        frame: Frame,
4328        request: RouteOpenRequest,
4329        timings: &mut RouteOpenTimings,
4330    ) -> Result<Vec<Frame>, RouterError> {
4331        let RouteOpenRequest {
4332            target,
4333            mut identity,
4334            consumer_identity,
4335            consumer_capabilities,
4336            role_versions,
4337            admission_facts,
4338            scope,
4339        } = request;
4340        let target_module_id = target_module_id(&target).to_string();
4341        timings.module_id.clone_from(&target_module_id);
4342        if self
4343            .registry
4344            .get_module_by_connection(ctx.connection_id)
4345            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4346            .is_some()
4347        {
4348            return Ok(vec![control_error_frame(
4349                &frame,
4350                "invalid_request",
4351                "module connections cannot open client routes",
4352            )?]);
4353        }
4354        debug!(
4355            connection_id = ctx.connection_id.get(),
4356            corr = frame.header.corr,
4357            module_id = %target_module_id,
4358            "handling route.open"
4359        );
4360
4361        // The caller's attested principal, requested scope and harness, for
4362        // every `route.open refused` line below. The launch nonce is checked
4363        // here, once, and the answer is reused where the principal is
4364        // enforced further down, so the order of refusals is unchanged.
4365        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4366        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4367
4368        // A malformed declaration is refused first, before anything about the
4369        // target is looked up: the same body would be refused against any
4370        // module, so the caller learns nothing by retrying or waiting. An empty
4371        // map declares nothing and travels as no field at all, so a provider
4372        // only ever sees a missing field or a non-empty one.
4373        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4374        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4375            self.observe_route_open_refusal(
4376                ctx,
4377                &caller,
4378                &target_module_id,
4379                "invalid_role_versions",
4380                error_codes::INVALID_REQUEST,
4381            );
4382            return Ok(vec![control_error_body_frame(
4383                &frame,
4384                ErrorBody {
4385                    code: error_codes::INVALID_REQUEST.to_string(),
4386                    message: error.to_string(),
4387                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4388                },
4389            )?]);
4390        }
4391
4392        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4393            if status.waiting_for_previous_pid.is_some() {
4394                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4395                    ctx,
4396                    &caller,
4397                    &frame,
4398                    &target_module_id,
4399                    "module_warming",
4400                    &status,
4401                )?]);
4402            }
4403        }
4404
4405        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4406        // opposite. Below, a caller learns whether a module is unregistered,
4407        // supervised-but-down (with state/enabled/live), or registered without the
4408        // requested role. Elsewhere that is an enumeration leak: a probe learning
4409        // the shape of a fleet it cannot otherwise see.
4410        //
4411        // It is not one here, and the reason is the ACCESS MODEL rather than
4412        // anything about these errors. Reaching route.open requires the
4413        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4414        // connection file, so any caller who completes it already runs as this
4415        // user -- and can read subc.jsonc for the module list and `ck module
4416        // status` for live state. The reply discloses nothing the caller cannot
4417        // read more easily from disk, while the precision is load-bearing:
4418        // `unknown_module` is retryable and a missing role is not.
4419        //
4420        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4421        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4422        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4423        let Some(registration) = self
4424            .registry
4425            .get_module(&target_module_id)
4426            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4427        else {
4428            if let Some((status, warming)) =
4429                self.supervisor_status(&target_module_id, frame.header.corr)?
4430            {
4431                // BEFORE the two availability codes below, because for a module
4432                // that speaks no subc wire both of them are false comfort: they
4433                // say "not right now" and are retried, and this module will
4434                // never register no matter how long the caller waits. The
4435                // absence here is the declaration being honoured, not a module
4436                // that is late.
4437                if status.protocol == ModuleProtocol::None {
4438                    return Ok(vec![self.route_open_refusal_frame(
4439                        ctx,
4440                        &caller,
4441                        &frame,
4442                        &target_module_id,
4443                        "protocol_none",
4444                        error_codes::MODULE_NO_PROTOCOL,
4445                        format!(
4446                            "module_id '{target_module_id}' is declared protocol: none; \
4447                             it speaks no subc wire and serves no routes"
4448                        ),
4449                    )?]);
4450                }
4451                let code = if warming {
4452                    "module_warming"
4453                } else {
4454                    "target_unavailable"
4455                };
4456                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4457                    ctx,
4458                    &caller,
4459                    &frame,
4460                    &target_module_id,
4461                    code,
4462                    &status,
4463                )?]);
4464            }
4465            if let Some(removed_ago_ms) =
4466                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4467            {
4468                return Ok(vec![self.route_open_refusal_frame(
4469                    ctx,
4470                    &caller,
4471                    &frame,
4472                    &target_module_id,
4473                    "removed",
4474                    error_codes::MODULE_REMOVED,
4475                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4476                )?]);
4477            }
4478            return Ok(vec![self.route_open_refusal_frame(
4479                ctx,
4480                &caller,
4481                &frame,
4482                &target_module_id,
4483                "not_registered",
4484                error_codes::UNKNOWN_MODULE,
4485                format!("module_id '{target_module_id}' is not registered"),
4486            )?]);
4487        };
4488
4489        // Best-effort only: registry readiness and forwarding reservation use
4490        // different locks, so a module can flip readiness between this read and
4491        // the relay. Modules must still tolerate an `on_bind` while not ready.
4492        if !registration.ready {
4493            self.counters
4494                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4495            info!(
4496                target: "control",
4497                code = error_codes::MODULE_WARMING,
4498                module_id = ?target_module_id,
4499                connection_id = ctx.connection_id.get(),
4500                principal = %caller.principal,
4501                scope = %caller.scope,
4502                harness = %caller.harness,
4503                reason = "declared_not_ready",
4504                "route.open refused"
4505            );
4506            // The module is registered but says it cannot take work, which is
4507            // an outage from the caller's side even though its process is up.
4508            self.route_outages
4509                .record_not_serving(&target_module_id, "declared_not_ready");
4510            return Ok(vec![control_error_body_frame(
4511                &frame,
4512                ErrorBody {
4513                    code: error_codes::MODULE_WARMING.to_string(),
4514                    message: format!(
4515                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4516                    ),
4517                    detail: Some(serde_json::json!({
4518                        "reason": "declared_not_ready"
4519                    })),
4520                },
4521            )?]);
4522        }
4523
4524        // Effective readiness, second half: a module that declares a capability
4525        // `need: required` is not routable while that capability has no
4526        // registered provider. It is enforced HERE, as a retryable routing
4527        // refusal, and deliberately not as spawn ordering or a boot block. The
4528        // module is still started and registered and can make its own calls;
4529        // spawn ordering is a promise that cannot be kept once a provider
4530        // crashes at runtime, and refusing to boot would stop the whole
4531        // machine, including the tools needed to fix its configuration.
4532        //
4533        // "Provided" is the evaluator's verdict, which counts a provider as
4534        // soon as it has REGISTERED, not once it is ready. Two modules that
4535        // require each other's capabilities are therefore both routable once
4536        // both register; counting readiness instead would deadlock them.
4537        //
4538        // Only new opens are refused. Routes already bound when a provider
4539        // goes away stay bound: nothing here tears them down, and the module
4540        // answers them as it can. Like the readiness read above this is
4541        // best-effort against a provider registering or leaving concurrently.
4542        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4543            self.counters
4544                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4545            info!(
4546                target: "control",
4547                code = error_codes::MODULE_WARMING,
4548                module_id = ?target_module_id,
4549                connection_id = ctx.connection_id.get(),
4550                principal = %caller.principal,
4551                scope = %caller.scope,
4552                harness = %caller.harness,
4553                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4554                capability = %capability,
4555                "route.open refused"
4556            );
4557            return Ok(vec![control_error_body_frame(
4558                &frame,
4559                ErrorBody {
4560                    code: error_codes::MODULE_WARMING.to_string(),
4561                    message: format!(
4562                        "module_id '{target_module_id}' requires capability '{capability}', \
4563                         which no registered module provides; retry"
4564                    ),
4565                    detail: Some(serde_json::json!({
4566                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4567                        "capability": capability,
4568                    })),
4569                },
4570            )?]);
4571        }
4572
4573        if !target_has_required_role(&target, &registration.manifest.provides) {
4574            return Ok(vec![self.route_open_refusal_frame(
4575                ctx,
4576                &caller,
4577                &frame,
4578                &target_module_id,
4579                "role_not_provided",
4580                "target_unavailable",
4581                format!("module_id '{target_module_id}' does not provide the requested target"),
4582            )?]);
4583        }
4584
4585        if registration.state != ChannelState::Active {
4586            return Ok(vec![self.route_open_refusal_frame(
4587                ctx,
4588                &caller,
4589                &frame,
4590                &target_module_id,
4591                "registration_not_active",
4592                "target_unavailable",
4593                format!("module_id '{target_module_id}' is not active"),
4594            )?]);
4595        }
4596
4597        if self
4598            .forwarding
4599            .module_is_draining(&target_module_id)
4600            .map_err(RouterError::Forwarding)?
4601        {
4602            return Ok(vec![self.route_open_refusal_frame(
4603                ctx,
4604                &caller,
4605                &frame,
4606                &target_module_id,
4607                "reloading",
4608                "module_reloading",
4609                format!("module_id '{target_module_id}' is reloading"),
4610            )?]);
4611        }
4612
4613        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4614            process_liveness.process_live(&target_module_id) == Some(false)
4615        }) {
4616            // A module the supervisor is restarting or reloading can still hold
4617            // a registration: the old process before its connection closes, or
4618            // a new one that registered while the supervisor was draining. The
4619            // forwarding table does not see that as draining, but the consumer
4620            // should still be told to retry soon, exactly as for the drain
4621            // above, rather than that the target is unavailable.
4622            if process_liveness.process_replacing(&target_module_id) {
4623                return Ok(vec![self.route_open_refusal_frame(
4624                    ctx,
4625                    &caller,
4626                    &frame,
4627                    &target_module_id,
4628                    "reloading",
4629                    "module_reloading",
4630                    format!("module_id '{target_module_id}' is reloading"),
4631                )?]);
4632            }
4633            return Ok(vec![self.route_open_refusal_frame(
4634                ctx,
4635                &caller,
4636                &frame,
4637                &target_module_id,
4638                "supervisor_not_live",
4639                "target_unavailable",
4640                format!("module_id '{target_module_id}' is not live"),
4641            )?]);
4642        }
4643
4644        if !self
4645            .forwarding
4646            .has_live_module_connection(&target_module_id)
4647            .map_err(RouterError::Forwarding)?
4648        {
4649            return Ok(vec![self.route_open_refusal_frame(
4650                ctx,
4651                &caller,
4652                &frame,
4653                &target_module_id,
4654                "no_forwarding_connection",
4655                "target_unavailable",
4656                format!("module_id '{target_module_id}' has no live forwarding connection"),
4657            )?]);
4658        }
4659
4660        if let Some(error) =
4661            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4662        {
4663            self.observe_route_open_refusal(
4664                ctx,
4665                &caller,
4666                &target_module_id,
4667                "op_not_allowed",
4668                "op_not_allowed",
4669            );
4670            return Ok(vec![error]);
4671        }
4672
4673        let principal =
4674            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4675                Ok(principal) => principal,
4676                Err(error) => {
4677                    self.observe_route_open_refusal(
4678                        ctx,
4679                        &caller,
4680                        &target_module_id,
4681                        "bad_consumer_identity",
4682                        "bad_consumer_identity",
4683                    );
4684                    return Ok(vec![error]);
4685                }
4686            };
4687        timings.principal = principal_label(&principal);
4688
4689        // This is attested, control-plane policy for supervised module origins.
4690        // Keep it before route reservation and out of the opaque forwarding hot
4691        // path: data frames must never acquire a per-frame capability check.
4692        if let Principal::Reserved {
4693            module_id: opening_module_id,
4694        } = &principal
4695        {
4696            if let Some(opening_registration) = self
4697                .registry
4698                .get_module(opening_module_id)
4699                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4700            {
4701                if let Some(capability) =
4702                    denied_capability(&opening_registration.manifest, &registration.manifest)
4703                {
4704                    warn!(
4705                        opening_module_id,
4706                        target_module_id,
4707                        capability,
4708                        "refusing route.open because an attested capability deny edge matches"
4709                    );
4710                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4711                        &frame,
4712                        &target_module_id,
4713                        "capability_deny_edge",
4714                        "capability_forbidden",
4715                        format!(
4716                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4717                        ),
4718                    )?]);
4719                }
4720            }
4721        }
4722
4723        if admission_facts.is_some() {
4724            let carrier_matches = matches!(
4725                &principal,
4726                Principal::Reserved { module_id }
4727                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4728            );
4729            if !carrier_matches {
4730                return Ok(vec![self.route_open_refusal_frame(
4731                    ctx,
4732                    &caller,
4733                    &frame,
4734                    &target_module_id,
4735                    "admission_facts_carrier_not_permitted",
4736                    "admission_facts_not_permitted",
4737                    "admission facts may only be carried by the configured reserved module",
4738                )?]);
4739            }
4740
4741            let target_allowed = self
4742                .admission_facts_targets
4743                .as_ref()
4744                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4745            if !target_allowed {
4746                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4747                    &frame,
4748                    &target_module_id,
4749                    "admission_facts_target_not_listed",
4750                    "admission_facts_target_not_allowed",
4751                    format!(
4752                        "admission facts are not permitted for target module_id '{target_module_id}'"
4753                    ),
4754                )?]);
4755            }
4756
4757            // Keep the value opaque to subc. The downstream admission validator owns
4758            // schema and semantic checks; this daemon only enforces carrier authority
4759            // and the configured destination allowlist.
4760        }
4761
4762        // Scope admission, on the attested principal above and never on the
4763        // request body. The tag read here travels with the pending bind and is
4764        // compared with the published one at commit, so a sync between here
4765        // and the module's ack refuses the open instead of binding a stamp
4766        // that is no longer true.
4767        let (bound_scope, mut scope_stamp) = match &scope {
4768            None => (None, None),
4769            Some(selector) => {
4770                let owner_configured = match &selector.owner {
4771                    // A reserved owner counts as configured from before its
4772                    // process is spawned (see `SupervisorHandle::is_configured`).
4773                    // So an owner that has not synced its scopes yet is refused
4774                    // as retryable (`scope_not_synced`), not as one that will
4775                    // never sync.
4776                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4777                    _ => false,
4778                };
4779                let mut scopes = self
4780                    .scopes
4781                    .write()
4782                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4783                let admitted = scopes
4784                    .admit(&principal, &target_module_id, selector, owner_configured)
4785                    .and_then(|admission| {
4786                        crate::scopes::check_target_flow_support(
4787                            &admission.stamp,
4788                            &target_module_id,
4789                            registration.manifest.capabilities.as_ref(),
4790                        )?;
4791                        crate::scopes::check_target_agent_run_support(
4792                            &admission.stamp,
4793                            &target_module_id,
4794                            registration.manifest.capabilities.as_ref(),
4795                        )?;
4796                        let bound = scopes.reserve_bind(
4797                            &admission,
4798                            &principal,
4799                            ctx.connection_id,
4800                            selector.hold_presence,
4801                        );
4802                        Ok((admission, bound))
4803                    });
4804                match admitted {
4805                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4806                    Err(refusal) => {
4807                        return Ok(vec![self.route_open_refusal_frame(
4808                            ctx,
4809                            &caller,
4810                            &frame,
4811                            &target_module_id,
4812                            refusal.code,
4813                            refusal.code,
4814                            refusal.message,
4815                        )?]);
4816                    }
4817                }
4818            }
4819        };
4820        // From here on a refusal names the scope by the stamp admission
4821        // issued, exactly as the `route.open accepted` line does.
4822        if let Some(stamp) = &scope_stamp {
4823            caller.scope = route_open_scope_label(stamp);
4824        }
4825
4826        // Bind admits a root that no longer exists on disk, because refusing here
4827        // closes the only exit from a paused run: cancel needs a bound route, and a
4828        // renamed or reclaimed directory makes that route unopenable forever. The
4829        // run itself is intact and still addressable by its recorded identity.
4830        //
4831        // This does NOT relax the rule the strict constructor protects. That rule is
4832        // that no root is ever aliased into NEW durable state -- a missing component
4833        // can reappear as a symlink elsewhere, which would move the identity and
4834        // split a session's history across two of them. The engine now refuses the
4835        // two operations that create such state (send and import) at admission,
4836        // which is a narrower way to hold the same invariant: reads and terminations
4837        // are admitted, writes are not. That refusal had to ship before this line
4838        // changed, or there is an interval where a send commits under a provisional
4839        // identity -- the exact failure the original policy existed to prevent.
4840        //
4841        // Resolution follows realpath rather than lexical cleanup: the longest
4842        // existing ancestor is canonicalized and the missing tail re-appended, so a
4843        // live root is unchanged and a vanished leaf keeps the identity it was
4844        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4845        // same caller the moment the directory vanished, which strands the run more
4846        // quietly than refusing it.
4847        timings.enter(RouteOpenPhase::ProjectRoot);
4848        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4849            Ok(project_root) => project_root,
4850            Err(err) => {
4851                return Ok(vec![control_error_frame(
4852                    &frame,
4853                    "invalid_project_root",
4854                    err.to_string(),
4855                )?])
4856            }
4857        };
4858        timings.enter(RouteOpenPhase::Admission);
4859        identity.project_root = project_root.as_path().to_path_buf();
4860
4861        // Last gate before any relay work, and deliberately after the cheap
4862        // registry and availability checks above: those name a more precise
4863        // condition (unknown, removed, reloading) and a caller is better served
4864        // by the precise code than by this one.
4865        //
4866        // Everything below this point costs an egress permit, a reserved handle
4867        // pair and, if the module does not answer, the whole relay budget. The
4868        // reader no longer waits for that budget, so cap each target explicitly;
4869        // serial dispatch used to provide the accidental cap of one relay per
4870        // connection. Admission is a mutex-protected count and never waits.
4871        let _concurrency_guard = match self
4872            .route_bind_concurrency
4873            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4874        {
4875            Ok(guard) => guard,
4876            Err(in_flight) => {
4877                return Ok(vec![self.route_open_target_capacity_refusal(
4878                    ctx,
4879                    &caller,
4880                    &frame,
4881                    &target_module_id,
4882                    in_flight,
4883                )?]);
4884            }
4885        };
4886
4887        // A module that has already burned the whole budget `threshold` times
4888        // in a row does not get to charge it again until a probe says it recovered.
4889        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4890            RouteBindAdmission::Admitted { guard, probe } => {
4891                if probe {
4892                    info!(
4893                        module_id = %target_module_id,
4894                        connection_id = ctx.connection_id.get(),
4895                        "route.bind breaker half-open: admitting one probe"
4896                    );
4897                }
4898                guard
4899            }
4900            RouteBindAdmission::Refused {
4901                consecutive_timeouts,
4902                retry_in,
4903                probe_in_flight,
4904            } => {
4905                return Ok(vec![self.route_open_breaker_refusal_frame(
4906                    ctx,
4907                    &caller,
4908                    &frame,
4909                    &target_module_id,
4910                    consecutive_timeouts,
4911                    retry_in,
4912                    probe_in_flight,
4913                )?]);
4914            }
4915        };
4916
4917        // Resolve the per-module budget here so the wait matches the operator's
4918        // intent for this specific target. A per-module override in
4919        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4920        // daemons) wins over the daemon-wide default.
4921        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4922        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4923        timings.enter(RouteOpenPhase::EgressReservation);
4924        let pending = match self
4925            .forwarding
4926            .begin_route_bind_relay_for(
4927                ctx.connection_id,
4928                ctx.egress.clone(),
4929                response_version(&frame),
4930                frame.header.corr,
4931                &target_module_id,
4932                principal.clone(),
4933                bound_scope,
4934                Some(project_root),
4935                relay_deadline,
4936            )
4937            .await
4938        {
4939            Ok(pending) => pending,
4940            Err(err) => {
4941                return Ok(vec![self.route_open_refusal_frame(
4942                    ctx,
4943                    &caller,
4944                    &frame,
4945                    &target_module_id,
4946                    "relay_reservation_failed",
4947                    forwarding_error_code(&err),
4948                    err.to_string(),
4949                )?])
4950            }
4951        };
4952        timings.enter(RouteOpenPhase::Admission);
4953        let crate::forwarding::PendingRouteBindRelay {
4954            endpoint,
4955            module_sink,
4956            negotiated_ver,
4957            client_channel,
4958            client_epoch,
4959            module_channel,
4960            module_epoch,
4961            corr: relay_corr,
4962            receiver,
4963        } = pending;
4964        let mut reservation =
4965            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4966
4967        // Reserving egress can wait while a module reconnects or a swap cuts
4968        // over. Check the relay's captured connection, not the earlier by-id
4969        // lookup: the original module's flow/run capabilities cannot authorize
4970        // its replacement. The captured sink stays bound to that connection.
4971        let relay_registration = self
4972            .registry
4973            .get_module_by_connection(endpoint.connection_id)
4974            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4975        let relay_capabilities = relay_registration
4976            .as_ref()
4977            .and_then(|registration| registration.manifest.capabilities.as_ref());
4978        if let Some(selector) = &scope {
4979            // Egress reservation may wait across a scope update. Recheck the
4980            // proof against the current record before sending the captured bind.
4981            if let Err(refusal) = self
4982                .scopes
4983                .read()
4984                .unwrap_or_else(|poisoned| poisoned.into_inner())
4985                .check_bind_proof(&principal, selector)
4986            {
4987                reservation.release_and_disarm();
4988                return Ok(vec![self.route_open_refusal_frame(
4989                    ctx,
4990                    &caller,
4991                    &frame,
4992                    &target_module_id,
4993                    refusal.code,
4994                    refusal.code,
4995                    refusal.message,
4996                )?]);
4997            }
4998        }
4999        if let Some(stamp) = scope_stamp
5000            .as_ref()
5001            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5002        {
5003            if let Err(refusal) = crate::scopes::check_target_flow_support(
5004                stamp,
5005                &target_module_id,
5006                relay_capabilities,
5007            )
5008            .and_then(|()| {
5009                crate::scopes::check_target_agent_run_support(
5010                    stamp,
5011                    &target_module_id,
5012                    relay_capabilities,
5013                )
5014            }) {
5015                reservation.release_and_disarm();
5016                return Ok(vec![self.route_open_refusal_frame(
5017                    ctx,
5018                    &caller,
5019                    &frame,
5020                    &target_module_id,
5021                    refusal.code,
5022                    refusal.code,
5023                    refusal.message,
5024                )?]);
5025            }
5026        }
5027
5028        // Older readers may reject unknown fields. Select optional fields from
5029        // the captured endpoint's declaration, including on unscoped binds.
5030        if !relay_capabilities.is_some_and(|caps| {
5031            caps.provides
5032                .iter()
5033                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5034        }) {
5035            if let Some(stamp) = scope_stamp.as_mut() {
5036                stamp.workdir = None;
5037                stamp.project_id = None;
5038            }
5039        }
5040        if !relay_capabilities.is_some_and(|caps| {
5041            caps.provides
5042                .iter()
5043                .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5044        }) {
5045            if let Some(stamp) = scope_stamp.as_mut() {
5046                stamp.attributes.on_behalf_of_agent = None;
5047            }
5048        }
5049        let call_mode = if relay_capabilities
5050            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5051        {
5052            Some(derive_call_mode(scope_stamp.as_ref()))
5053        } else {
5054            if let Some(stamp) = scope_stamp.as_mut() {
5055                stamp.expires_at_ms = None;
5056            }
5057            None
5058        };
5059
5060        debug!(
5061            connection_id = ctx.connection_id.get(),
5062            client_channel,
5063            client_epoch,
5064            module_channel,
5065            module_epoch,
5066            "reserved route handle pair"
5067        );
5068        // Rendered BEFORE the move into the relay, because the accept arm below
5069        // is where it is logged and the principal is gone by then.
5070        let route_principal_label = principal_label(&principal);
5071        let relay = ModuleControlRequest::RouteBind {
5072            route_channel: module_channel,
5073            epoch: module_epoch,
5074            target,
5075            identity,
5076            principal: Some(principal),
5077            consumer_capabilities,
5078            role_versions,
5079            admission_facts,
5080            scope: scope_stamp,
5081            call_mode,
5082        };
5083        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5084            RouterError::backend(
5085                0,
5086                frame.header.corr,
5087                format!("failed to encode route.bind request: {err}"),
5088            )
5089        })?;
5090        let relay_frame = Frame::build_with_version(
5091            negotiated_ver,
5092            FrameType::Request,
5093            control_flags(),
5094            0,
5095            0,
5096            relay_corr,
5097            relay_body,
5098        )
5099        .map_err(RouterError::FrameBuild)?;
5100
5101        timings.enter(RouteOpenPhase::BindSend);
5102        let sent = module_sink.send(relay_frame).await;
5103        timings.enter(RouteOpenPhase::Admission);
5104        if let Err(err) = sent {
5105            reservation.release_and_disarm();
5106            return Ok(vec![self.route_open_refusal_frame(
5107                ctx,
5108                &caller,
5109                &frame,
5110                &target_module_id,
5111                "relay_send_failed",
5112                "target_unavailable",
5113                err.to_string(),
5114            )?]);
5115        }
5116
5117        #[cfg(test)]
5118        if let Some(hold) = &self.route_bind_relay_sent_hold {
5119            // First wait: tell the test the bind is queued. Second wait: stay
5120            // here until the test has driven whatever it wants to happen
5121            // between queueing the bind and marking it enqueued.
5122            hold.wait().await;
5123            hold.wait().await;
5124        }
5125
5126        // A missing relay is not by itself an abandoned bind: the module may
5127        // already have acked it and the route may be committed. Only a relay
5128        // dropped without any module reply needs this GOODBYE; one a reply
5129        // settled has already had its GOODBYE decided by the reply path.
5130        if self
5131            .forwarding
5132            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5133            .map_err(RouterError::Forwarding)?
5134            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5135        {
5136            self.send_abandoned_route_bind_goodbye(
5137                &module_sink,
5138                negotiated_ver,
5139                module_channel,
5140                module_epoch,
5141            );
5142        }
5143
5144        timings.enter(RouteOpenPhase::BindAckWait);
5145        let outcome = timeout_at(relay_deadline, receiver).await;
5146        timings.enter(RouteOpenPhase::Admission);
5147        match outcome {
5148            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5149                reservation.disarm();
5150                if breaker.record_accepted() {
5151                    info!(
5152                        module_id = %target_module_id,
5153                        "route.bind breaker closed: the probe was accepted"
5154                    );
5155                }
5156                self.observe_route_open_accept(
5157                    ctx,
5158                    &target_module_id,
5159                    &route_principal_label,
5160                    &caller.scope,
5161                );
5162                Ok(Vec::new())
5163            }
5164            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5165                reservation.release_and_disarm();
5166                // A module that says no in microseconds is healthy. Rejection
5167                // is a different condition with its own refusal and must not
5168                // move the breaker.
5169                breaker.record_inconclusive();
5170                // The daemon's own commit re-check refused the bind because the
5171                // scope ended or changed after admission. The module accepted;
5172                // counting it as a module rejection would blame the module.
5173                let scope_code = match body.code.as_str() {
5174                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5175                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5176                    _ => None,
5177                };
5178                if let Some(code) = scope_code {
5179                    self.observe_route_open_refusal(
5180                        ctx,
5181                        &caller,
5182                        &target_module_id,
5183                        "scope_changed_before_commit",
5184                        code,
5185                    );
5186                    return Ok(vec![control_error_body_frame(&frame, body)?]);
5187                }
5188                self.counters
5189                    .increment_route_open_refused("module_rejected");
5190                info!(
5191                    target: "control",
5192                    code = "module_rejected",
5193                    module_code = ?body.code,
5194                    module_id = ?target_module_id,
5195                    connection_id = ctx.connection_id.get(),
5196                    principal = %caller.principal,
5197                    scope = %caller.scope,
5198                    harness = %caller.harness,
5199                    "route.open refused"
5200                );
5201                Ok(vec![control_error_body_frame(&frame, body)?])
5202            }
5203            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5204                reservation.release_and_disarm();
5205                breaker.record_inconclusive();
5206                // Fires when the module's connection closes while a relayed
5207                // bind is pending -- typically a caller racing a module restart
5208                // whose bind was relayed BEFORE the drain mark went up. Logged
5209                // because the caller sees only its own error and the fleet has
5210                // already spent one diagnosis round unable to tell this arm
5211                // from a relay timeout without daemon-side evidence.
5212                tracing::warn!(
5213                    module_id = %target_module_id,
5214                    "route.bind relay abandoned: {message}"
5215                );
5216                Ok(vec![self.route_open_refusal_frame(
5217                    ctx,
5218                    &caller,
5219                    &frame,
5220                    &target_module_id,
5221                    "relay_abandoned",
5222                    "target_unavailable",
5223                    message,
5224                )?])
5225            }
5226            Ok(Err(_)) => {
5227                reservation.release_and_disarm();
5228                breaker.record_inconclusive();
5229                Ok(vec![self.route_open_refusal_frame(
5230                    ctx,
5231                    &caller,
5232                    &frame,
5233                    &target_module_id,
5234                    "relay_waiter_canceled",
5235                    "target_unavailable",
5236                    "route.bind relay waiter was canceled before the module responded",
5237                )?])
5238            }
5239            Err(_) => {
5240                reservation.release_and_disarm();
5241                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5242                // answer at all is the one condition a fast refusal can
5243                // usefully stand in for; every other arm already answered.
5244                if let Some(opened) = breaker.record_timeout(
5245                    self.route_bind_breaker_threshold,
5246                    self.route_bind_breaker_cooldown,
5247                ) {
5248                    warn!(
5249                        module_id = %target_module_id,
5250                        consecutive_timeouts = opened.consecutive_timeouts,
5251                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5252                        reopened_after_probe = opened.reopened_after_probe,
5253                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5254                    );
5255                }
5256                // The generous budget just burned to no answer: the module is
5257                // registered and its connection is up, but its bind handler sat
5258                // on the ack for the full budget (warm-on-bind, cold configure,
5259                // or a wedged handler). Every earlier unavailability shape
5260                // fast-refuses BEFORE the relay, so this arm firing means the
5261                // slowness is module-side -- log it so the per-module timeline
5262                // is reconstructable without client audit rows.
5263                tracing::warn!(
5264                    module_id = %target_module_id,
5265                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5266                    "route.bind relay timed out: module did not ack within budget"
5267                );
5268                Ok(vec![self.route_open_refusal_frame(
5269                    ctx,
5270                    &caller,
5271                    &frame,
5272                    &target_module_id,
5273                    "relay_timed_out",
5274                    "module_timeout",
5275                    format!(
5276                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5277                        route_bind_relay_timeout
5278                    ),
5279                )?])
5280            }
5281        }
5282    }
5283
5284    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5285        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5286            snapshot: self.supervisor.spawn_snapshot(),
5287        };
5288        Ok(vec![control_response_body_frame(
5289            &frame,
5290            &response,
5291            "ClientControlResponse::SupervisorSpawnSnapshot",
5292        )?])
5293    }
5294
5295    fn handle_supervisor_spawn_subscribe(
5296        &self,
5297        ctx: &RouteCtx,
5298        frame: Frame,
5299        since: Option<SpawnCursor>,
5300    ) -> Result<Vec<Frame>, RouterError> {
5301        match self.supervisor.subscribe_spawns(
5302            ctx.connection_id,
5303            frame.header.corr,
5304            response_version(&frame),
5305            since,
5306            ctx.egress.clone(),
5307        ) {
5308            Ok(()) => Ok(Vec::new()),
5309            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5310                Ok(vec![control_error_body_frame(
5311                    &frame,
5312                    ErrorBody {
5313                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5314                        message: "spawn cursor belongs to a different daemon incarnation"
5315                            .to_string(),
5316                        detail: Some(serde_json::json!({
5317                            "current_daemon_incarnation": current
5318                        })),
5319                    },
5320                )?])
5321            }
5322            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5323                &frame,
5324                ErrorBody {
5325                    code: "spawn_cursor_too_old".to_string(),
5326                    message: "spawn cursor predates the retained event ring".to_string(),
5327                    detail: Some(serde_json::json!({
5328                        "oldest_retained_cursor": oldest
5329                    })),
5330                },
5331            )?]),
5332            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5333                0,
5334                frame.header.corr,
5335                format!("failed to open supervisor spawn subscription: {error}"),
5336            )),
5337        }
5338    }
5339
5340    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5341        let generation = self
5342            .registry
5343            .generation()
5344            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5345        let mut modules = Vec::new();
5346        for module in self.supervisor.list() {
5347            let status = module.status_for_control("list").map_err(|err| {
5348                RouterError::backend(
5349                    0,
5350                    frame.header.corr,
5351                    format!("failed to read supervisor status: {err}"),
5352                )
5353            })?;
5354            let (configured, _) = module.configuration().map_err(|err| {
5355                RouterError::backend(
5356                    0,
5357                    frame.header.corr,
5358                    format!("failed to read module configuration: {err}"),
5359                )
5360            })?;
5361            // Status and configuration snapshots release their locks before the image probe awaits.
5362            let image = module.running_image_agreement().await;
5363            // Read per request so the figure is current when the operator asks;
5364            // the daemon samples nothing in between.
5365            let resources = Some(module.child_resource_usage());
5366            let mut verdict =
5367                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5368            verdict.spec = Some(launch_spec_agreement(
5369                &configured,
5370                status.process_alive,
5371                status.launch_spec_digest.as_ref(),
5372            ));
5373            let pending_reload = Some(verdict);
5374            let configured_enabled = self
5375                .capability_evaluator
5376                .configured_enabled(&status.module_id);
5377            // Keep the retired policy field on the wire for one release so
5378            // existing status consumers still receive the platform policy.
5379            modules.push(
5380                SupervisorEntry::new(
5381                    status.module_id,
5382                    status.state.to_string(),
5383                    status.enabled,
5384                    status.live,
5385                    status.health.status,
5386                )
5387                .with_configured_enabled(configured_enabled)
5388                .with_launch_nonce_env(Some(!cfg!(unix)))
5389                .with_launch_nonce_source(status.launch_nonce_source)
5390                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5391                .with_protocol(status.protocol)
5392                .with_pending_reload(pending_reload)
5393                .with_last_probe_ms(status.health.last_probe_ms)
5394                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5395                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5396                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5397                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5398                .with_restart_count(Some(status.restart_count))
5399                .with_max_restarts(Some(status.max_restarts))
5400                .with_lifetime_restarts(Some(status.lifetime_restarts))
5401                .with_spawn_generation(Some(status.spawn_generation))
5402                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5403                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5404                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5405                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5406                .with_resources(resources),
5407            );
5408        }
5409        let response = ClientControlResponse::SupervisorList {
5410            generation,
5411            modules,
5412        };
5413        Ok(vec![control_response_body_frame(
5414            &frame,
5415            &response,
5416            "ClientControlResponse::SupervisorList",
5417        )?])
5418    }
5419
5420    fn handle_supervisor_stderr_tail(
5421        &self,
5422        frame: Frame,
5423        module_id: String,
5424        max_lines: Option<u32>,
5425        max_bytes: Option<u32>,
5426    ) -> Result<Vec<Frame>, RouterError> {
5427        let Some(module) = self.supervisor.get(&module_id) else {
5428            return Ok(vec![control_error_frame(
5429                &frame,
5430                "unknown_module",
5431                format!("module_id '{module_id}' is not supervised"),
5432            )?]);
5433        };
5434
5435        let snapshot = module.stderr_tail(
5436            max_lines.map(|value| value as usize),
5437            max_bytes.map(|value| value as usize),
5438        );
5439
5440        let response = ClientControlResponse::SupervisorStderrTail {
5441            module_id,
5442            tail: StderrTail {
5443                capture: match snapshot.capture {
5444                    CaptureState::Captured => StderrCaptureState::Captured,
5445                    CaptureState::Incomplete { reason } => {
5446                        StderrCaptureState::Incomplete { reason }
5447                    }
5448                    CaptureState::NotCaptured { reason } => {
5449                        StderrCaptureState::NotCaptured { reason }
5450                    }
5451                },
5452                entries: snapshot
5453                    .entries
5454                    .into_iter()
5455                    .map(|entry| match entry {
5456                        TailEntry::Line {
5457                            text,
5458                            truncated,
5459                            at_ms,
5460                        } => StderrTailEntry::Line {
5461                            text,
5462                            truncated,
5463                            at_ms,
5464                        },
5465                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5466                    })
5467                    .collect(),
5468                dropped_lines: snapshot.dropped_lines,
5469            },
5470        };
5471        Ok(vec![control_response_body_frame(
5472            &frame,
5473            &response,
5474            "ClientControlResponse::SupervisorStderrTail",
5475        )?])
5476    }
5477
5478    async fn handle_supervisor_terminals(
5479        &self,
5480        frame: Frame,
5481        module_id: String,
5482    ) -> Result<Vec<Frame>, RouterError> {
5483        let Some(module) = self.supervisor.get(&module_id) else {
5484            return Ok(vec![control_error_frame(
5485                &frame,
5486                "unknown_module",
5487                format!("module_id '{module_id}' is not supervised"),
5488            )?]);
5489        };
5490
5491        // The journal read runs on a blocking thread: it can be megabytes of
5492        // file I/O and must not occupy a runtime worker.
5493        let terminals = module
5494            .read_durable_terminal_history()
5495            .await
5496            .map_err(|error| {
5497                RouterError::backend(
5498                    0,
5499                    frame.header.corr,
5500                    format!("failed to read terminal history: {error}"),
5501                )
5502            })?;
5503        let response = ClientControlResponse::SupervisorTerminals {
5504            module_id,
5505            terminals,
5506        };
5507        Ok(vec![control_response_body_frame(
5508            &frame,
5509            &response,
5510            "ClientControlResponse::SupervisorTerminals",
5511        )?])
5512    }
5513
5514    fn handle_supervisor_routes(
5515        &self,
5516        frame: Frame,
5517        module_id: Option<String>,
5518    ) -> Result<Vec<Frame>, RouterError> {
5519        let modules = self
5520            .forwarding
5521            .route_census(module_id.as_deref())
5522            .map_err(RouterError::Forwarding)?
5523            .into_iter()
5524            .map(|(module_id, routes)| SupervisorRouteModule {
5525                module_id,
5526                routes: routes
5527                    .into_iter()
5528                    .map(|route| SupervisorRoute {
5529                        consumer: match route.principal {
5530                            Principal::Reserved { module_id } => {
5531                                SupervisorRouteConsumer::Reserved { module_id }
5532                            }
5533                            Principal::Direct | Principal::Unverified => {
5534                                SupervisorRouteConsumer::Direct {
5535                                    connection_id: route.goodbye_target.connection_id.get(),
5536                                }
5537                            }
5538                        },
5539                        age_ms: Instant::now()
5540                            .saturating_duration_since(route.bound_at)
5541                            .as_millis()
5542                            .try_into()
5543                            .unwrap_or(u64::MAX),
5544                        draining: route.draining,
5545                        drain_reason: route.drain_reason,
5546                    })
5547                    .collect(),
5548            })
5549            .collect();
5550        let response = ClientControlResponse::SupervisorRoutes { modules };
5551        Ok(vec![control_response_body_frame(
5552            &frame,
5553            &response,
5554            "ClientControlResponse::SupervisorRoutes",
5555        )?])
5556    }
5557
5558    async fn handle_supervisor_provenance(
5559        &self,
5560        frame: Frame,
5561        module_id: Option<String>,
5562    ) -> Result<Vec<Frame>, RouterError> {
5563        let mut selected = if let Some(module_id) = module_id {
5564            let Some(module) = self.supervisor.get(&module_id) else {
5565                return Ok(vec![control_error_frame(
5566                    &frame,
5567                    "unknown_module",
5568                    format!("module_id '{module_id}' is not supervised"),
5569                )?]);
5570            };
5571            vec![module]
5572        } else {
5573            self.supervisor.list()
5574        };
5575
5576        let mut modules = Vec::with_capacity(selected.len());
5577        for module in selected.drain(..) {
5578            let (status, observed_image) = module
5579                .status_and_running_image_agreement()
5580                .await
5581                .map_err(|err| {
5582                    RouterError::backend(
5583                        0,
5584                        frame.header.corr,
5585                        format!("failed to read supervisor status: {err}"),
5586                    )
5587                })?;
5588            let module_declared = self
5589                .registry
5590                .get_module(&status.module_id)
5591                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5592                .and_then(|registration| registration.manifest.provenance)
5593                .map(|build| ModuleDeclaredProvenance::Reported { build })
5594                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5595            #[cfg(test)]
5596            let running_image = match &self.provenance_probe_override {
5597                Some(result) => result.clone(),
5598                None => observed_image,
5599            };
5600            #[cfg(not(test))]
5601            let running_image = observed_image;
5602            modules.push(SupervisorModuleProvenance {
5603                module_id: status.module_id,
5604                module_declared,
5605                daemon_observed: SupervisorObservedProcess {
5606                    launch_nonce_source: status.launch_nonce_source,
5607                    pid: status.pid,
5608                    spawned_at_ms: status.spawned_at_ms,
5609                    spawned_from: status.spawned_from,
5610                    running_image,
5611                },
5612            });
5613        }
5614        let daemon = SupervisorDaemonProvenance {
5615            daemon_build: self.daemon_provenance.build.clone(),
5616            daemon_observed: DaemonObservedProcess {
5617                pid: self.daemon_provenance.pid,
5618                started_at_ms: self
5619                    .daemon_provenance
5620                    .start_clock
5621                    .map(|clock| clock.started_at_ms())
5622                    .or(self.daemon_provenance.started_at_ms),
5623                running_image: self
5624                    .daemon_provenance
5625                    .probe
5626                    .observe(
5627                        self.daemon_provenance.pid,
5628                        self.daemon_provenance.executable_path.as_deref(),
5629                        self.daemon_provenance.executable_identity.clone(),
5630                        self.daemon_provenance.process_start_time,
5631                    )
5632                    .await,
5633            },
5634        };
5635        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5636        Ok(vec![control_response_body_frame(
5637            &frame,
5638            &response,
5639            "ClientControlResponse::SupervisorProvenance",
5640        )?])
5641    }
5642
5643    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5644        self.refresh_capability_requirements();
5645        let generation = self
5646            .registry
5647            .generation()
5648            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5649        let modules = self
5650            .supervisor
5651            .list()
5652            .into_iter()
5653            .map(|module| {
5654                let status = module.status_for_control("health").map_err(|err| {
5655                    RouterError::backend(
5656                        0,
5657                        frame.header.corr,
5658                        format!("failed to read supervisor health: {err}"),
5659                    )
5660                })?;
5661                let module_id = status.module_id;
5662                let capability_detail = self
5663                    .capability_evaluator
5664                    .required_problem_detail(&module_id);
5665                Ok(SupervisorHealthEntry {
5666                    module_id,
5667                    status: status.health.status,
5668                    detail: append_capability_problem_detail(
5669                        status.health.detail,
5670                        capability_detail,
5671                    ),
5672                    metrics: status.health.metrics,
5673                    consecutive_failures: status.health.consecutive_failures,
5674                    late_answer_count: status.health.late_answer_count,
5675                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5676                    last_action: status.health.last_action,
5677                    last_action_ms: status.health.last_action_ms,
5678                    last_probe_ms: status.health.last_probe_ms,
5679                })
5680            })
5681            .collect::<Result<Vec<_>, RouterError>>()?;
5682        let response = ClientControlResponse::SupervisorHealth {
5683            generation,
5684            modules,
5685        };
5686        Ok(vec![control_response_body_frame(
5687            &frame,
5688            &response,
5689            "ClientControlResponse::SupervisorHealth",
5690        )?])
5691    }
5692
5693    async fn handle_supervisor_restart(
5694        &self,
5695        frame: Frame,
5696        module_id: String,
5697        drain_timeout_ms: Option<u64>,
5698        requester: Principal,
5699    ) -> Result<Vec<Frame>, RouterError> {
5700        let operation_lock = self.supervisor.operation_lock();
5701        let _operation_guard = operation_lock.lock().await;
5702        let Some(module) = self.supervisor.get(&module_id) else {
5703            return Ok(vec![control_error_frame(
5704                &frame,
5705                "unknown_module",
5706                format!("module_id '{module_id}' is not supervised"),
5707            )?]);
5708        };
5709
5710        self.route_outages.mark_operator_action(&module_id);
5711        if let Err(err) = module
5712            .restart_requested(drain_timeout_ms, Some(requester))
5713            .await
5714        {
5715            self.route_outages
5716                .operator_action_ended_unrefused(&module_id);
5717            let (code, message) = match err {
5718                crate::supervise::SuperviseError::Disabled { .. } => {
5719                    ("module_disabled", err.to_string())
5720                }
5721                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5722                    ("swap_in_progress", err.to_string())
5723                }
5724                _ => (
5725                    "target_unavailable",
5726                    format!("failed to restart module_id '{module_id}': {err}"),
5727                ),
5728            };
5729            return Ok(vec![control_error_frame(&frame, code, message)?]);
5730        }
5731
5732        let response = ClientControlResponse::SupervisorAck {
5733            module_id,
5734            applied: true,
5735        };
5736        Ok(vec![control_response_body_frame(
5737            &frame,
5738            &response,
5739            "ClientControlResponse::SupervisorAck",
5740        )?])
5741    }
5742
5743    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5744    /// when the old process has finished draining: a caller whose own lane
5745    /// rides the old process must get its reply before that drain waits on it.
5746    async fn handle_supervisor_swap(
5747        &self,
5748        frame: Frame,
5749        module_id: String,
5750        ready_timeout_ms: Option<u64>,
5751        requester: Principal,
5752    ) -> Result<Vec<Frame>, RouterError> {
5753        // The daemon-wide operation lock is held only to resolve the handle,
5754        // not across the swap. The swap can take its whole readiness budget,
5755        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5756        // holding it here would park an operator's stop behind the swap it is
5757        // meant to abort. A rescan or stop that reaches the module during the
5758        // swap is served by the swap itself (see `supervise_swap`).
5759        let module = {
5760            let operation_lock = self.supervisor.operation_lock();
5761            let _operation_guard = operation_lock.lock().await;
5762            self.supervisor.get(&module_id)
5763        };
5764        let Some(module) = module else {
5765            return Ok(vec![control_error_frame(
5766                &frame,
5767                "unknown_module",
5768                format!("module_id '{module_id}' is not supervised"),
5769            )?]);
5770        };
5771
5772        self.route_outages.mark_operator_action(&module_id);
5773        if let Err(err) = module
5774            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5775            .await
5776        {
5777            self.route_outages
5778                .operator_action_ended_unrefused(&module_id);
5779            use crate::supervise::SuperviseError;
5780            let message = err.to_string();
5781            let error = match err {
5782                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5783                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5784                    code: "swap_refused".to_string(),
5785                    message,
5786                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5787                },
5788                SuperviseError::SwapFailed {
5789                    arm,
5790                    candidate_exit,
5791                    ..
5792                } => ErrorBody {
5793                    code: "swap_failed".to_string(),
5794                    message,
5795                    detail: Some(serde_json::json!({
5796                        "arm": arm.as_str(),
5797                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5798                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5799                    })),
5800                },
5801                _ => ErrorBody::new(
5802                    "target_unavailable",
5803                    format!("failed to swap module_id '{module_id}': {message}"),
5804                ),
5805            };
5806            return Ok(vec![control_error_body_frame(&frame, error)?]);
5807        }
5808        // A completed swap kept the incumbent serving until cutover, so it
5809        // usually opened no outage; a mark left behind would make the next,
5810        // unrelated outage read as requested.
5811        self.route_outages
5812            .operator_action_ended_unrefused(&module_id);
5813
5814        let response = ClientControlResponse::SupervisorAck {
5815            module_id,
5816            applied: true,
5817        };
5818        Ok(vec![control_response_body_frame(
5819            &frame,
5820            &response,
5821            "ClientControlResponse::SupervisorAck",
5822        )?])
5823    }
5824
5825    async fn handle_supervisor_reload(
5826        &self,
5827        frame: Frame,
5828        module_id: String,
5829        requester: Principal,
5830    ) -> Result<Vec<Frame>, RouterError> {
5831        let operation_lock = self.supervisor.operation_lock();
5832        let _operation_guard = operation_lock.lock().await;
5833        let Some(module) = self.supervisor.get(&module_id) else {
5834            return Ok(vec![control_error_frame(
5835                &frame,
5836                "unknown_module",
5837                format!("module_id '{module_id}' is not supervised"),
5838            )?]);
5839        };
5840
5841        self.route_outages.mark_operator_action(&module_id);
5842        if let Err(err) = module.reload_requested(Some(requester)).await {
5843            self.route_outages
5844                .operator_action_ended_unrefused(&module_id);
5845            let (code, message) = match err {
5846                crate::supervise::SuperviseError::Disabled { .. } => {
5847                    ("module_disabled", err.to_string())
5848                }
5849                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5850                    ("swap_in_progress", err.to_string())
5851                }
5852                _ => (
5853                    "reload_failed",
5854                    format!("failed to reload module_id '{module_id}': {err}"),
5855                ),
5856            };
5857            return Ok(vec![control_error_frame(&frame, code, message)?]);
5858        }
5859
5860        let response = ClientControlResponse::SupervisorAck {
5861            module_id,
5862            applied: true,
5863        };
5864        Ok(vec![control_response_body_frame(
5865            &frame,
5866            &response,
5867            "ClientControlResponse::SupervisorAck",
5868        )?])
5869    }
5870
5871    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5872    /// the configuration comes from and whether to apply it; every mode then
5873    /// runs the same comparison, so a preview describes the apply it previews.
5874    async fn handle_supervisor_rescan(
5875        &self,
5876        frame: Frame,
5877        mode: RescanMode,
5878        requester: Principal,
5879    ) -> Result<Vec<Frame>, RouterError> {
5880        let preview = mode.previews();
5881        let Some(context) = self.rescan.clone() else {
5882            return Ok(vec![control_error_frame(
5883                &frame,
5884                "rescan_unavailable",
5885                "the daemon was not started with a reloadable config path".to_string(),
5886            )?]);
5887        };
5888
5889        let operation_lock = self.supervisor.operation_lock();
5890        let _operation_guard = operation_lock.lock().await;
5891        // Loaded under the operation lock, so no other rescan can change the
5892        // staged configuration between a checked apply's comparison and the
5893        // apply itself.
5894        let loaded = match self.load_rescan_config(&mode, &context) {
5895            Ok(config) => config,
5896            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5897        };
5898        // `load` reports a missing file as Ok(None), which is correct at boot
5899        // (no config, nothing to supervise) and catastrophic here: rescan treats
5900        // "not in the config" as "remove it", so an absent file would read as an
5901        // empty module list and retire the entire running fleet. An editor
5902        // writing via write-new-then-rename, or a half-finished edit, is enough
5903        // to open that window. Refuse instead: a config that cannot be read
5904        // carries no instruction to remove anything.
5905        let Some((config, activity_config)) = loaded else {
5906            return Ok(vec![control_error_frame(
5907                &frame,
5908                "invalid_daemon_config",
5909                format!(
5910                    "daemon config not found at {}; refusing to rescan (an absent config would \
5911                     retire every supervised module)",
5912                    context.config_path.display()
5913                ),
5914            )?]);
5915        };
5916        let (
5917            configured_port,
5918            storage_config,
5919            admission_facts_carrier_module_id,
5920            admission_facts_targets,
5921            scope_authority_owners,
5922            privileged_operator_prompt,
5923            modules,
5924            reserved_capabilities,
5925        ) = (
5926            config.port,
5927            config.storage,
5928            config.admission_facts_carrier_module_id,
5929            config.admission_facts_targets,
5930            config.scope_authority_owners,
5931            config.privileged_operator_prompt,
5932            config.modules,
5933            config.reserved_capabilities,
5934        );
5935
5936        // Collect the sections rescan cannot apply, so the REPLY carries them.
5937        //
5938        // The warning below has always been correct and has always gone only to
5939        // the journal -- addressed to whoever reads logs, while the person who
5940        // just edited the config is looking at the CLI. Naming each section
5941        // individually rather than setting a flag: "something outside modules
5942        // changed" sends the operator back to diffing their own file, which is
5943        // the work this is meant to save.
5944        let mut restart_required = Vec::new();
5945        for section in RestartRequiredSection::ALL {
5946            let changed = match section {
5947                RestartRequiredSection::Port => configured_port != context.configured_port,
5948                RestartRequiredSection::Storage => storage_config != context.storage_config,
5949                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5950                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5951                }
5952                RestartRequiredSection::AdmissionFactsTargets => {
5953                    admission_facts_targets != context.admission_facts_targets
5954                }
5955                RestartRequiredSection::ScopeAuthorityOwners => {
5956                    scope_authority_owners != context.scope_authority_owners
5957                }
5958                RestartRequiredSection::PrivilegedOperatorPrompt => {
5959                    privileged_operator_prompt != context.privileged_operator_prompt
5960                }
5961            };
5962            if changed {
5963                restart_required.push(section.label().to_string());
5964            }
5965        }
5966        if !restart_required.is_empty() {
5967            warn!(
5968                config_path = %context.config_path.display(),
5969                sections = %restart_required.join(", "),
5970                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5971            );
5972        }
5973
5974        for configured in &modules {
5975            if let Err(err) = validate_spec(&configured.module_spec()) {
5976                return Ok(vec![control_error_frame(
5977                    &frame,
5978                    "invalid_daemon_config",
5979                    format!("supervisor rescan rejected daemon config: {err}"),
5980                )?]);
5981            }
5982        }
5983
5984        let configured_capabilities = modules
5985            .iter()
5986            .map(|module| (module.module_id.clone(), module.enabled))
5987            .collect::<Vec<_>>();
5988        let preview_capability_warnings = if preview {
5989            let (_, registrations) = self.runtime_capability_snapshot()?;
5990            let current_modules = self
5991                .supervisor
5992                .list()
5993                .into_iter()
5994                .map(|module| module.module_id().to_string())
5995                .collect::<BTreeSet<_>>();
5996            let resulting_modules = configured_capabilities.clone();
5997            let removed = current_modules
5998                .into_iter()
5999                .filter(|module_id| {
6000                    !resulting_modules
6001                        .iter()
6002                        .any(|(configured_id, _)| configured_id == module_id)
6003                })
6004                .collect::<Vec<_>>();
6005            // A candidate preview also names providers the candidate disables.
6006            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6007            let disabled = match &mode {
6008                RescanMode::Candidate(_) => resulting_modules
6009                    .iter()
6010                    .filter(|(_, enabled)| !enabled)
6011                    .map(|(module_id, _)| module_id.clone())
6012                    .collect::<Vec<_>>(),
6013                _ => Vec::new(),
6014            };
6015            self.capability_evaluator.preview_warnings(
6016                resulting_modules,
6017                &removed,
6018                &disabled,
6019                &registrations,
6020            )
6021        } else {
6022            Vec::new()
6023        };
6024        let reconciled = match &mode {
6025            RescanMode::Candidate(candidate) => self
6026                .preview_candidate(&context, modules, candidate)
6027                .map(|result| (result, Vec::new())),
6028            _ => {
6029                self.reconcile_supervised_modules(
6030                    &context.supervisor,
6031                    modules,
6032                    preview,
6033                    requester.clone(),
6034                )
6035                .await
6036            }
6037        };
6038        let (result, changed) = match reconciled {
6039            Ok(result) => result,
6040            Err(message) => {
6041                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6042            }
6043        };
6044        if !preview {
6045            if let Some(ledger) = &self.activity {
6046                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6047                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6048            }
6049            if let Err(code) = self.configure_activity(activity_config).await {
6050                warn!(
6051                    code,
6052                    "activity retention could not be applied during rescan"
6053                );
6054            }
6055            self.capability_evaluator
6056                .configure(configured_capabilities, reserved_capabilities);
6057            self.capability_evaluator.wake_deadline_loop();
6058            self.refresh_capability_requirements();
6059        }
6060        let mut result = result;
6061        result.restart_required = restart_required;
6062        result.capability_warnings = preview_capability_warnings;
6063        let response = ClientControlResponse::SupervisorRescan { result };
6064        Ok(vec![control_response_body_frame(
6065            &frame,
6066            &response,
6067            "ClientControlResponse::SupervisorRescan",
6068        )?])
6069    }
6070
6071    async fn handle_supervisor_release_reserved(
6072        &self,
6073        frame: Frame,
6074        module_id: String,
6075    ) -> Result<Vec<Frame>, RouterError> {
6076        let Some(context) = self.rescan.clone() else {
6077            return Ok(vec![control_error_frame(
6078                &frame,
6079                "release_unavailable",
6080                "reserved-id release requires a daemon started with a reloadable config path",
6081            )?]);
6082        };
6083        let operation_lock = self.supervisor.operation_lock();
6084        let _operation_guard = operation_lock.lock().await;
6085        let loaded = match crate::daemon_config::load(&context.config_path) {
6086            Ok(Some(config)) => config,
6087            Ok(None) => {
6088                return Ok(vec![control_error_frame(
6089                    &frame,
6090                    "invalid_daemon_config",
6091                    format!(
6092                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6093                        context.config_path.display()
6094                    ),
6095                )?])
6096            }
6097            Err(err) => {
6098                return Ok(vec![control_error_frame(
6099                    &frame,
6100                    "invalid_daemon_config",
6101                    format!("unable to verify reserved-id release against daemon config: {err}"),
6102                )?])
6103            }
6104        };
6105        if loaded
6106            .modules
6107            .iter()
6108            .any(|configured| configured.module_id == module_id)
6109        {
6110            return Ok(vec![control_error_frame(
6111                &frame,
6112                "reserved_module_configured",
6113                format!(
6114                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6115                ),
6116            )?]);
6117        }
6118        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6119            return Ok(vec![control_error_frame(
6120                &frame,
6121                "reserved_gate_not_retained",
6122                format!(
6123                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6124                ),
6125            )?]);
6126        }
6127
6128        let response = ClientControlResponse::SupervisorAck {
6129            module_id,
6130            applied: true,
6131        };
6132        Ok(vec![control_response_body_frame(
6133            &frame,
6134            &response,
6135            "ClientControlResponse::SupervisorAck",
6136        )?])
6137    }
6138
6139    /// Compare the configured module set with the supervised one, changing
6140    /// nothing. The staged spec each supervised module holds is what the
6141    /// configured entry is compared with.
6142    fn plan_rescan(
6143        &self,
6144        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6145    ) -> Result<RescanPlan, String> {
6146        let mut current = BTreeMap::new();
6147        for module in self.supervisor.list() {
6148            let (spec, health) = module.configuration().map_err(|err| {
6149                format!(
6150                    "failed to read configuration for module_id '{}': {err}",
6151                    module.module_id()
6152                )
6153            })?;
6154            let enabled = module
6155                .status()
6156                .map_err(|err| {
6157                    format!(
6158                        "failed to read status for module_id '{}': {err}",
6159                        module.module_id()
6160                    )
6161                })?
6162                .enabled;
6163            current.insert(
6164                module.module_id().to_string(),
6165                (module, spec, health, enabled),
6166            );
6167        }
6168        let configured = configured_modules
6169            .into_iter()
6170            .map(|module| (module.module_id.clone(), module))
6171            .collect::<BTreeMap<_, _>>();
6172
6173        let added = configured
6174            .keys()
6175            .filter(|module_id| !current.contains_key(*module_id))
6176            .cloned()
6177            .collect::<Vec<_>>();
6178        let removed = current
6179            .keys()
6180            .filter(|module_id| !configured.contains_key(*module_id))
6181            .cloned()
6182            .collect::<Vec<_>>();
6183        let mut changed_pending_reload = Vec::new();
6184        let mut configuration_changes = BTreeSet::new();
6185        let mut enabled_changes = BTreeSet::new();
6186        let mut unchanged = 0_u32;
6187
6188        for (module_id, configured_module) in &configured {
6189            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6190            else {
6191                continue;
6192            };
6193            // Compare the whole launch spec so a future launch field cannot
6194            // accidentally become a live-only policy change. Health is stored
6195            // separately and applies live without replacing the process.
6196            let launch_changed = *current_spec != configured_module.module_spec();
6197            let configuration_changed =
6198                launch_changed || *current_health != configured_module.health;
6199            let enabled_changed = *current_enabled != configured_module.enabled;
6200            if configuration_changed {
6201                configuration_changes.insert(module_id.clone());
6202            }
6203            if launch_changed {
6204                changed_pending_reload.push(module_id.clone());
6205            }
6206            if enabled_changed {
6207                enabled_changes.insert(module_id.clone());
6208            }
6209            if !configuration_changed && !enabled_changed {
6210                unchanged = unchanged.saturating_add(1);
6211            }
6212        }
6213        Ok(RescanPlan {
6214            current,
6215            configured,
6216            added,
6217            removed,
6218            changed_pending_reload,
6219            configuration_changes,
6220            enabled_changes,
6221            unchanged,
6222        })
6223    }
6224
6225    /// Reconcile the running module set against the configured one.
6226    ///
6227    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6228    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6229    /// deliberately shares this function with the executing path rather than
6230    /// computing the same diff somewhere else -- two implementations of one
6231    /// decision agree until they do not, and the whole value of a preview is that
6232    /// it describes the operation that will actually run.
6233    async fn reconcile_supervised_modules(
6234        &self,
6235        supervisor: &Supervisor,
6236        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6237        preview: bool,
6238        requester: Principal,
6239    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6240        let plan = self.plan_rescan(configured_modules)?;
6241
6242        // Everything above this point is pure computation over two snapshots.
6243        // Everything below MUTATES. The preview returns here so the boundary is a
6244        // single early return rather than a condition repeated at each mutation
6245        // site, where one missed guard would apply part of a change the caller was
6246        // told would not happen.
6247        if preview {
6248            return Ok((plan.preview_result(), Vec::new()));
6249        }
6250        let changed = plan
6251            .configuration_changes
6252            .union(&plan.enabled_changes)
6253            .cloned()
6254            .collect::<Vec<_>>();
6255        let RescanPlan {
6256            current,
6257            configured,
6258            added,
6259            removed,
6260            changed_pending_reload,
6261            configuration_changes,
6262            enabled_changes,
6263            unchanged,
6264        } = plan;
6265        // The staged generation moves before the first change rather than
6266        // after the last, so a rescan that fails partway still invalidates
6267        // every earlier `supervisor.rescan_checked` preview: part of its diff
6268        // may have run.
6269        if !(added.is_empty()
6270            && removed.is_empty()
6271            && configuration_changes.is_empty()
6272            && enabled_changes.is_empty())
6273        {
6274            if let Some(context) = &self.rescan {
6275                context
6276                    .generation
6277                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6278            }
6279        }
6280
6281        for module_id in &removed {
6282            let module = &current
6283                .get(module_id)
6284                .expect("removed module came from current supervisor state")
6285                .0;
6286            module
6287                .retire_requested(Some(requester.clone()))
6288                .await
6289                .map_err(|err| {
6290                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6291                })?;
6292            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6293            //
6294            // `handle_route_open` resolves an absent module in three steps:
6295            // registry, then supervisor status, then tombstone. Retiring first
6296            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6297            // went with the teardown above, the supervisor entry went with
6298            // `retire`, and the tombstone does not exist yet -- so a route.open
6299            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6300            // it") for a module that was deliberately removed and whose caller
6301            // should get `module_removed` (TERMINAL, carrying a removal age).
6302            //
6303            // Writing the tombstone first closes it: during the window the
6304            // supervisor entry still answers, so the caller gets
6305            // `target_unavailable` -- retryable, and TRUE, because the module
6306            // is mid-teardown. After both statements it is `module_removed`.
6307            // No instant remains where a removed module reads as one that
6308            // never existed.
6309            //
6310            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6311            // the absence of a test beside a fix invites deletion: these are
6312            // two sync statements with no await between them, so reaching the
6313            // window needs a second worker thread to land exactly between them
6314            // and there is no hook to force it. MEASURED: the 25 daemon_config
6315            // tests pass identically with the old order and the new one, so
6316            // the existing suite cannot see this and a green run is not
6317            // evidence either way. What the suite does hold is the
6318            // post-condition -- a removed module answers `module_removed` --
6319            // which this preserves.
6320            //
6321            // Found by an Athena panel reading the shipped tree against a
6322            // design note (2026-09-19), as the one concrete instance of that
6323            // note's class that survived contact with source. Direction is
6324            // benign: retryable where terminal was intended, never the reverse.
6325            self.supervisor.record_rescan_removal(module_id);
6326            self.supervisor.retire(module_id);
6327            self.route_outages.forget(module_id);
6328        }
6329
6330        for module_id in configured.keys() {
6331            let Some((module, _, _, _)) = current.get(module_id) else {
6332                continue;
6333            };
6334            let configured_module = configured
6335                .get(module_id)
6336                .expect("configured module id came from configured map");
6337            if configuration_changes.contains(module_id) {
6338                module
6339                    .update_configuration(
6340                        configured_module.module_spec(),
6341                        configured_module.health.clone(),
6342                        configured_module.drain_timeout_ms,
6343                    )
6344                    .await
6345                    .map_err(|err| {
6346                        format!(
6347                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6348                        )
6349                    })?;
6350            }
6351            if enabled_changes.contains(module_id) {
6352                // A rescan that starts or stops a module applies an operator's
6353                // edit to the config, so the resulting outage was asked for.
6354                self.route_outages.mark_operator_action(module_id);
6355                module
6356                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6357                    .await
6358                    .map_err(|err| {
6359                        self.route_outages.operator_action_ended_unrefused(module_id);
6360                        format!(
6361                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6362                            configured_module.enabled
6363                        )
6364                    })?;
6365            }
6366        }
6367
6368        for module_id in &added {
6369            let configured_module = configured
6370                .get(module_id)
6371                .expect("added module id came from configured map");
6372            supervisor
6373                .supervise_configured_requested(
6374                    configured_module.module_spec(),
6375                    configured_module.enabled,
6376                    configured_module.health.clone(),
6377                    configured_module.drain_timeout_ms,
6378                    configured_module.restart,
6379                    Some(requester.clone()),
6380                )
6381                .map_err(|err| {
6382                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6383                })?;
6384        }
6385
6386        Ok((
6387            SupervisorRescanResult {
6388                added,
6389                removed,
6390                changed_pending_reload,
6391                enabled_changes: enabled_changes.iter().cloned().collect(),
6392                unchanged,
6393                preview: false,
6394                // Filled by the caller, which is the only layer that can see the
6395                // previous config to diff against.
6396                restart_required: Vec::new(),
6397                capability_warnings: Vec::new(),
6398                module_actions: Vec::new(),
6399                earlier_pending: Vec::new(),
6400                expect: None,
6401            },
6402            changed,
6403        ))
6404    }
6405
6406    async fn handle_supervisor_set_enabled(
6407        &self,
6408        frame: Frame,
6409        module_id: String,
6410        enabled: bool,
6411        requester: Principal,
6412    ) -> Result<Vec<Frame>, RouterError> {
6413        let operation_lock = self.supervisor.operation_lock();
6414        let _operation_guard = operation_lock.lock().await;
6415        let Some(module) = self.supervisor.get(&module_id) else {
6416            return Ok(vec![control_error_frame(
6417                &frame,
6418                "unknown_module",
6419                format!("module_id '{module_id}' is not supervised"),
6420            )?]);
6421        };
6422
6423        // Enabling counts as well as disabling: a module an operator starts
6424        // is refused until it registers, and that wait was asked for.
6425        self.route_outages.mark_operator_action(&module_id);
6426        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6427            Ok(applied) => applied,
6428            Err(err) => {
6429                self.route_outages
6430                    .operator_action_ended_unrefused(&module_id);
6431                return Ok(vec![control_error_frame(
6432                    &frame,
6433                    "target_unavailable",
6434                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6435                )?]);
6436            }
6437        };
6438        if !applied {
6439            // Already in the requested state: nothing was made unavailable,
6440            // so the mark must not outlive this request.
6441            self.route_outages
6442                .operator_action_ended_unrefused(&module_id);
6443        }
6444
6445        self.capability_evaluator.wake_deadline_loop();
6446        self.refresh_capability_requirements();
6447        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6448        Ok(vec![control_response_body_frame(
6449            &frame,
6450            &response,
6451            "ClientControlResponse::SupervisorAck",
6452        )?])
6453    }
6454
6455    async fn handle_supervisor_health_probe(
6456        &self,
6457        frame: Frame,
6458        module_id: String,
6459    ) -> Result<Vec<Frame>, RouterError> {
6460        self.refresh_capability_requirements();
6461        let Some(registration) = self
6462            .registry
6463            .get_module(&module_id)
6464            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6465        else {
6466            return Ok(vec![control_error_frame(
6467                &frame,
6468                "unknown_module",
6469                format!("module_id '{module_id}' is not registered"),
6470            )?]);
6471        };
6472
6473        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6474        // named for it. Making `module_registration_grants_op` return false
6475        // unconditionally reddens five tests, and every one is named for something
6476        // else -- capability relay, probe/bind demultiplexing, supervision-only
6477        // probing. They exercise a successful advertisement check on the way to their
6478        // own subject.
6479        //
6480        // Real protection, fragile in a specific way: narrowing any of those tests to
6481        // focus on its stated subject would silently remove coverage nobody knows
6482        // they are carrying. Recorded here rather than as a sixth test, because the
6483        // useful fact is WHICH tests hold the guard up -- a new test would add
6484        // coverage without telling the next person what the existing ones quietly do.
6485        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6486        {
6487            return Ok(vec![control_error_frame(
6488                &frame,
6489                "health_not_advertised",
6490                format!("module_id '{module_id}' did not advertise health.check"),
6491            )?]);
6492        }
6493
6494        let deadline = Instant::now() + self.health_probe_timeout;
6495        let pending = match self.forwarding.begin_module_control_rpc_for(
6496            &module_id,
6497            MODULE_CONTROL_OP_HEALTH_CHECK,
6498            deadline,
6499        ) {
6500            Ok(pending) => pending,
6501            Err(err) => {
6502                return Ok(vec![control_error_frame(
6503                    &frame,
6504                    forwarding_error_code(&err),
6505                    err.to_string(),
6506                )?])
6507            }
6508        };
6509
6510        let PendingModuleControlRpc {
6511            endpoint,
6512            module_sink,
6513            negotiated_ver,
6514            corr: probe_corr,
6515            receiver,
6516        } = pending;
6517        let mut guard =
6518            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6519        let probe_body =
6520            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6521                RouterError::backend(
6522                    0,
6523                    frame.header.corr,
6524                    format!("failed to encode health.check request: {err}"),
6525                )
6526            })?;
6527        let probe_frame = Frame::build_with_version(
6528            negotiated_ver,
6529            FrameType::Request,
6530            control_flags(),
6531            0,
6532            0,
6533            probe_corr,
6534            probe_body,
6535        )
6536        .map_err(RouterError::FrameBuild)?;
6537
6538        // Queue admission and the reply share one deadline, so a module that
6539        // stops reading cannot park the requesting connection's reader forever.
6540        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6541            Ok(Ok(())) => {}
6542            Ok(Err(err)) => {
6543                return Ok(vec![control_error_frame(
6544                    &frame,
6545                    "target_unavailable",
6546                    err.to_string(),
6547                )?]);
6548            }
6549            Err(_) => {
6550                return Ok(vec![control_error_frame(
6551                    &frame,
6552                    "module_timeout",
6553                    format!(
6554                        "module_id '{module_id}' did not answer health.check within {:?}",
6555                        self.health_probe_timeout
6556                    ),
6557                )?]);
6558            }
6559        }
6560
6561        match timeout_at(deadline, receiver).await {
6562            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6563                guard.disarm();
6564                let Some(report) = response.health_report() else {
6565                    return Ok(vec![control_error_frame(
6566                        &frame,
6567                        "invalid_control_body",
6568                        "health.check RPC returned a non-health response",
6569                    )?]);
6570                };
6571                // Metrics go out whole here. The supervisor's cached snapshot
6572                // caps this blob (see truncate_health_metrics), and this path
6573                // exists precisely to answer without that cap -- so applying it
6574                // here would leave no way to see what the cached view drops.
6575                let HealthReport {
6576                    status,
6577                    detail,
6578                    metrics,
6579                } = report;
6580                let capability_detail = self
6581                    .capability_evaluator
6582                    .required_problem_detail(&module_id);
6583                let response = ClientControlResponse::SupervisorHealthProbe {
6584                    module_id,
6585                    status,
6586                    detail: append_capability_problem_detail(detail, capability_detail),
6587                    metrics,
6588                };
6589                Ok(vec![control_response_body_frame(
6590                    &frame,
6591                    &response,
6592                    "ClientControlResponse::SupervisorHealthProbe",
6593                )?])
6594            }
6595            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6596                guard.disarm();
6597                Ok(vec![control_error_body_frame(&frame, body)?])
6598            }
6599            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6600                guard.disarm();
6601                Ok(vec![control_error_frame(
6602                    &frame,
6603                    "target_unavailable",
6604                    message,
6605                )?])
6606            }
6607            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6608                guard.disarm();
6609                Ok(vec![control_error_frame(
6610                    &frame,
6611                    "invalid_control_body",
6612                    message,
6613                )?])
6614            }
6615            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6616                guard.disarm();
6617                Ok(vec![control_error_frame(
6618                    &frame,
6619                    "invalid_control_body",
6620                    format!("expected module-control op '{expected}', got '{actual}'"),
6621                )?])
6622            }
6623            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6624                guard.disarm();
6625                Ok(vec![control_error_frame(
6626                    &frame,
6627                    "module_timeout",
6628                    format!(
6629                        "module_id '{module_id}' answered health.check after {:?}",
6630                        self.health_probe_timeout
6631                    ),
6632                )?])
6633            }
6634            Ok(Err(_)) => Ok(vec![control_error_frame(
6635                &frame,
6636                "target_unavailable",
6637                "health.check waiter was canceled before the module responded",
6638            )?]),
6639            Err(_) => Ok(vec![control_error_frame(
6640                &frame,
6641                "module_timeout",
6642                format!(
6643                    "module_id '{module_id}' did not answer health.check within {:?}",
6644                    self.health_probe_timeout
6645                ),
6646            )?]),
6647        }
6648    }
6649
6650    fn supervisor_status(
6651        &self,
6652        module_id: &str,
6653        corr: u64,
6654    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6655        self.supervisor
6656            .get(module_id)
6657            .map(|module| {
6658                let warming = module.is_warming_for_control("status").map_err(|err| {
6659                    RouterError::backend(
6660                        0,
6661                        corr,
6662                        format!(
6663                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6664                        ),
6665                    )
6666                })?;
6667                module.status_for_control("status").map_err(|err| {
6668                    RouterError::backend(
6669                        0,
6670                        corr,
6671                        format!(
6672                            "failed to read supervisor status for module_id '{module_id}': {err}"
6673                        ),
6674                    )
6675                }).map(|status| (status, warming))
6676            })
6677            .transpose()
6678    }
6679
6680    fn guard_module_control_op(
6681        &self,
6682        frame: &Frame,
6683        module_id: &str,
6684        op: &str,
6685    ) -> Result<Option<Frame>, RouterError> {
6686        if self.module_grants_op(module_id, op, frame.header.corr)? {
6687            return Ok(None);
6688        }
6689
6690        Ok(Some(control_error_frame(
6691            frame,
6692            "op_not_allowed",
6693            format!("module_id '{module_id}' did not grant control op '{op}'"),
6694        )?))
6695    }
6696
6697    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6698        let Some(registration) = self
6699            .registry
6700            .get_module(module_id)
6701            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6702        else {
6703            return Ok(false);
6704        };
6705        Ok(module_registration_grants_op(&registration.control_ops, op))
6706    }
6707
6708    fn handle_status_update(
6709        &self,
6710        endpoint: ModuleEndpointId,
6711        frame: Frame,
6712    ) -> Result<Vec<Frame>, RouterError> {
6713        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6714            Ok(update) => update,
6715            Err(err) => {
6716                // Forward-compat: a newer module may push a channel-0 op this subc
6717                // version doesn't know. The control contract says unknown push ops
6718                // are IGNORED, never answered with an error. Only a malformed body
6719                // for an op we DO know is a real error worth surfacing.
6720                if is_known_module_push_op(&frame.body) {
6721                    return Ok(vec![control_error_frame(
6722                        &frame,
6723                        "invalid_control_body",
6724                        format!("malformed module control push body: {err}"),
6725                    )?]);
6726                }
6727                return Ok(Vec::new());
6728            }
6729        };
6730
6731        match update {
6732            ModuleControlPush::RouteStatus {
6733                route_channel,
6734                route_epoch,
6735                status,
6736            } => {
6737                self.forwarding
6738                    .cache_status(endpoint, route_channel, route_epoch, status)
6739                    .map_err(RouterError::Forwarding)?;
6740            }
6741        }
6742        Ok(Vec::new())
6743    }
6744
6745    fn handle_route_poll(
6746        &self,
6747        ctx: &RouteCtx,
6748        frame: Frame,
6749        route_channel: u16,
6750        route_epoch: u32,
6751        kind: PollKind,
6752    ) -> Result<Vec<Frame>, RouterError> {
6753        let snapshot = self
6754            .forwarding
6755            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6756            .map_err(RouterError::Forwarding)?;
6757        let response = match (kind, snapshot) {
6758            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6759                ClientControlResponse::RoutePoll {
6760                    route_channel,
6761                    route_epoch,
6762                    status,
6763                    live: None,
6764                }
6765            }
6766            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6767                route_channel,
6768                route_epoch,
6769                status: None,
6770                live: None,
6771            },
6772            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6773                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6774                // is what makes reporting `true` correct rather than a
6775                // confident guess. `process_live` returns None only when the
6776                // module id has no supervisor snapshot at all -- an
6777                // externally-started module the daemon did not spawn -- and
6778                // for those the supervisor has no opinion to offer, ever. It
6779                // is never None for a supervised module in an unknown state:
6780                // a supervised module always has a snapshot, and the answer
6781                // comes from `state == Running && process_alive`.
6782                //
6783                // The route is Bound, so the module completed a HELLO on a
6784                // live connection; "the process this route points at is
6785                // running" is therefore attested by the binding rather than
6786                // assumed. Reporting `false` for an unsupervised module would
6787                // be the actual lie -- it would tell a client its healthy
6788                // route is dead because the daemon does not manage the
6789                // process.
6790                //
6791                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6792                // module whose liveness is genuinely unknown, e.g. a snapshot
6793                // that has not been populated yet -- THIS DEFAULT BECOMES
6794                // WRONG and must split: unsupervised stays true, unknown
6795                // becomes null so the client can tell the two apart. The
6796                // response field is already `Option<bool>`, so the wire can
6797                // carry that distinction today.
6798                let live = self
6799                    .process_liveness
6800                    .as_ref()
6801                    .and_then(|source| source.process_live(&module_id))
6802                    .unwrap_or(true);
6803                ClientControlResponse::RoutePoll {
6804                    route_channel,
6805                    route_epoch,
6806                    status: None,
6807                    live: Some(live),
6808                }
6809            }
6810            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6811                route_channel,
6812                route_epoch,
6813                status: None,
6814                live: Some(false),
6815            },
6816        };
6817
6818        Ok(vec![control_response_body_frame(
6819            &frame,
6820            &response,
6821            "ClientControlResponse::RoutePoll",
6822        )?])
6823    }
6824
6825    pub(crate) fn observe_module_control_completion(
6826        &self,
6827        completion: ModuleControlRpcCompletion,
6828    ) -> bool {
6829        match completion {
6830            ModuleControlRpcCompletion::Unknown => false,
6831            ModuleControlRpcCompletion::Settled => true,
6832            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6833                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6834                info!(
6835                    module_id = %module_id,
6836                    latency_ms,
6837                    "late health.check answer proves the module is alive"
6838                );
6839                match self
6840                    .supervisor
6841                    .record_late_health_answer(&module_id, latency_ms)
6842                {
6843                    Ok(true) => {}
6844                    Ok(false) => debug!(
6845                        module_id = %module_id,
6846                        latency_ms,
6847                        "late health.check answer has no active supervisor snapshot"
6848                    ),
6849                    Err(err) => warn!(
6850                        module_id = %module_id,
6851                        latency_ms,
6852                        error = %err,
6853                        "failed to record late health.check answer"
6854                    ),
6855                }
6856                true
6857            }
6858        }
6859    }
6860
6861    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6862    /// the module connection whose frame is being handled, or to the client that
6863    /// relay was opened for.
6864    ///
6865    /// This runs on the MODULE connection's frame handler, where returning `Err`
6866    /// ends that connection -- and a module connection carries every client's
6867    /// routes to that module, so ending it costs the whole fleet its tools.
6868    /// `ConnectionClosing` carries the id of the connection that is closing, and
6869    /// when that id is a CLIENT's, the condition is entirely about that one
6870    /// client's route.open. A client-scoped condition has no authority over a
6871    /// shared module connection, so it is logged and the single relay is dropped:
6872    /// the client is going away, and `complete_pending_relay` already removed the
6873    /// relay before failing, so there is nothing left to settle. Anything that
6874    /// relay still reserved is released by that client's own connection teardown,
6875    /// which is already under way -- that is what "closing" means.
6876    ///
6877    /// Every other failure is a statement about THIS connection and stays fatal:
6878    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6879    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6880    /// frames correctly.
6881    fn refuse_to_end_module_connection_for_a_client(
6882        &self,
6883        module_connection_id: ConnectionId,
6884        corr: u64,
6885        err: ForwardingError,
6886    ) -> Result<(), RouterError> {
6887        if let ForwardingError::ConnectionClosing { connection_id } = err {
6888            if connection_id != module_connection_id {
6889                warn!(
6890                    module_connection_id = module_connection_id.get(),
6891                    client_connection_id = connection_id.get(),
6892                    corr,
6893                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6894                );
6895                return Ok(());
6896            }
6897        }
6898        Err(RouterError::Forwarding(err))
6899    }
6900
6901    fn handle_module_relay_response(
6902        &self,
6903        connection_id: ConnectionId,
6904        frame: Frame,
6905    ) -> Result<Vec<Frame>, RouterError> {
6906        let mut secondary_error = None;
6907        let outcome = match frame.header.ty {
6908            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6909                Ok(probe) if probe.op == "route.bind" => {
6910                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6911                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6912                            RouteBindRelayOutcome::Accepted
6913                        }
6914                        Ok(other) => {
6915                            let message =
6916                                format!("route.bind response carried unexpected body: {other:?}");
6917                            secondary_error = Some(control_error_frame(
6918                                &frame,
6919                                "invalid_control_body",
6920                                message.clone(),
6921                            )?);
6922                            RouteBindRelayOutcome::ModuleGone(message)
6923                        }
6924                        Err(err) => {
6925                            let message = format!("malformed route.bind response body: {err}");
6926                            secondary_error = Some(control_error_frame(
6927                                &frame,
6928                                "invalid_control_body",
6929                                message.clone(),
6930                            )?);
6931                            RouteBindRelayOutcome::ModuleGone(message)
6932                        }
6933                    }
6934                }
6935                Ok(probe) => {
6936                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6937                    {
6938                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6939                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6940                            "malformed {} response body: {err}",
6941                            probe.op
6942                        )),
6943                    };
6944                    let completion = self
6945                        .forwarding
6946                        .complete_module_control_rpc(
6947                            connection_id,
6948                            frame.header.corr,
6949                            Some(&probe.op),
6950                            outcome,
6951                        )
6952                        .map_err(RouterError::Forwarding)?;
6953                    if !self.observe_module_control_completion(completion) {
6954                        debug!(
6955                            connection_id = connection_id.get(),
6956                            corr = frame.header.corr,
6957                            op = %probe.op,
6958                            "dropping late or unknown module-control RPC response"
6959                        );
6960                    }
6961                    return Ok(Vec::new());
6962                }
6963                Err(err) => {
6964                    if let Some(expected_op) = self
6965                        .forwarding
6966                        .pending_module_control_op(connection_id, frame.header.corr)
6967                        .map_err(RouterError::Forwarding)?
6968                    {
6969                        let completion = self
6970                            .forwarding
6971                            .complete_module_control_rpc(
6972                                connection_id,
6973                                frame.header.corr,
6974                                None,
6975                                ModuleControlRpcOutcome::MalformedResponse(format!(
6976                                    "malformed {expected_op} response body: {err}"
6977                                )),
6978                            )
6979                            .map_err(RouterError::Forwarding)?;
6980                        if !self.observe_module_control_completion(completion) {
6981                            debug!(
6982                                connection_id = connection_id.get(),
6983                                corr = frame.header.corr,
6984                                "dropping late malformed module-control RPC response"
6985                            );
6986                        }
6987                        return Ok(Vec::new());
6988                    }
6989                    let message = format!("malformed route.bind response body: {err}");
6990                    secondary_error = Some(control_error_frame(
6991                        &frame,
6992                        "invalid_control_body",
6993                        message.clone(),
6994                    )?);
6995                    RouteBindRelayOutcome::ModuleGone(message)
6996                }
6997            },
6998            FrameType::Error => {
6999                if self
7000                    .forwarding
7001                    .pending_module_control_op(connection_id, frame.header.corr)
7002                    .map_err(RouterError::Forwarding)?
7003                    .is_some()
7004                {
7005                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7006                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7007                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7008                            "malformed module-control ERROR body: {err}"
7009                        )),
7010                    };
7011                    let completion = self
7012                        .forwarding
7013                        .complete_module_control_rpc(
7014                            connection_id,
7015                            frame.header.corr,
7016                            None,
7017                            outcome,
7018                        )
7019                        .map_err(RouterError::Forwarding)?;
7020                    if !self.observe_module_control_completion(completion) {
7021                        debug!(
7022                            connection_id = connection_id.get(),
7023                            corr = frame.header.corr,
7024                            "dropping late or unknown module-control RPC error"
7025                        );
7026                    }
7027                    return Ok(Vec::new());
7028                }
7029                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7030                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7031                    Err(err) => {
7032                        let message = format!("malformed route.bind ERROR body: {err}");
7033                        secondary_error = Some(control_error_frame(
7034                            &frame,
7035                            "invalid_control_body",
7036                            message.clone(),
7037                        )?);
7038                        RouteBindRelayOutcome::ModuleGone(message)
7039                    }
7040                }
7041            }
7042            ty => {
7043                return Ok(vec![control_error_frame(
7044                    &frame,
7045                    "unsupported_control_frame",
7046                    format!("unsupported module channel-0 frame {ty:?}"),
7047                )?])
7048            }
7049        };
7050
7051        let settled =
7052            self.forwarding
7053                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7054        let completion = match settled {
7055            Ok(completion) => completion,
7056            Err(err) => {
7057                self.refuse_to_end_module_connection_for_a_client(
7058                    connection_id,
7059                    frame.header.corr,
7060                    err,
7061                )?;
7062                return Ok(secondary_error.into_iter().collect());
7063            }
7064        };
7065        if let Some(target) = completion.abandoned.as_ref() {
7066            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7067        }
7068        if !completion.settled {
7069            debug!(
7070                connection_id = connection_id.get(),
7071                corr = frame.header.corr,
7072                frame_type = ?frame.header.ty,
7073                "dropping late or unknown route.bind relay response"
7074            );
7075        }
7076        Ok(secondary_error.into_iter().collect())
7077    }
7078
7079    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7080        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7081        // GOODBYE ends the connection's logical session even when its socket
7082        // stays open. Use disconnect teardown so verdicts, client notices and
7083        // scope authority are released at the same lifecycle boundary.
7084        self.cleanup_connection_with_end_reason(
7085            connection_id,
7086            RegistrationEndReason::ExplicitGoodbye,
7087        )
7088        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7089        Ok(Vec::new())
7090    }
7091}
7092
7093impl Default for ControlHandler {
7094    fn default() -> Self {
7095        Self::new(Arc::new(Registry::default()))
7096    }
7097}
7098
7099impl crate::supervise::SwapPromotionObserver for ControlHandler {
7100    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7101        self.apply_registration_capabilities(registration);
7102    }
7103}
7104
7105fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7106    CapabilityRequirementStatus {
7107        consumer: status.consumer,
7108        capability: status.capability,
7109        need: match status.need {
7110            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7111            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7112        },
7113        verdict: status.verdict.as_str().to_string(),
7114        episode_seq: status.episode_seq,
7115        config_satisfiable: status.config_satisfiable,
7116        runtime_available: status.runtime_available,
7117        detail: status.detail,
7118    }
7119}
7120
7121fn append_capability_problem_detail(
7122    detail: Option<String>,
7123    capability_detail: Option<String>,
7124) -> Option<String> {
7125    match (detail, capability_detail) {
7126        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7127        (Some(detail), None) => Some(detail),
7128        (None, Some(capability_detail)) => Some(capability_detail),
7129        (None, None) => None,
7130    }
7131}
7132
7133fn subc_ops() -> Vec<String> {
7134    SUBC_CONTROL_OPS
7135        .iter()
7136        .map(|op| (*op).to_string())
7137        .collect()
7138}
7139
7140fn module_subc_ops() -> Vec<String> {
7141    SUBC_CONTROL_OPS
7142        .iter()
7143        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7144        .map(|op| (*op).to_string())
7145        .collect()
7146}
7147
7148#[cfg(test)]
7149fn module_baseline_control_ops() -> Vec<String> {
7150    MODULE_BASELINE_CONTROL_OPS
7151        .iter()
7152        .map(|op| (*op).to_string())
7153        .collect()
7154}
7155
7156fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7157    let mut seen = HashSet::new();
7158    let mut effective = Vec::new();
7159    for op in MODULE_BASELINE_CONTROL_OPS {
7160        if seen.insert((*op).to_string()) {
7161            effective.push((*op).to_string());
7162        }
7163    }
7164    for op in declared.unwrap_or_default() {
7165        if seen.insert(op.clone()) {
7166            effective.push(op);
7167        }
7168    }
7169    effective
7170}
7171
7172fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7173    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7174}
7175
7176fn target_module_id(target: &RouteTarget) -> &str {
7177    match target {
7178        RouteTarget::ToolProvider { module_id }
7179        | RouteTarget::ManagementSurface { module_id }
7180        | RouteTarget::InternalService { module_id, .. } => module_id,
7181    }
7182}
7183
7184fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7185    roles.iter().any(|role| match (target, role) {
7186        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7187        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7188        (
7189            RouteTarget::InternalService { service_id, .. },
7190            ProviderRole::InternalService {
7191                service_id: provided,
7192                ..
7193            },
7194        ) => service_id == provided,
7195        _ => false,
7196    })
7197}
7198
7199fn is_routable_role(role: &ProviderRole) -> bool {
7200    matches!(
7201        role,
7202        ProviderRole::ToolProvider { .. }
7203            | ProviderRole::ManagementSurface { .. }
7204            | ProviderRole::InternalService { .. }
7205    )
7206}
7207
7208#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7209enum ControlRequestBodyError {
7210    UnknownOp,
7211    InvalidBody,
7212}
7213
7214#[derive(Debug, Deserialize)]
7215struct ControlOpProbe {
7216    op: String,
7217}
7218
7219/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7220/// this set is treated as a forward-compat unknown and ignored rather than errored.
7221const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7222
7223fn is_known_module_push_op(body: &[u8]) -> bool {
7224    serde_json::from_slice::<ControlOpProbe>(body)
7225        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7226        .unwrap_or(false)
7227}
7228
7229fn is_known_module_request_op(body: &[u8]) -> bool {
7230    serde_json::from_slice::<ControlOpProbe>(body)
7231        .map(|probe| is_module_to_subc_op(&probe.op))
7232        .unwrap_or(false)
7233}
7234
7235fn is_module_to_subc_op(op: &str) -> bool {
7236    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7237}
7238
7239fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7240    debug!(
7241        op = %op,
7242        connection_id = connection_id.get(),
7243        corr,
7244        "control dispatch"
7245    );
7246}
7247
7248fn log_slow_control_dispatch(
7249    dispatch_started_at: Option<StdInstant>,
7250    op: &'static str,
7251    connection_id: ConnectionId,
7252    corr: u64,
7253    route_open_timings: Option<&RouteOpenTimings>,
7254) {
7255    let Some(dispatch_started_at) = dispatch_started_at else {
7256        return;
7257    };
7258    // Use the same monotonic clock as the phase timers and relay deadline.
7259    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7260    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7261        if let Some(timings) = route_open_timings {
7262            warn!(
7263                op = %op,
7264                connection_id = connection_id.get(),
7265                corr,
7266                elapsed_ms = elapsed.as_millis() as u64,
7267                module_id = ?timings.module_id,
7268                principal = %timings.principal,
7269                admission_ms = timings.admission.as_millis() as u64,
7270                project_root_ms = timings.project_root.as_millis() as u64,
7271                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7272                bind_send_ms = timings.bind_send.as_millis() as u64,
7273                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7274                "slow control dispatch"
7275            );
7276            return;
7277        }
7278        warn!(
7279            op = %op,
7280            connection_id = connection_id.get(),
7281            corr,
7282            elapsed_ms = elapsed.as_millis() as u64,
7283            "slow control dispatch"
7284        );
7285    }
7286}
7287
7288/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7289#[cfg(test)]
7290type TestCallerLookup =
7291    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7292
7293/// Every new client operation needs an explicit privilege decision here.
7294fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7295    match request {
7296        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7297            Some(PrivilegedOperation::Restart { module_id })
7298        }
7299        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7300            Some(PrivilegedOperation::Swap { module_id })
7301        }
7302        ClientControlRequest::SupervisorReload { module_id } => {
7303            Some(PrivilegedOperation::Reload { module_id })
7304        }
7305        ClientControlRequest::SupervisorRescan { preview } => {
7306            if *preview {
7307                None
7308            } else {
7309                Some(PrivilegedOperation::Rescan)
7310            }
7311        }
7312        // Only the well-formed apply asks for approval. A preview changes
7313        // nothing, and any other combination is refused before it touches
7314        // anything, so prompting the operator for it would ask about a request
7315        // that cannot run.
7316        ClientControlRequest::SupervisorRescanChecked {
7317            preview,
7318            candidate,
7319            expect,
7320        } => (!*preview && candidate.is_none() && expect.is_some())
7321            .then_some(PrivilegedOperation::Rescan),
7322        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7323            Some(PrivilegedOperation::SetEnabled {
7324                module_id,
7325                enabled: *enabled,
7326            })
7327        }
7328        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7329            Some(PrivilegedOperation::ReleaseReserved { module_id })
7330        }
7331        ClientControlRequest::ServerDescribe {}
7332        | ClientControlRequest::CatalogList { .. }
7333        | ClientControlRequest::RouteOpen { .. }
7334        | ClientControlRequest::RoutePoll { .. }
7335        | ClientControlRequest::SupervisorList {}
7336        | ClientControlRequest::SupervisorHealth {}
7337        | ClientControlRequest::SupervisorHealthProbe { .. }
7338        | ClientControlRequest::SupervisorRoutes { .. }
7339        | ClientControlRequest::SupervisorProvenance { .. }
7340        | ClientControlRequest::SupervisorStderrTail { .. }
7341        | ClientControlRequest::SupervisorTerminals { .. }
7342        | ClientControlRequest::SupervisorSpawnSnapshot {}
7343        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7344    }
7345}
7346
7347fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7348    match request {
7349        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7350        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7351        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7352        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7353        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7354        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7355        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7356        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7357        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7358        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7359        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7360        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7361        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7362        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7363        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7364        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7365        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7366        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7367        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7368        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7369    }
7370}
7371
7372/// The caller of one `route.open`, as its `route.open refused` line names it.
7373///
7374/// A connection id alone cannot name a caller after the fact: ids restart at
7375/// every daemon restart, so the same number belongs to different modules
7376/// before and after one. These three fields are what the refused line needs
7377/// to be attributable on its own, and they are rendered with the same helpers
7378/// as the `route.open accepted` line so both directions grep alike.
7379struct RouteOpenCaller {
7380    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
7381    /// when the request presented a consumer identity whose launch nonce did
7382    /// not verify. The claimed module id is never printed as a principal.
7383    principal: String,
7384    /// `none` for an unscoped open; the admitted stamp's label once scope
7385    /// admission issued one; before that, the requested selector's label.
7386    scope: String,
7387    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
7388    /// because it is a string the caller chose.
7389    harness: String,
7390}
7391
7392impl RouteOpenCaller {
7393    fn new(
7394        principal: Option<&Principal>,
7395        scope: Option<&ScopeSelector>,
7396        identity: &BindIdentity,
7397    ) -> Self {
7398        Self {
7399            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7400            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7401            harness: bounded_log_text(&identity.harness),
7402        }
7403    }
7404
7405    /// For a refusal that has no parsed request at hand: names whatever the
7406    /// frame still says, and `unknown` where it says nothing parseable.
7407    fn unknown() -> Self {
7408        Self {
7409            principal: "unknown".to_string(),
7410            scope: "unknown".to_string(),
7411            harness: "unknown".to_string(),
7412        }
7413    }
7414}
7415
7416/// Longest client-chosen text (a scope ref, a harness) a route.open log line
7417/// prints before cutting it.
7418const MAX_LOGGED_TEXT_CHARS: usize = 64;
7419
7420/// `value` Debug-quoted, so control characters land escaped and cannot forge
7421/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
7422/// `…` inside the quotes so one request cannot flood the line.
7423fn bounded_log_text(value: &str) -> String {
7424    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7425    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7426        bounded.push('…');
7427    }
7428    format!("{bounded:?}")
7429}
7430
7431/// The `scope=` field of the `route.open accepted` line, from the stamp that
7432/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7433/// so a ref holding control characters cannot forge a log line, and cut to 64
7434/// characters with a trailing `…` to bound the line. The bind proof and the
7435/// scope's attributes never appear.
7436fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7437    use subc_protocol::scope::ScopeKind;
7438    let kind = match stamp.kind {
7439        ScopeKind::Head => "head",
7440        ScopeKind::Worker => "worker",
7441        ScopeKind::Ephemeral => "ephemeral",
7442    };
7443    format!(
7444        "{kind}/{}/{}@{}",
7445        principal_label(&stamp.owner),
7446        bounded_log_text(&stamp.scope_ref),
7447        stamp.scope_epoch
7448    )
7449}
7450
7451/// The `scope=` field of a `route.open refused` line for an open refused
7452/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
7453/// The kind is not known yet, so the literal `requested` stands in its place.
7454/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
7455/// owner it has not been checked against the configured modules: a quoted
7456/// owner means the value was never validated. The bind proof never appears.
7457fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7458    let epoch = selector
7459        .scope_epoch
7460        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7461    format!(
7462        "requested/{}/{}@{epoch}",
7463        bounded_log_text(&principal_label(&selector.owner)),
7464        bounded_log_text(&selector.scope_ref),
7465    )
7466}
7467
7468fn principal_label(principal: &Principal) -> String {
7469    match principal {
7470        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7471        Principal::Direct => "direct".to_string(),
7472        other => format!("{other:?}"),
7473    }
7474}
7475
7476fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7477    match request {
7478        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7479        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7480        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7481        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7482        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7483    }
7484}
7485
7486fn parse_client_control_request(
7487    body: &[u8],
7488) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7489    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7490        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7491            Ok(probe)
7492                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7493                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7494            {
7495                ControlRequestBodyError::InvalidBody
7496            }
7497            Ok(_) => ControlRequestBodyError::UnknownOp,
7498            Err(_) => ControlRequestBodyError::InvalidBody,
7499        };
7500        (err, classification)
7501    })
7502}
7503
7504fn parse_module_control_request_from_module(
7505    body: &[u8],
7506) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7507    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7508        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7509            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7510            Ok(_) => ControlRequestBodyError::UnknownOp,
7511            Err(_) => ControlRequestBodyError::InvalidBody,
7512        };
7513        (err, classification)
7514    })
7515}
7516
7517#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7518enum ProviderRoleKind {
7519    ToolProvider,
7520    PipelineStage,
7521    ManagementSurface,
7522    InternalService,
7523}
7524
7525fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7526    match role {
7527        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7528        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7529        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7530        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7531    }
7532}
7533
7534fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7535    roles.iter().map(provider_role_kind).collect()
7536}
7537
7538/// Most refused scope records named individually in the log per sync; the
7539/// `refused` count on the accepted line is always complete.
7540const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7541
7542/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7543#[derive(Debug, Default, PartialEq, Eq)]
7544struct ScopeOutcomeCounts {
7545    created: usize,
7546    replaced: usize,
7547    updated: usize,
7548    unchanged: usize,
7549    refused: usize,
7550}
7551
7552impl ScopeOutcomeCounts {
7553    fn of(results: &[ScopeRecordResult]) -> Self {
7554        let mut counts = Self::default();
7555        for result in results {
7556            let slot = match result.outcome {
7557                ScopeRecordOutcome::Created => &mut counts.created,
7558                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7559                ScopeRecordOutcome::Updated => &mut counts.updated,
7560                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7561                ScopeRecordOutcome::Refused => &mut counts.refused,
7562            };
7563            *slot += 1;
7564        }
7565        counts
7566    }
7567}
7568
7569#[cfg(test)]
7570mod scope_outcome_count_tests {
7571    use super::*;
7572
7573    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7574        ScopeRecordResult {
7575            scope_ref: "r".to_string(),
7576            scope_epoch: 1,
7577            outcome,
7578            code: None,
7579            message: None,
7580            version: None,
7581            parent_state: None,
7582        }
7583    }
7584
7585    /// Each outcome lands in its own count, so a refused record can never be
7586    /// hidden inside the total the log already printed.
7587    #[test]
7588    fn every_outcome_is_counted_in_its_own_field() {
7589        let results = [
7590            result(ScopeRecordOutcome::Created),
7591            result(ScopeRecordOutcome::Created),
7592            result(ScopeRecordOutcome::Replaced),
7593            result(ScopeRecordOutcome::Updated),
7594            result(ScopeRecordOutcome::Unchanged),
7595            result(ScopeRecordOutcome::Refused),
7596            result(ScopeRecordOutcome::Refused),
7597            result(ScopeRecordOutcome::Refused),
7598        ];
7599        assert_eq!(
7600            ScopeOutcomeCounts::of(&results),
7601            ScopeOutcomeCounts {
7602                created: 2,
7603                replaced: 1,
7604                updated: 1,
7605                unchanged: 1,
7606                refused: 3,
7607            }
7608        );
7609    }
7610}
7611
7612/// Return whether a catalog change can create a newly violating live route.
7613/// Removing an attested claim is intentionally excluded: it makes fewer routes
7614/// forbidden and therefore must leave the existing route census untouched.
7615fn capability_census_trigger(
7616    old: Option<&CapabilityDeclarations>,
7617    new: Option<&CapabilityDeclarations>,
7618) -> bool {
7619    let old_provides = old
7620        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7621        .unwrap_or_default();
7622    let old_denies = old
7623        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7624        .unwrap_or_default();
7625    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7626        provides: Vec::new(),
7627        requires: Vec::new(),
7628        must_never_reach: Vec::new(),
7629    });
7630
7631    new.provides
7632        .iter()
7633        .any(|capability| !old_provides.contains(capability))
7634        || new
7635            .must_never_reach
7636            .iter()
7637            .any(|capability| !old_denies.contains(capability))
7638}
7639
7640/// Find the first capability an attested opener denies that an attested target
7641/// claims. Both manifests are live registry records, never cached or client data.
7642fn denied_capability<'a>(
7643    opening_manifest: &'a ModuleManifest,
7644    target_manifest: &ModuleManifest,
7645) -> Option<&'a str> {
7646    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7647    let target_capabilities = target_manifest.capabilities.as_ref()?;
7648    opening_capabilities
7649        .must_never_reach
7650        .iter()
7651        .find(|denied| {
7652            target_capabilities
7653                .provides
7654                .iter()
7655                .any(|provided| provided == *denied)
7656        })
7657        .map(String::as_str)
7658}
7659
7660fn catalog_update_frozen_field_message(
7661    registered: &ModuleManifest,
7662    provides: &[ProviderRole],
7663) -> Option<String> {
7664    let old_has_provides = !registered.provides.is_empty();
7665    let new_has_provides = !provides.is_empty();
7666    if old_has_provides != new_has_provides {
7667        return Some(format!(
7668            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7669            registered.module_id
7670        ));
7671    }
7672
7673    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7674        return Some(format!(
7675            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7676            registered.module_id
7677        ));
7678    }
7679
7680    let registered_concurrency = manifest_concurrency(registered);
7681    let mut candidate = registered.clone();
7682    candidate.provides = provides.to_vec();
7683    let candidate_concurrency = manifest_concurrency(&candidate);
7684    if candidate_concurrency != registered_concurrency {
7685        return Some(format!(
7686            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7687            registered.module_id, registered_concurrency, candidate_concurrency
7688        ));
7689    }
7690
7691    // control_ops live beside the manifest in the HELLO body, not inside
7692    // ModuleManifest, so a provides-only catalog.update cannot change them.
7693    None
7694}
7695
7696fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7697    manifest.provides.iter().any(is_routable_role)
7698}
7699
7700/// Returns the routable-provider concurrency subc should enforce for this manifest.
7701///
7702/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7703/// InternalService has no role-specific concurrency field, so it retains the
7704/// existing ModuleManaged default for backward compatibility.
7705fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7706    manifest
7707        .provides
7708        .iter()
7709        .find_map(|provider| match provider {
7710            ProviderRole::ToolProvider { concurrency, .. }
7711            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7712            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7713        })
7714        .unwrap_or(Concurrency::ModuleManaged)
7715}
7716
7717/// True when the manifest carries a ManagementSurface role whose concurrency
7718/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7719/// bytes because the typed manifest deliberately erases that distinction: the
7720/// default exists for wire compatibility, and this probe exists so the default
7721/// stays observable. Any parse irregularity returns false -- the caller only
7722/// logs, and a malformed body already failed registration upstream.
7723fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7724    let has_management_surface = manifest
7725        .provides
7726        .iter()
7727        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7728    if !has_management_surface {
7729        return false;
7730    }
7731    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7732        return false;
7733    };
7734    let Some(provides) = raw
7735        .get("manifest")
7736        .and_then(|manifest| manifest.get("provides"))
7737        .and_then(serde_json::Value::as_array)
7738    else {
7739        return false;
7740    };
7741    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7742    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7743    // against the management_surface_manifest_without_concurrency golden, not
7744    // recalled (the externally-tagged guess was this function's first bug).
7745    provides.iter().any(|role| {
7746        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7747            && role.get("concurrency").is_none()
7748    })
7749}
7750
7751fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7752    if peer_version != PROTOCOL_VERSION {
7753        return Err(format!(
7754            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7755        ));
7756    }
7757    Ok(PROTOCOL_VERSION)
7758}
7759
7760fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7761    Frame::build_with_version(
7762        response_version(frame),
7763        FrameType::Pong,
7764        frame.header.flags,
7765        0,
7766        0,
7767        frame.header.corr,
7768        Vec::new(),
7769    )
7770    .map_err(RouterError::FrameBuild)
7771}
7772
7773pub(crate) fn control_error_frame(
7774    frame: &Frame,
7775    code: &'static str,
7776    message: impl Into<String>,
7777) -> Result<Frame, RouterError> {
7778    control_error_body_frame(
7779        frame,
7780        ErrorBody {
7781            code: code.to_string(),
7782            message: message.into(),
7783            detail: None,
7784        },
7785    )
7786}
7787
7788fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7789    let body = serde_json::to_vec(&error).map_err(|err| {
7790        RouterError::backend(
7791            0,
7792            frame.header.corr,
7793            format!("failed to encode control ERROR: {err}"),
7794        )
7795    })?;
7796
7797    Frame::build_with_version(
7798        response_version(frame),
7799        FrameType::Error,
7800        control_flags(),
7801        0,
7802        0,
7803        frame.header.corr,
7804        body,
7805    )
7806    .map_err(RouterError::FrameBuild)
7807}
7808
7809fn control_response_body_frame<T: Serialize>(
7810    frame: &Frame,
7811    reply: &T,
7812    label: &'static str,
7813) -> Result<Frame, RouterError> {
7814    let body = serde_json::to_vec(reply).map_err(|err| {
7815        RouterError::backend(
7816            0,
7817            frame.header.corr,
7818            format!("failed to encode {label}: {err}"),
7819        )
7820    })?;
7821
7822    Frame::build_with_version(
7823        response_version(frame),
7824        FrameType::Response,
7825        control_flags(),
7826        0,
7827        0,
7828        frame.header.corr,
7829        body,
7830    )
7831    .map_err(RouterError::FrameBuild)
7832}
7833
7834/// Map a forwarding failure to the wire code a client sees.
7835///
7836/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7837/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7838/// chosen here decides whether a caller retries or gives up.
7839///
7840/// That makes attribution the load-bearing property, not merely having a code. A
7841/// permanent fault published as a retryable one produces a fleet-wide retry storm
7842/// against something that can never recover; a transient fault published as
7843/// permanent gives up on work that would have succeeded. Both look correct in a
7844/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7845/// the mapping per variant rather than merely asserting that some code exists.
7846///
7847/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7848/// two codes on the same side of the boundary passes it. Measured rather than
7849/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7850/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7851/// a test named for something else that happens to assert the string.
7852///
7853/// That accidental coverage is deliberately left alone rather than promoted to a
7854/// named test, because it guards a property this function does not promise.
7855/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7856/// specific code within a class, so identity is free to change and only the
7857/// partition is a contract. Splitting it out would assert a guarantee nothing
7858/// depends on — and a suite that promises more than the code does is the harder
7859/// thing to correct later, because the next reader cannot tell which assertions
7860/// are load-bearing.
7861///
7862/// Pin identity here the moment a consumer branches on a specific code.
7863fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7864    match err {
7865        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7866        ForwardingError::NoModuleConnection => "target_unavailable",
7867        ForwardingError::ModuleReloading { .. } => "module_reloading",
7868        ForwardingError::ClientRouteChannelExhausted { .. }
7869        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7870        ForwardingError::StaleModuleEndpoint
7871        | ForwardingError::UnknownReservation { .. }
7872        | ForwardingError::ConnectionClosing { .. }
7873        | ForwardingError::ClientEgressClosed { .. }
7874        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7875        // Only a swap candidate's registration can produce this, and it means
7876        // exactly what a second active HELLO for a live id means.
7877        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7878        ForwardingError::RelayCorrelationExhausted
7879        | ForwardingError::RouteOpenBuild(_)
7880        | ForwardingError::Poisoned => "forwarding_error",
7881    }
7882}
7883
7884fn response_version(frame: &Frame) -> u8 {
7885    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7886        frame.header.ver
7887    } else {
7888        PROTOCOL_VERSION
7889    }
7890}
7891
7892fn control_flags() -> Flags {
7893    Flags::new(false, Priority::Passive, false)
7894}
7895
7896/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7897/// channel. The target is the module (a client never saw the route), so this
7898/// takes the module path: delivered late rather than dropped when the module's
7899/// queue is momentarily full, and never closing its connection.
7900fn send_goodbye_target_best_effort(
7901    counters: &DaemonCounters,
7902    target: &GoodbyeTarget,
7903    context: &'static str,
7904) {
7905    let Ok(frame) = Frame::build_with_version(
7906        target.negotiated_ver,
7907        FrameType::Goodbye,
7908        control_flags(),
7909        target.channel,
7910        target.epoch,
7911        0,
7912        Vec::new(),
7913    ) else {
7914        return;
7915    };
7916    crate::forwarding::send_module_route_goodbye(
7917        counters,
7918        &target.sink,
7919        frame,
7920        target.module_id.as_deref(),
7921        context,
7922    );
7923}
7924
7925pub(crate) fn send_route_control_pushes(
7926    forwarding: &ForwardingTable,
7927    routes: Vec<EndpointRoute>,
7928    push: ClientControlPush,
7929) {
7930    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7931    for route in routes {
7932        let target = route.goodbye_target;
7933        if let Some((existing, channels)) = targets
7934            .iter_mut()
7935            .find(|(existing, _)| existing.connection_id == target.connection_id)
7936        {
7937            debug_assert_eq!(
7938                existing.negotiated_ver, target.negotiated_ver,
7939                "one connection cannot negotiate multiple frame versions"
7940            );
7941            if !channels.contains(&target.channel) {
7942                channels.push(target.channel);
7943            }
7944            continue;
7945        }
7946        let channel = target.channel;
7947        targets.push((target, vec![channel]));
7948    }
7949    for (target, mut channels) in targets {
7950        channels.sort_unstable();
7951        let mut push = push.clone();
7952        match &mut push {
7953            ClientControlPush::RouteClosing {
7954                channels: covered, ..
7955            }
7956            | ClientControlPush::RouteClosed {
7957                channels: covered, ..
7958            } => *covered = channels,
7959        }
7960        let body = match serde_json::to_vec(&push) {
7961            Ok(body) => body,
7962            Err(err) => {
7963                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7964                continue;
7965            }
7966        };
7967        let frame = match Frame::build_with_version(
7968            target.negotiated_ver,
7969            FrameType::Push,
7970            control_flags(),
7971            0,
7972            0,
7973            0,
7974            body.clone(),
7975        ) {
7976            Ok(frame) => frame,
7977            Err(err) => {
7978                warn!(
7979                    route_channel = target.channel,
7980                    error = %err,
7981                    "failed to build route lifecycle control PUSH frame"
7982                );
7983                continue;
7984            }
7985        };
7986        if let Err(err) = target.sink.try_send(frame) {
7987            if target.close_on_delivery_failure() {
7988                warn!(
7989                    target_connection_id = target.connection_id.get(),
7990                    route_channel = target.channel,
7991                    error = %err,
7992                    "route lifecycle control PUSH was not delivered to client; closing target connection"
7993                );
7994                let _ = forwarding.escalate_client_delivery_failure(
7995                    target.connection_id,
7996                    target.channel,
7997                    target.epoch,
7998                    CloseReason::new(
7999                        "route_lifecycle_push_delivery_failed",
8000                        format!(
8001                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8002                            target.channel
8003                        ),
8004                    ),
8005                    crate::forwarding::UndeliveredFrame {
8006                        module_id: target.module_id.as_deref(),
8007                        sink: &target.sink,
8008                    },
8009                );
8010            }
8011        }
8012    }
8013}
8014
8015#[cfg(test)]
8016mod tests {
8017    mod activity_read {
8018        include!("activity_read_tests.rs");
8019    }
8020
8021    mod lifecycle_activity {
8022        include!("control_lifecycle_tests.rs");
8023    }
8024    #[cfg(unix)]
8025    #[tokio::test]
8026    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8027        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8028        let path = dir.join("subc.jsonc");
8029        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8030            "program":"/bin/sleep","args":["60"],"protocol":"none",
8031            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8032        }}}).to_string()).unwrap();
8033        let mut configured = crate::daemon_config::load(&path)
8034            .unwrap()
8035            .unwrap()
8036            .modules
8037            .pop()
8038            .unwrap();
8039        let registry = std::sync::Arc::new(crate::Registry::default());
8040        let handle = crate::SupervisorHandle::new();
8041        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8042            .with_handle(handle.clone());
8043        let module = supervisor
8044            .supervise_configured_with_health(
8045                configured.module_spec(),
8046                true,
8047                configured.health.clone(),
8048                None,
8049                configured.restart,
8050            )
8051            .unwrap();
8052        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8053        let before = module.status().unwrap().pid;
8054        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8055        configured.health.cadence = std::time::Duration::from_secs(3600);
8056        let health_only = handler
8057            .reconcile_supervised_modules(
8058                &supervisor,
8059                vec![configured.clone()],
8060                false,
8061                Principal::Direct,
8062            )
8063            .await
8064            .unwrap();
8065        assert!(
8066            health_only.0.changed_pending_reload.is_empty(),
8067            "health policy is already applied live"
8068        );
8069        assert_eq!(module.status().unwrap().pid, before);
8070        assert_eq!(
8071            module.configuration().unwrap().1.http,
8072            configured.health.http
8073        );
8074        configured.args = vec!["61".into()];
8075        let launch = handler
8076            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8077            .await
8078            .unwrap();
8079        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8080        assert_eq!(
8081            module.status().unwrap().pid,
8082            before,
8083            "a launch edit is stored until reload"
8084        );
8085        module.drain().await.unwrap();
8086    }
8087    use cortexkit_test_support::ScratchDir;
8088    use std::{
8089        collections::BTreeMap,
8090        fmt,
8091        path::PathBuf,
8092        sync::{Arc, Mutex},
8093        time::Duration,
8094    };
8095
8096    use serde_json::{json, Value};
8097    use subc_protocol::{
8098        manifest::{
8099            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8100            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8101        },
8102        session::HealthStatus,
8103        FrameType,
8104    };
8105
8106    use super::*;
8107    use crate::{
8108        forwarding::{DataRoute, DataRouteState},
8109        registry::ChannelState,
8110        router::FrameSink,
8111        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8112        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8113        RouteCtx, Router,
8114    };
8115    use tokio::{
8116        sync::mpsc,
8117        time::{sleep, Instant},
8118    };
8119    use tracing::{
8120        field::{Field, Visit},
8121        Event, Subscriber,
8122    };
8123    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8124
8125    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8126    ///
8127    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8128    /// is only populated for `tests/*.rs` integration test binaries -- this file
8129    /// compiles as part of the library target, which gets neither. This test's
8130    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8131    /// and the sibling binary lives two directories up at
8132    /// `<target-dir>/<profile>/fake-aft-stub`.
8133    ///
8134    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8135    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8136    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8137    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8138    /// `NotFound`, which reads as a broken test rather than an unbuilt
8139    /// dependency -- so state the cause and the remedy instead. Deliberately a
8140    /// panic and not a silent skip: a test that quietly passes when it could not
8141    /// run is worse than one that fails, because it reports health it never
8142    /// verified.
8143    fn fake_aft_stub_path() -> PathBuf {
8144        let mut path = std::env::current_exe().expect("current_exe available in tests");
8145        path.pop(); // .../deps/
8146        path.pop(); // .../<profile>/
8147        path.push(if cfg!(windows) {
8148            "fake-aft-stub.exe"
8149        } else {
8150            "fake-aft-stub"
8151        });
8152        assert!(
8153            path.exists(),
8154            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8155             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8156            path.display()
8157        );
8158        path
8159    }
8160
8161    /// Whether clients retry `code` in place: the predicate itself, never a copy
8162    /// of its set. A copied list breaks silently when a code is added to or
8163    /// removed from the real one, and a stale copy here would let exactly the
8164    /// failure this test exists to catch pass.
8165    fn client_retries(code: &str) -> bool {
8166        subc_protocol::error_codes::is_retryable_route_open(code)
8167    }
8168
8169    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8170    /// of failure is worse than publishing none. A permanent fault dressed as
8171    /// retryable makes every client in the fleet retry forever against something
8172    /// that cannot recover; a transient fault dressed as permanent abandons work
8173    /// that would have succeeded.
8174    ///
8175    /// Asserting "a code exists" cannot catch either, because the string is free
8176    /// to say anything. This enumerates every variant and pins which side of the
8177    /// retry boundary it lands on, so a new variant must be classified here
8178    /// deliberately rather than inheriting whichever arm it was appended to.
8179    #[test]
8180    fn retryability_of_forwarding_codes_matches_the_failure() {
8181        // Transient by nature: the target is booting, reloading, or its endpoint
8182        // was swapped mid-flight. Retrying is how these resolve.
8183        let transient = [
8184            ForwardingError::NoModuleConnection,
8185            ForwardingError::ModuleReloading {
8186                module_id: "m".into(),
8187            },
8188            ForwardingError::StaleModuleEndpoint,
8189            ForwardingError::UnknownReservation {
8190                client_channel: 1,
8191                module_channel: 1,
8192            },
8193            ForwardingError::ConnectionClosing {
8194                connection_id: ConnectionId::new(1),
8195            },
8196            ForwardingError::ClientEgressClosed {
8197                connection_id: ConnectionId::new(1),
8198            },
8199            ForwardingError::ModuleEgressUnavailable {
8200                connection_id: ConnectionId::new(1),
8201            },
8202        ];
8203        for err in transient {
8204            let code = forwarding_error_code(&err);
8205            assert!(
8206                client_retries(code),
8207                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8208            );
8209        }
8210
8211        // Not fixed by retrying. Channel and correlation exhaustion need the
8212        // caller to close routes, and a poisoned lock is a daemon that cannot
8213        // recover at all — the worst thing to advertise as retryable, since every
8214        // client would storm a daemon that will never answer.
8215        let permanent = [
8216            ForwardingError::ConnectionRoleConflict {
8217                connection_id: ConnectionId::new(1),
8218            },
8219            ForwardingError::ClientRouteChannelExhausted {
8220                connection_id: ConnectionId::new(1),
8221            },
8222            ForwardingError::ModuleRouteChannelExhausted {
8223                endpoint: ModuleEndpointId {
8224                    connection_id: ConnectionId::new(1),
8225                    generation: 1,
8226                },
8227            },
8228            ForwardingError::RelayCorrelationExhausted,
8229            ForwardingError::RouteOpenBuild("x".into()),
8230            ForwardingError::Poisoned,
8231        ];
8232        for err in permanent {
8233            let code = forwarding_error_code(&err);
8234            assert!(
8235                !client_retries(code),
8236                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8237            );
8238        }
8239    }
8240
8241    /// The principal is the daemon's answer to "who is calling", and modules
8242    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8243    /// plexus gates connector invocation. So a stamp is an authorization input in
8244    /// another process, not a label — and both possible answers SUCCEED, which is
8245    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8246    /// first-party capability to something that never proved it; a supervised one
8247    /// stamped `Direct` silently strips a module of capability it is entitled to.
8248    ///
8249    /// Neither shows up in a test that only checks the bind succeeded. Before this
8250    /// test the only coverage was accidental —
8251    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8252    /// stamped principal on its way past, so narrowing that wire-shape test to its
8253    /// stated subject would have deleted the last assertion on this value. It
8254    /// still asserts the stamp, which is now redundancy rather than the only
8255    /// guard: both fail under the same mutation, and this one names the reason.
8256    /// SCOPE: this handler's supervisor has spawned nothing, so
8257    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8258    /// is unreachable here. Both assertions below are refusals, and a mutant that
8259    /// refuses everything would satisfy them.
8260    ///
8261    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8262    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8263    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8264    /// at source rather than assumed, since a citation is a claim about another
8265    /// file and ages like one. Recorded because a harness that structurally
8266    /// cannot reach an arm reports "none" for that arm identically to one that
8267    /// covers it and found nothing.
8268    #[tokio::test]
8269    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8270        let handler = ControlHandler::default();
8271        let frame =
8272            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8273
8274        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8275        // any process holding the connection file. Nothing was proved, so nothing
8276        // may be granted beyond the unattested floor.
8277        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8278        assert_eq!(
8279            stamped,
8280            Principal::Direct,
8281            "a caller that proved nothing must not be stamped as a supervised module"
8282        );
8283
8284        // A claimed module_id with a nonce no supervised child was given is a
8285        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8286        // quietly demoted to Direct, or an impersonation attempt looks identical
8287        // to an ordinary unattested connection.
8288        let forged = handler
8289            .route_open_principal(
8290                &frame,
8291                Some(ConsumerIdentity {
8292                    module_id: "aft".to_string(),
8293                    launch_nonce: "not-a-real-nonce".to_string(),
8294                }),
8295            )
8296            .unwrap();
8297        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8298        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8299    }
8300
8301    /// The test above hands `route_open_principal` an identity it built itself,
8302    /// which proves the stamping rule and nothing about where the identity comes
8303    /// from. The real producer is a wire body, and the two are joined by a serde
8304    /// field name that nothing else asserts.
8305    ///
8306    /// That join fails quietly in one specific way: an unrecognised key is simply
8307    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8308    /// yields `None` and every supervised module silently drops to `Direct`.
8309    /// Capability-wise that is the safe direction, but it surfaces far from its
8310    /// cause — as a module mysteriously refused bash — and it would pass every
8311    /// test that builds its own input.
8312    ///
8313    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8314    /// would break every client the moment the daemon gains a field, trading a
8315    /// quiet demotion for a hard refusal on additive change. Asserting the join
8316    /// instead means a rename breaks a test here rather than the fleet.
8317    #[test]
8318    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8319        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8320        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8321        let ClientControlRequest::RouteOpen {
8322            consumer_identity, ..
8323        } = parsed
8324        else {
8325            panic!("route.open body must parse as RouteOpen");
8326        };
8327        assert_eq!(
8328            consumer_identity,
8329            Some(ConsumerIdentity {
8330                module_id: "aft".to_string(),
8331                launch_nonce: "n".to_string(),
8332            }),
8333            "the wire field name must reach the value route_open_principal reads"
8334        );
8335    }
8336
8337    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8338        ModuleManifest::builder(module_id, "0.1.0")
8339            .protocol_ver(protocol_ver)
8340            .provides(vec![ProviderRole::ToolProvider {
8341                tools: vec![Tool {
8342                    name: "read".to_string(),
8343                    description: None,
8344                    execution_mode: ExecutionMode::Pure,
8345                    schema: json!({"type": "object"}),
8346                }],
8347                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8348                concurrency: Concurrency::ModuleManaged,
8349                emits_push: true,
8350                sub_supervises: true,
8351            }])
8352            .build()
8353    }
8354
8355    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8356        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8357    }
8358
8359    fn hello_frame_with_control_ops(
8360        module_id: &str,
8361        protocol_ver: u8,
8362        corr: u64,
8363        control_ops: Option<Vec<String>>,
8364    ) -> Frame {
8365        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8366    }
8367
8368    fn hello_frame_with_nonce(
8369        module_id: &str,
8370        protocol_ver: u8,
8371        corr: u64,
8372        launch_nonce: Option<&str>,
8373    ) -> Frame {
8374        hello_frame_full(
8375            module_id,
8376            protocol_ver,
8377            corr,
8378            None,
8379            launch_nonce.map(ToOwned::to_owned),
8380        )
8381    }
8382
8383    fn hello_frame_full(
8384        module_id: &str,
8385        protocol_ver: u8,
8386        corr: u64,
8387        control_ops: Option<Vec<String>>,
8388        launch_nonce: Option<String>,
8389    ) -> Frame {
8390        let body = serde_json::to_vec(&ModuleHelloBody {
8391            manifest: manifest(module_id, protocol_ver),
8392            protocol_ver,
8393            control_ops,
8394            launch_nonce,
8395        })
8396        .unwrap();
8397        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8398    }
8399
8400    fn hello_frame_with_rich_ui(
8401        module_id: &str,
8402        protocol_ver: u8,
8403        corr: u64,
8404        launch_nonce: Option<&str>,
8405        rich_ui: Value,
8406    ) -> Frame {
8407        let mut body = serde_json::to_value(ModuleHelloBody {
8408            manifest: manifest(module_id, protocol_ver),
8409            protocol_ver,
8410            control_ops: None,
8411            launch_nonce: launch_nonce.map(str::to_string),
8412        })
8413        .unwrap();
8414        body["manifest"]["rich_ui"] = rich_ui;
8415        Frame::build(
8416            FrameType::Hello,
8417            control_flags(),
8418            0,
8419            0,
8420            corr,
8421            serde_json::to_vec(&body).unwrap(),
8422        )
8423        .unwrap()
8424    }
8425
8426    fn non_routable_hello_frame_with_control_ops(
8427        module_id: &str,
8428        corr: u64,
8429        control_ops: Option<Vec<String>>,
8430    ) -> Frame {
8431        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8432        manifest.provides.clear();
8433        let body = serde_json::to_vec(&ModuleHelloBody {
8434            manifest,
8435            protocol_ver: PROTOCOL_VERSION,
8436            control_ops,
8437            launch_nonce: None,
8438        })
8439        .unwrap();
8440        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8441    }
8442
8443    fn capability_grammar_hello_frame(
8444        capabilities: Value,
8445        runtime_computed: Option<Value>,
8446        corr: u64,
8447    ) -> Frame {
8448        let mut body = serde_json::to_value(ModuleHelloBody {
8449            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8450            protocol_ver: PROTOCOL_VERSION,
8451            control_ops: None,
8452            launch_nonce: None,
8453        })
8454        .expect("HELLO body serializes");
8455        body["manifest"]["capabilities"] = capabilities;
8456        if let Some(runtime_computed) = runtime_computed {
8457            body["runtime_computed"] = runtime_computed;
8458        }
8459        Frame::build(
8460            FrameType::Hello,
8461            control_flags(),
8462            0,
8463            0,
8464            corr,
8465            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8466        )
8467        .expect("HELLO frame builds")
8468    }
8469
8470    fn channel_request(channel: u16, corr: u64) -> Frame {
8471        Frame::build(
8472            FrameType::Request,
8473            Flags::new(true, Priority::Interactive, false),
8474            channel,
8475            0,
8476            corr,
8477            b"opaque".to_vec(),
8478        )
8479        .unwrap()
8480    }
8481
8482    fn route_ctx(
8483        connection_id: ConnectionId,
8484    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8485        let (tx, rx) = mpsc::channel(8);
8486        (
8487            RouteCtx {
8488                connection_id,
8489                egress: FrameSink::new(tx),
8490            },
8491            rx,
8492        )
8493    }
8494
8495    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8496        serde_json::from_slice(&frame.body).unwrap()
8497    }
8498
8499    /// Register a module over a connection that has a sink and return the
8500    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8501    /// module's own sink rather than returning it as a reply, so the ack is
8502    /// taken off `rx` here and whatever the test reads next is what followed it.
8503    async fn hello_via_sink(
8504        handler: &ControlHandler,
8505        ctx: &RouteCtx,
8506        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8507        hello: Frame,
8508    ) -> Frame {
8509        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8510        assert!(
8511            replies.is_empty(),
8512            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8513        );
8514        let ack = rx
8515            .try_recv()
8516            .expect("HELLO_ACK is queued on the module sink")
8517            .frame;
8518        assert_eq!(ack.header.ty, FrameType::HelloAck);
8519        ack
8520    }
8521
8522    fn parse_error(frame: &Frame) -> Value {
8523        serde_json::from_slice(&frame.body).unwrap()
8524    }
8525
8526    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8527        serde_json::from_slice(&frame.body).unwrap()
8528    }
8529
8530    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8531        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8532            route_channel,
8533            route_epoch: 0,
8534            kind,
8535        })
8536        .unwrap();
8537        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8538    }
8539
8540    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8541        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8542            module_id: module_id.to_string(),
8543        })
8544        .unwrap();
8545        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8546    }
8547
8548    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8549        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8550    }
8551
8552    fn route_open_frame_with_consumer_capabilities(
8553        corr: u64,
8554        module_id: &str,
8555        project_root: ScratchDir,
8556        consumer_capabilities: Option<Vec<String>>,
8557    ) -> Frame {
8558        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8559            target: RouteTarget::ToolProvider {
8560                module_id: module_id.to_string(),
8561            },
8562            identity: BindIdentity::new(
8563                project_root.path().to_path_buf(),
8564                "unit".to_string(),
8565                "session".to_string(),
8566            ),
8567            consumer_identity: None,
8568            consumer_capabilities,
8569            role_versions: None,
8570            admission_facts: None,
8571            scope: None,
8572        })
8573        .unwrap();
8574        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8575    }
8576
8577    fn route_open_frame_with_role_versions(
8578        corr: u64,
8579        module_id: &str,
8580        project_root: ScratchDir,
8581        role_versions: Option<BTreeMap<String, String>>,
8582    ) -> Frame {
8583        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8584            target: RouteTarget::ToolProvider {
8585                module_id: module_id.to_string(),
8586            },
8587            identity: BindIdentity::new(
8588                project_root.path().to_path_buf(),
8589                "unit".to_string(),
8590                format!("session-{corr}"),
8591            ),
8592            consumer_identity: None,
8593            consumer_capabilities: None,
8594            role_versions,
8595            admission_facts: None,
8596            scope: None,
8597        })
8598        .unwrap();
8599        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8600    }
8601
8602    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8603        entries
8604            .iter()
8605            .map(|(role, version)| (role.to_string(), version.to_string()))
8606            .collect()
8607    }
8608
8609    fn route_open_frame_with_admission_facts(
8610        corr: u64,
8611        module_id: &str,
8612        project_root: ScratchDir,
8613        consumer_identity: Option<subc_control::ConsumerIdentity>,
8614        facts: Option<Value>,
8615    ) -> Frame {
8616        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8617            target: RouteTarget::ToolProvider {
8618                module_id: module_id.to_string(),
8619            },
8620            identity: BindIdentity::new(
8621                project_root.path().to_path_buf(),
8622                "unit".to_string(),
8623                format!("session-{corr}"),
8624            ),
8625            consumer_identity,
8626            consumer_capabilities: None,
8627            role_versions: None,
8628            admission_facts: facts,
8629            scope: None,
8630        })
8631        .unwrap();
8632        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8633    }
8634
8635    #[derive(Clone, Default)]
8636    struct EventCapture {
8637        events: Arc<Mutex<Vec<CapturedEvent>>>,
8638    }
8639
8640    #[derive(Clone, Debug)]
8641    struct CapturedEvent {
8642        target: String,
8643        level: tracing::Level,
8644        fields: BTreeMap<String, String>,
8645    }
8646
8647    impl EventCapture {
8648        fn events(&self) -> Vec<CapturedEvent> {
8649            self.events.lock().unwrap().clone()
8650        }
8651    }
8652
8653    impl<S> Layer<S> for EventCapture
8654    where
8655        S: Subscriber,
8656    {
8657        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8658            let mut visitor = EventFieldVisitor::default();
8659            event.record(&mut visitor);
8660            self.events.lock().unwrap().push(CapturedEvent {
8661                target: event.metadata().target().to_string(),
8662                level: *event.metadata().level(),
8663                fields: visitor.fields,
8664            });
8665        }
8666    }
8667
8668    #[derive(Default)]
8669    struct EventFieldVisitor {
8670        fields: BTreeMap<String, String>,
8671    }
8672
8673    impl Visit for EventFieldVisitor {
8674        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8675            self.fields
8676                .insert(field.name().to_string(), format!("{value:?}"));
8677        }
8678    }
8679
8680    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8681        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8682            status,
8683            detail: Some("warming".to_string()),
8684            metrics: Some(json!({"queue_depth": 3})),
8685        })
8686        .unwrap();
8687        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8688    }
8689
8690    fn route_bind_ack(corr: u64) -> Frame {
8691        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8692        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8693    }
8694
8695    fn unique_project_root(label: &str) -> ScratchDir {
8696        ScratchDir::new(label)
8697    }
8698
8699    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8700        match parse_route_poll(frame) {
8701            ClientControlResponse::RoutePoll {
8702                status: None,
8703                live: Some(live),
8704                ..
8705            } => assert_eq!(live, expected_live),
8706            other => panic!("unexpected route.poll response: {other:?}"),
8707        }
8708    }
8709
8710    fn bind_liveness_route(
8711        registry: &Registry,
8712        forwarding: &ForwardingTable,
8713        module_id: &str,
8714    ) -> (RouteCtx, u16, u32) {
8715        let module_connection = ConnectionId::new(101);
8716        let client_connection = ConnectionId::new(202);
8717        let registration = registry
8718            .register_with_control_ops(
8719                manifest(module_id, PROTOCOL_VERSION),
8720                PROTOCOL_VERSION,
8721                module_connection,
8722                module_baseline_control_ops(),
8723            )
8724            .unwrap();
8725        let (module_tx, _module_rx) = mpsc::channel(8);
8726        let endpoint = forwarding
8727            .register_module_connection(
8728                module_connection,
8729                module_id.to_string(),
8730                PROTOCOL_VERSION,
8731                manifest_concurrency(&registration.manifest),
8732                FrameSink::new(module_tx),
8733            )
8734            .unwrap();
8735        let (client_ctx, _client_rx) = route_ctx(client_connection);
8736        let pending = forwarding
8737            .begin_route_bind_relay_for_test(
8738                client_connection,
8739                client_ctx.egress.clone(),
8740                1,
8741                module_id,
8742            )
8743            .unwrap();
8744        assert_eq!(pending.endpoint, endpoint);
8745        let route_channel = pending.client_channel;
8746        let route_epoch = pending.client_epoch;
8747        forwarding
8748            .complete_pending_relay(
8749                module_connection,
8750                pending.corr,
8751                RouteBindRelayOutcome::Accepted,
8752            )
8753            .unwrap();
8754        (client_ctx, route_channel, route_epoch)
8755    }
8756
8757    struct FakeProcessLiveness {
8758        live: Option<bool>,
8759    }
8760
8761    impl ModuleProcessLiveness for FakeProcessLiveness {
8762        fn process_live(&self, _module_id: &str) -> Option<bool> {
8763            self.live
8764        }
8765    }
8766
8767    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8768    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8769    {
8770        let registry = Arc::new(Registry::default());
8771        let supervisor_handle = SupervisorHandle::new();
8772        let supervisor = Supervisor::new_for_test(
8773            Arc::clone(&registry),
8774            RestartPolicy::new(1, Duration::from_millis(10)),
8775        )
8776        .with_handle(supervisor_handle.clone());
8777        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8778        let module = supervisor
8779            .spawn(ModuleSpec {
8780                module_id: "stderr-tail-wire".to_string(),
8781                program: fake_aft_stub_path(),
8782                args: Vec::new(),
8783                env: vec![
8784                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8785                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8786                ],
8787                reserved: false,
8788                reserved_prefixes: Vec::new(),
8789                protocol: ModuleProtocol::Subc,
8790                overlap: Default::default(),
8791            })
8792            .unwrap();
8793
8794        let deadline = Instant::now() + Duration::from_secs(5);
8795        loop {
8796            let tail = module.stderr_tail(None, None);
8797            if tail
8798                .entries
8799                .iter()
8800                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8801                && tail.entries.iter().any(|entry| {
8802                    matches!(
8803                        entry,
8804                        TailEntry::Line {
8805                            truncated: true,
8806                            ..
8807                        }
8808                    )
8809                })
8810            {
8811                break;
8812            }
8813            assert!(
8814                Instant::now() < deadline,
8815                "module did not produce a truncated line and restart boundary: {tail:?}"
8816            );
8817            sleep(Duration::from_millis(10)).await;
8818        }
8819
8820        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8821        let request = ClientControlRequest::SupervisorStderrTail {
8822            module_id: "stderr-tail-wire".to_string(),
8823            max_lines: None,
8824            max_bytes: None,
8825        };
8826        let frame = Frame::build(
8827            FrameType::Request,
8828            control_flags(),
8829            0,
8830            0,
8831            1,
8832            serde_json::to_vec(&request).unwrap(),
8833        )
8834        .unwrap();
8835        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8836        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8837        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8838            serde_json::from_slice(&responses[0].body).unwrap()
8839        else {
8840            panic!("expected supervisor.stderr_tail response");
8841        };
8842
8843        assert!(
8844            tail.entries
8845                .iter()
8846                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8847            "the control response lost the restart boundary"
8848        );
8849        let Some(StderrTailEntry::Line {
8850            text,
8851            truncated,
8852            at_ms,
8853        }) = tail.entries.iter().find(|entry| {
8854            matches!(
8855                entry,
8856                StderrTailEntry::Line {
8857                    truncated: true,
8858                    ..
8859                }
8860            )
8861        })
8862        else {
8863            panic!("the control response lost the truncated line");
8864        };
8865        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8866        assert!(*truncated);
8867        assert!(
8868            at_ms.is_some(),
8869            "the control response lost the line's capture time"
8870        );
8871    }
8872
8873    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8874    /// read done on the worker thread would stall every other task until it
8875    /// finished; the read must run off the worker so this test's own task keeps
8876    /// running while the read is paused.
8877    #[tokio::test(flavor = "current_thread")]
8878    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8879        let dir = ScratchDir::new("terminals-off-worker");
8880        let journal_path = dir.join("terminals.jsonl");
8881        let registry = Arc::new(Registry::default());
8882        let supervisor_handle = SupervisorHandle::new();
8883        let supervisor =
8884            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8885                .with_handle(supervisor_handle.clone())
8886                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8887        let module = supervisor
8888            .spawn(ModuleSpec {
8889                module_id: "terminal-off-worker".to_string(),
8890                program: fake_aft_stub_path(),
8891                args: Vec::new(),
8892                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8893                reserved: false,
8894                reserved_prefixes: Vec::new(),
8895                protocol: ModuleProtocol::Subc,
8896                overlap: Default::default(),
8897            })
8898            .unwrap();
8899        let deadline = Instant::now() + Duration::from_secs(5);
8900        while module.terminal_history().entries.len() != 2 {
8901            assert!(Instant::now() < deadline, "module did not record two exits");
8902            sleep(Duration::from_millis(10)).await;
8903        }
8904
8905        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8906        let handler =
8907            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8908        let frame = Frame::build(
8909            FrameType::Request,
8910            control_flags(),
8911            0,
8912            0,
8913            1,
8914            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8915                module_id: "terminal-off-worker".to_string(),
8916            })
8917            .unwrap(),
8918        )
8919        .unwrap();
8920        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8921        let spawned_at = std::time::Instant::now();
8922        let read = tokio::spawn({
8923            let handler = Arc::clone(&handler);
8924            async move { handler.handle_control_frame(&ctx, frame).await }
8925        });
8926        // Waiting for the pause from a blocking thread keeps this task pending,
8927        // so the runtime's single worker is free to run the read task.
8928        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8929            .await
8930            .unwrap()
8931            .expect("the history read reached its pause");
8932        let elapsed = spawned_at.elapsed();
8933        assert!(
8934            elapsed < Duration::from_secs(2) && !read.is_finished(),
8935            "this task could not run while the history read was paused \
8936             (resumed after {elapsed:?}, read finished: {})",
8937            read.is_finished()
8938        );
8939
8940        drop(release);
8941        let responses = read.await.unwrap().unwrap();
8942        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8943        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8944            panic!("expected supervisor.terminals response");
8945        };
8946        assert_eq!(terminals.entries.len(), 2);
8947        assert_eq!(terminals.journal_skipped_lines, 0);
8948        assert_eq!(terminals.journal_read_errors, 0);
8949    }
8950
8951    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8952    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8953        let registry = Arc::new(Registry::default());
8954        let supervisor_handle = SupervisorHandle::new();
8955        let supervisor =
8956            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8957                .with_handle(supervisor_handle.clone());
8958        let module = supervisor
8959            .spawn(ModuleSpec {
8960                module_id: "terminal-golden".to_string(),
8961                program: fake_aft_stub_path(),
8962                args: Vec::new(),
8963                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8964                reserved: false,
8965                reserved_prefixes: Vec::new(),
8966                protocol: ModuleProtocol::Subc,
8967                overlap: Default::default(),
8968            })
8969            .unwrap();
8970
8971        let deadline = Instant::now() + Duration::from_secs(5);
8972        while module.terminal_history().entries.len() != 2 {
8973            assert!(
8974                Instant::now() < deadline,
8975                "module did not retain two terminal exits: {:?}",
8976                module.terminal_history()
8977            );
8978            sleep(Duration::from_millis(10)).await;
8979        }
8980
8981        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8982        let request = ClientControlRequest::SupervisorTerminals {
8983            module_id: "terminal-golden".to_string(),
8984        };
8985        let frame = Frame::build(
8986            FrameType::Request,
8987            control_flags(),
8988            0,
8989            0,
8990            1,
8991            serde_json::to_vec(&request).unwrap(),
8992        )
8993        .unwrap();
8994        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8995        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8996        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8997        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8998            panic!("expected supervisor.terminals response");
8999        };
9000        assert_eq!(terminals.entries.len(), 2);
9001        assert_eq!(terminals.dropped, 0);
9002
9003        let mut rendered = serde_json::to_value(response).unwrap();
9004        // Wall-clock fields are the observation contract, but not stable fixture
9005        // bytes; normalize only them after the real handler has shaped the response.
9006        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9007        for (index, entry) in rendered["entries"]
9008            .as_array_mut()
9009            .expect("terminal response entries array")
9010            .iter_mut()
9011            .enumerate()
9012        {
9013            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9014        }
9015
9016        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9017            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9018        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9019        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9020            std::fs::write(&golden_path, &serialized).unwrap();
9021        }
9022        let expected: Value =
9023            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9024        assert_eq!(rendered, expected);
9025    }
9026
9027    #[test]
9028    fn hello_registers_manifest_and_returns_ack() {
9029        let registry = Arc::new(Registry::default());
9030        let handler = ControlHandler::new(Arc::clone(&registry));
9031        let conn = ConnectionId::new(1);
9032
9033        let responses = handler
9034            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9035            .unwrap();
9036
9037        assert_eq!(responses.len(), 1);
9038        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9039        assert_eq!(responses[0].header.channel, 0);
9040        assert_eq!(responses[0].header.corr, 7);
9041        let ack = parse_ack(&responses[0]);
9042        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9043        assert!(ack
9044            .subc_capabilities
9045            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9046        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9047        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9048        assert!(ack
9049            .subc_ops
9050            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9051        assert!(ack
9052            .subc_ops
9053            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9054
9055        let registration = registry.get_module("aft").unwrap().unwrap();
9056        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9057        assert_eq!(registration.state, ChannelState::Active);
9058        assert_eq!(registration.connection_id, conn);
9059        assert_eq!(registration.control_ops, module_baseline_control_ops());
9060    }
9061
9062    #[test]
9063    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9064        let invalid_identifiers = [
9065            ("case_change", "credentials-Provider/v1"),
9066            ("leading_zero", "credentials-provider/v01"),
9067            ("trailing_hyphen", "credentials-provider-/v1"),
9068            ("consecutive_hyphens", "credentials--provider/v1"),
9069            ("uppercase", "Credentials-provider/v1"),
9070            ("missing_v", "credentials-provider/1"),
9071            ("whitespace", "credentials provider/v1"),
9072            ("zero_version", "credentials-provider/v0"),
9073            ("out_of_range_version", "credentials-provider/v4294967296"),
9074            (
9075                "overlength_name",
9076                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9077            ),
9078        ];
9079        let mut cases = invalid_identifiers
9080            .into_iter()
9081            .map(|(name, identifier)| {
9082                (
9083                    format!("identifier_{name}"),
9084                    "capabilities.provides[0]".to_string(),
9085                    identifier.to_string(),
9086                    json!({ "provides": [identifier] }),
9087                    None,
9088                )
9089            })
9090            .collect::<Vec<_>>();
9091        cases.extend([
9092            (
9093                "unknown_need".to_string(),
9094                "capabilities.requires[0].need".to_string(),
9095                "deferred".to_string(),
9096                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9097                None,
9098            ),
9099            (
9100                "duplicate_provides".to_string(),
9101                "capabilities.provides[1]".to_string(),
9102                "credentials-provider/v1".to_string(),
9103                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9104                None,
9105            ),
9106            (
9107                "duplicate_must_never_reach".to_string(),
9108                "capabilities.must_never_reach[1]".to_string(),
9109                "credentials-provider/v1".to_string(),
9110                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9111                None,
9112            ),
9113            (
9114                "duplicate_requires_same_need".to_string(),
9115                "capabilities.requires[1]".to_string(),
9116                "credentials-provider/v1".to_string(),
9117                json!({ "requires": [
9118                    { "capability": "credentials-provider/v1", "need": "required" },
9119                    { "capability": "credentials-provider/v1", "need": "required" }
9120                ] }),
9121                None,
9122            ),
9123            (
9124                "duplicate_requires_conflicting_need".to_string(),
9125                "capabilities.requires[1]".to_string(),
9126                "credentials-provider/v1".to_string(),
9127                json!({ "requires": [
9128                    { "capability": "credentials-provider/v1", "need": "required" },
9129                    { "capability": "credentials-provider/v1", "need": "optional" }
9130                ] }),
9131                None,
9132            ),
9133            (
9134                "capabilities_root_pointer".to_string(),
9135                "runtime_computed[0]".to_string(),
9136                "/capabilities".to_string(),
9137                json!({}),
9138                Some(json!(["/capabilities"])),
9139            ),
9140            (
9141                "capabilities_descendant_pointer".to_string(),
9142                "runtime_computed[0]".to_string(),
9143                "/capabilities/provides".to_string(),
9144                json!({}),
9145                Some(json!(["/capabilities/provides"])),
9146            ),
9147            (
9148                "malformed_pointer_without_leading_slash".to_string(),
9149                "runtime_computed[0]".to_string(),
9150                "capabilities".to_string(),
9151                json!({}),
9152                Some(json!(["capabilities"])),
9153            ),
9154            (
9155                "malformed_pointer_escape".to_string(),
9156                "runtime_computed[0]".to_string(),
9157                "/roles/~2/tools".to_string(),
9158                json!({}),
9159                Some(json!(["/roles/~2/tools"])),
9160            ),
9161            (
9162                "unknown_capabilities_field".to_string(),
9163                "capabilities.future".to_string(),
9164                "<array>".to_string(),
9165                json!({ "future": [] }),
9166                None,
9167            ),
9168        ]);
9169
9170        for (index, (name, field, value, capabilities, runtime_computed)) in
9171            cases.into_iter().enumerate()
9172        {
9173            let registry = Arc::new(Registry::default());
9174            let handler = ControlHandler::new(Arc::clone(&registry));
9175            let response = handler
9176                .handle_control(
9177                    ConnectionId::new((index + 1) as u64),
9178                    capability_grammar_hello_frame(
9179                        capabilities,
9180                        runtime_computed,
9181                        index as u64 + 1,
9182                    ),
9183                )
9184                .expect("invalid HELLO returns a refusal");
9185
9186            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9187            let error = parse_error(&response[0]);
9188            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9189            let message = error["message"]
9190                .as_str()
9191                .expect("error message is a string");
9192            assert!(
9193                message.contains(&field),
9194                "{name}: field missing from {message}"
9195            );
9196            assert!(
9197                message.contains(&value),
9198                "{name}: value missing from {message}"
9199            );
9200            assert_eq!(
9201                registry
9202                    .active_registration_count()
9203                    .expect("registry reads"),
9204                0,
9205                "{name}: refused HELLO must not create a catalog entry"
9206            );
9207        }
9208    }
9209
9210    #[test]
9211    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9212        let registry = Arc::new(Registry::default());
9213        let handler = ControlHandler::new(Arc::clone(&registry));
9214        let capabilities = json!({
9215            "provides": ["credentials-provider/v1"],
9216            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9217            "must_never_reach": ["federation-transport/v1"]
9218        });
9219        let response = handler
9220            .handle_control(
9221                ConnectionId::new(99),
9222                capability_grammar_hello_frame(
9223                    capabilities.clone(),
9224                    Some(json!(["/roles/0/tools"])),
9225                    99,
9226                ),
9227            )
9228            .expect("valid HELLO registers");
9229        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9230
9231        let request = Frame::build(
9232            FrameType::Request,
9233            control_flags(),
9234            0,
9235            0,
9236            100,
9237            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9238                .expect("catalog request serializes"),
9239        )
9240        .expect("catalog request frame builds");
9241        let response = handler
9242            .handle_catalog_list(request, None)
9243            .expect("catalog list succeeds");
9244        let ClientControlResponse::CatalogList { modules, .. } =
9245            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9246        else {
9247            panic!("catalog request must return catalog.list");
9248        };
9249        assert_eq!(modules.len(), 1);
9250        assert_eq!(
9251            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9252            capabilities
9253        );
9254    }
9255
9256    #[test]
9257    fn catalog_list_mirrors_management_operation_description() {
9258        let registry = Arc::new(Registry::default());
9259        let handler = ControlHandler::new(Arc::clone(&registry));
9260        let description = "List managed records and return their identifiers and metadata.";
9261        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9262        manifest.provides = vec![ProviderRole::ManagementSurface {
9263            operations: vec![ManagementOperation {
9264                name: "records.list".to_string(),
9265                kind: ManagementOperationKind::Query,
9266                description: Some(description.to_string()),
9267            }],
9268            config_schema: json!({"type": "object"}),
9269            observability: vec![ObservabilitySurface {
9270                name: "records.stats".to_string(),
9271                kind: ObservabilityKind::Snapshot,
9272            }],
9273            identity_scope: vec![IdentityScope::Project],
9274            concurrency: Concurrency::ModuleManaged,
9275        }];
9276        registry
9277            .register_with_control_ops(
9278                manifest,
9279                PROTOCOL_VERSION,
9280                ConnectionId::new(99),
9281                Vec::new(),
9282            )
9283            .expect("described management manifest registers");
9284
9285        let request = Frame::build(
9286            FrameType::Request,
9287            control_flags(),
9288            0,
9289            0,
9290            100,
9291            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9292                .expect("catalog request serializes"),
9293        )
9294        .expect("catalog request frame builds");
9295        let response = handler
9296            .handle_catalog_list(request, None)
9297            .expect("catalog list succeeds");
9298        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9299        assert_eq!(
9300            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9301            "catalog.list must preserve the declared operation description verbatim"
9302        );
9303    }
9304
9305    #[test]
9306    fn catalog_list_mirrors_registered_operation_delivery() {
9307        let registry = Arc::new(Registry::default());
9308        let handler = ControlHandler::new(Arc::clone(&registry));
9309        let delivery =
9310            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9311        let mut body = serde_json::to_value(ModuleHelloBody {
9312            manifest: manifest("watch-management", PROTOCOL_VERSION),
9313            protocol_ver: PROTOCOL_VERSION,
9314            control_ops: None,
9315            launch_nonce: None,
9316        })
9317        .unwrap();
9318        body["manifest"]["provides"] = json!([{
9319            "role": "management_surface",
9320            "operations": [
9321                {"name": "status", "kind": "query"},
9322                {"name": "future", "kind": "query"},
9323                {"name": "reset", "kind": "mutate"}
9324            ],
9325            "config_schema": {}, "observability": [], "identity_scope": []
9326        }]);
9327        body["manifest"]["operation_delivery"] = delivery.clone();
9328        let hello = Frame::build(
9329            FrameType::Hello,
9330            control_flags(),
9331            0,
9332            0,
9333            99,
9334            serde_json::to_vec(&body).unwrap(),
9335        )
9336        .unwrap();
9337        let response = handler
9338            .handle_control(ConnectionId::new(99), hello)
9339            .unwrap();
9340        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9341
9342        let (_, registrations) = registry.list_modules().unwrap();
9343        let registered = &registrations[0].manifest;
9344        assert!(registered.is_watch_operation("status"));
9345        assert!(!registered.is_watch_operation("future"));
9346        assert!(!registered.is_watch_operation("reset"));
9347        assert!(!registered.is_watch_operation("missing"));
9348
9349        let request = Frame::build(
9350            FrameType::Request,
9351            control_flags(),
9352            0,
9353            0,
9354            100,
9355            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9356        )
9357        .unwrap();
9358        let response = handler.handle_catalog_list(request, None).unwrap();
9359        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9360        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9361        let ClientControlResponse::CatalogList { modules, .. } =
9362            serde_json::from_slice(&response[0].body).unwrap()
9363        else {
9364            panic!("expected catalog.list");
9365        };
9366        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9367    }
9368
9369    #[test]
9370    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9371        let registry = Arc::new(Registry::default());
9372        let supervisor = SupervisorHandle::new();
9373        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9374        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9375        let descriptor = json!({
9376            "schema": "ckios.richui.manifest/1",
9377            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9378            "bytes": 4096,
9379            "chunks": 4
9380        });
9381        let connection = ConnectionId::new(201);
9382        let hello = handler
9383            .handle_control(
9384                connection,
9385                hello_frame_with_rich_ui(
9386                    "vault",
9387                    PROTOCOL_VERSION,
9388                    201,
9389                    Some("the-real-nonce"),
9390                    descriptor.clone(),
9391                ),
9392            )
9393            .unwrap();
9394        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9395
9396        let request =
9397            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9398        let provides = registry
9399            .get_module_by_connection(connection)
9400            .unwrap()
9401            .unwrap()
9402            .manifest
9403            .provides;
9404        handler
9405            .handle_catalog_update(connection, request, provides, None, Some(false))
9406            .unwrap();
9407
9408        let request = Frame::build(
9409            FrameType::Request,
9410            control_flags(),
9411            0,
9412            0,
9413            203,
9414            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9415        )
9416        .unwrap();
9417        let response = handler.handle_catalog_list(request, None).unwrap();
9418        let ClientControlResponse::CatalogList { modules, .. } =
9419            serde_json::from_slice(&response[0].body).unwrap()
9420        else {
9421            panic!("expected catalog.list");
9422        };
9423        assert_eq!(
9424            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9425            descriptor
9426        );
9427        assert!(modules[0].nonce_verified);
9428    }
9429
9430    fn hello_with_role_declarations(declarations: Value) -> Frame {
9431        let mut body = serde_json::to_value(ModuleHelloBody {
9432            manifest: manifest("role-test", PROTOCOL_VERSION),
9433            protocol_ver: PROTOCOL_VERSION,
9434            control_ops: None,
9435            launch_nonce: None,
9436        })
9437        .unwrap();
9438        body["manifest"]["role_declarations"] = declarations;
9439        Frame::build(
9440            FrameType::Hello,
9441            control_flags(),
9442            0,
9443            0,
9444            301,
9445            serde_json::to_vec(&body).unwrap(),
9446        )
9447        .unwrap()
9448    }
9449
9450    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9451        let request = Frame::build(
9452            FrameType::Request,
9453            control_flags(),
9454            0,
9455            0,
9456            302,
9457            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9458        )
9459        .unwrap();
9460        let response = handler.handle_catalog_list(request, None).unwrap();
9461        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9462        let ClientControlResponse::CatalogList { mut modules, .. } =
9463            serde_json::from_slice(&response[0].body).unwrap()
9464        else {
9465            panic!("expected catalog.list");
9466        };
9467        (raw["modules"][0].clone(), modules.remove(0))
9468    }
9469
9470    #[test]
9471    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9472        let registry = Arc::new(Registry::default());
9473        let handler = ControlHandler::new(Arc::clone(&registry));
9474        let declarations = json!({
9475            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9476            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9477        });
9478        let connection = ConnectionId::new(301);
9479        let response = handler
9480            .handle_control(
9481                connection,
9482                hello_with_role_declarations(declarations.clone()),
9483            )
9484            .unwrap();
9485        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9486        let registered = registry
9487            .get_module_by_connection(connection)
9488            .unwrap()
9489            .unwrap()
9490            .manifest;
9491        // Compared as JSON values, not bytes: object key order is not part of
9492        // the relay's promise, and workspace builds unify serde_json's
9493        // preserve_order feature, which changes the order a literal keeps.
9494        assert_eq!(
9495            serde_json::to_value(&registered.role_declarations).unwrap(),
9496            declarations
9497        );
9498        let (raw, entry) = role_declarations_catalog(&handler);
9499        assert_eq!(raw["role_declarations"], declarations);
9500        assert_eq!(entry.role_declarations, registered.role_declarations);
9501
9502        let request =
9503            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9504        handler
9505            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9506            .unwrap();
9507        let (raw, entry) = role_declarations_catalog(&handler);
9508        assert_eq!(raw["role_declarations"], declarations);
9509        assert_eq!(entry.role_declarations, registered.role_declarations);
9510    }
9511
9512    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9513        let registry = Arc::new(Registry::default());
9514        let handler = ControlHandler::new(Arc::clone(&registry));
9515        let capture = EventCapture::default();
9516        let _subscriber =
9517            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9518        let response = handler
9519            .handle_control(
9520                ConnectionId::new(301),
9521                hello_with_role_declarations(declarations),
9522            )
9523            .unwrap();
9524        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9525        assert!(registry
9526            .get_module_by_connection(ConnectionId::new(301))
9527            .unwrap()
9528            .unwrap()
9529            .manifest
9530            .role_declarations
9531            .is_empty());
9532        let (raw, entry) = role_declarations_catalog(&handler);
9533        assert!(raw.get("role_declarations").is_none());
9534        assert!(entry.role_declarations.is_empty());
9535        let warnings: Vec<_> = capture
9536            .events()
9537            .into_iter()
9538            .filter(|event| {
9539                event.fields.get("message").map(String::as_str)
9540                    == Some("dropping whole role_declarations map from HELLO")
9541            })
9542            .collect();
9543        assert_eq!(warnings.len(), 1);
9544        assert_eq!(warnings[0].level, tracing::Level::WARN);
9545        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9546        assert_eq!(warnings[0].fields["reason"], reason);
9547    }
9548
9549    #[test]
9550    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9551        for value in [
9552            json!(null),
9553            json!(false),
9554            json!(0),
9555            json!("opaque"),
9556            json!([]),
9557        ] {
9558            assert_hello_drops_role_declarations_with_warning(
9559                json!({"routing/v1": {}, "broken/v1": value}),
9560                "role_declarations values must be objects",
9561            );
9562        }
9563        assert_hello_drops_role_declarations_with_warning(
9564            json!([]),
9565            "role_declarations must be an object",
9566        );
9567    }
9568
9569    #[test]
9570    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9571        assert_hello_drops_role_declarations_with_warning(
9572            json!({"routing/v1": {}, "bad key": {}}),
9573            "role_declarations key must use capability-name grammar",
9574        );
9575    }
9576
9577    #[test]
9578    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9579        assert_hello_drops_role_declarations_with_warning(
9580            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9581            "role_declarations exceeds 16384 compact JSON bytes",
9582        );
9583    }
9584
9585    #[test]
9586    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9587        let registry = Arc::new(Registry::default());
9588        let handler = ControlHandler::new(registry.clone());
9589        let descriptor = json!({
9590            "schema": "ckios.richui.manifest/1",
9591            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9592            "bytes": 128,
9593            "chunks": 2
9594        });
9595        let malformed = [
9596            ("schema", json!("")),
9597            ("schema", json!("a".repeat(65))),
9598            ("schema", json!("Bad schema")),
9599            (
9600                "manifest_digest",
9601                json!(format!("sha256:{}", "A".repeat(64))),
9602            ),
9603            ("manifest_digest", json!("sha256:abc")),
9604            ("bytes", json!(0)),
9605            ("bytes", json!(16_777_217)),
9606            ("bytes", json!("128")),
9607            ("chunks", json!(0)),
9608            ("chunks", json!(1025)),
9609            ("chunks", json!("2")),
9610        ];
9611        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9612            let mut invalid = descriptor.clone();
9613            invalid[field] = invalid_value;
9614            let response = handler
9615                .handle_control(
9616                    ConnectionId::new(210 + index as u64),
9617                    hello_frame_with_rich_ui(
9618                        &format!("bad-rich-ui-{index}"),
9619                        PROTOCOL_VERSION,
9620                        210 + index as u64,
9621                        None,
9622                        invalid,
9623                    ),
9624                )
9625                .unwrap();
9626            assert_eq!(response[0].header.ty, FrameType::Error);
9627            let error = parse_error(&response[0]);
9628            assert_eq!(error["code"], "invalid_manifest");
9629            assert!(error["message"]
9630                .as_str()
9631                .unwrap()
9632                .contains(&format!("rich_ui.{field}")));
9633            assert!(registry
9634                .get_module(&format!("bad-rich-ui-{index}"))
9635                .unwrap()
9636                .is_none());
9637        }
9638    }
9639
9640    #[test]
9641    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9642        let registry = Arc::new(Registry::default());
9643        let handler = ControlHandler::new(Arc::clone(&registry));
9644        let connection = ConnectionId::new(219);
9645        registry
9646            .register_with_control_ops(
9647                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9648                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9649                        "bad schema",
9650                        format!("sha256:{}", "a".repeat(64)),
9651                        128,
9652                        2,
9653                    )))
9654                    .build(),
9655                PROTOCOL_VERSION,
9656                connection,
9657                Vec::new(),
9658            )
9659            .unwrap();
9660        let request =
9661            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9662        let response = handler
9663            .handle_catalog_update(connection, request, Vec::new(), None, None)
9664            .unwrap();
9665        assert_eq!(response[0].header.ty, FrameType::Error);
9666        let error = parse_error(&response[0]);
9667        assert_eq!(error["code"], "invalid_manifest");
9668        assert!(error["message"]
9669            .as_str()
9670            .unwrap()
9671            .contains("rich_ui.schema"));
9672    }
9673
9674    #[test]
9675    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9676        let registry = Arc::new(Registry::default());
9677        let supervisor = SupervisorHandle::new();
9678        supervisor.open_swap("provider", "candidate-nonce".to_string());
9679        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9680        let connection = ConnectionId::new(218);
9681        let hello = handler
9682            .handle_control(
9683                connection,
9684                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9685            )
9686            .unwrap();
9687        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9688        assert!(registry.list_modules().unwrap().1.is_empty());
9689
9690        registry.promote_candidate("provider").unwrap().unwrap();
9691        let request = Frame::build(
9692            FrameType::Request,
9693            control_flags(),
9694            0,
9695            0,
9696            219,
9697            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9698        )
9699        .unwrap();
9700        let response = handler.handle_catalog_list(request, None).unwrap();
9701        let ClientControlResponse::CatalogList { modules, .. } =
9702            serde_json::from_slice(&response[0].body).unwrap()
9703        else {
9704            panic!("expected catalog.list");
9705        };
9706        assert_eq!(modules.len(), 1);
9707        assert!(modules[0].nonce_verified);
9708    }
9709
9710    #[test]
9711    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9712        let registry = Arc::new(Registry::default());
9713        let handler = ControlHandler::new(Arc::clone(&registry));
9714        let hello = Frame::build(
9715            FrameType::Hello,
9716            control_flags(),
9717            0,
9718            0,
9719            220,
9720            serde_json::to_vec(&ModuleHelloBody {
9721                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9722                protocol_ver: PROTOCOL_VERSION,
9723                control_ops: None,
9724                launch_nonce: Some("unverified".to_string()),
9725            })
9726            .unwrap(),
9727        )
9728        .unwrap();
9729        assert_eq!(
9730            handler
9731                .handle_control(ConnectionId::new(220), hello)
9732                .unwrap()[0]
9733                .header
9734                .ty,
9735            FrameType::HelloAck
9736        );
9737        let request = Frame::build(
9738            FrameType::Request,
9739            control_flags(),
9740            0,
9741            0,
9742            221,
9743            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9744        )
9745        .unwrap();
9746        let response = handler.handle_catalog_list(request, None).unwrap();
9747        let ClientControlResponse::CatalogList { modules, .. } =
9748            serde_json::from_slice(&response[0].body).unwrap()
9749        else {
9750            panic!("expected catalog.list");
9751        };
9752        assert_eq!(
9753            serde_json::to_vec(&modules[0]).unwrap(),
9754            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9755        );
9756        assert!(!modules[0].nonce_verified);
9757    }
9758
9759    #[test]
9760    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9761        let registry = Arc::new(Registry::default());
9762        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9763            [("vault".to_string(), true), ("squatter".to_string(), true)],
9764            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9765        );
9766        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9767        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9768            provides: vec!["credentials-provider/v1".to_string()],
9769            requires: Vec::new(),
9770            must_never_reach: Vec::new(),
9771        });
9772        let frame = Frame::build(
9773            FrameType::Hello,
9774            control_flags(),
9775            0,
9776            0,
9777            77,
9778            serde_json::to_vec(&ModuleHelloBody {
9779                manifest: squatter,
9780                protocol_ver: PROTOCOL_VERSION,
9781                control_ops: None,
9782                launch_nonce: None,
9783            })
9784            .expect("HELLO serializes"),
9785        )
9786        .expect("HELLO frame builds");
9787        let response = handler
9788            .handle_control(ConnectionId::new(77), frame)
9789            .expect("reserved claim receives a typed refusal");
9790        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9791        assert_eq!(
9792            registry
9793                .active_registration_count()
9794                .expect("registry reads"),
9795            0,
9796            "a reserved capability refusal must not leave a catalog entry"
9797        );
9798    }
9799
9800    #[test]
9801    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9802        for settlement in ["timeout", "inconclusive", "drop"] {
9803            let breakers = RouteBindBreakers::default();
9804            let RouteBindAdmission::Admitted {
9805                guard: mut old,
9806                probe: false,
9807            } = breakers.admit("prov")
9808            else {
9809                panic!("ordinary relay admitted")
9810            };
9811            let RouteBindAdmission::Admitted {
9812                guard: mut opener, ..
9813            } = breakers.admit("prov")
9814            else {
9815                panic!("second relay admitted")
9816            };
9817            assert!(
9818                !opener
9819                    .record_timeout(1, Duration::ZERO)
9820                    .unwrap()
9821                    .reopened_after_probe
9822            );
9823            let RouteBindAdmission::Admitted {
9824                guard: mut probe,
9825                probe: true,
9826            } = breakers.admit("prov")
9827            else {
9828                panic!("one cooldown probe admitted")
9829            };
9830            match settlement {
9831                "timeout" => assert!(
9832                    !old.record_timeout(1, Duration::ZERO)
9833                        .unwrap()
9834                        .reopened_after_probe
9835                ),
9836                "inconclusive" => old.record_inconclusive(),
9837                "drop" => drop(old),
9838                _ => unreachable!(),
9839            }
9840            assert!(
9841                matches!(
9842                    breakers.admit("prov"),
9843                    RouteBindAdmission::Refused {
9844                        probe_in_flight: true,
9845                        ..
9846                    }
9847                ),
9848                "{settlement} of a pre-open relay cannot release the real probe"
9849            );
9850            assert!(
9851                probe
9852                    .record_timeout(1, Duration::ZERO)
9853                    .unwrap()
9854                    .reopened_after_probe
9855            );
9856            assert!(matches!(
9857                breakers.admit("prov"),
9858                RouteBindAdmission::Admitted { probe: true, .. }
9859            ));
9860        }
9861        let breakers = RouteBindBreakers::default();
9862        let admit = || match breakers.admit("prov") {
9863            RouteBindAdmission::Admitted { guard, .. } => guard,
9864            _ => panic!("relay admitted"),
9865        };
9866        admit().record_timeout(1, Duration::ZERO);
9867        let mut old_probe = admit();
9868        breakers.reset_for_new_module_connection("prov");
9869        admit().record_timeout(1, Duration::ZERO);
9870        let _new_probe = admit();
9871        old_probe.record_inconclusive();
9872        assert!(matches!(
9873            breakers.admit("prov"),
9874            RouteBindAdmission::Refused {
9875                probe_in_flight: true,
9876                ..
9877            }
9878        ));
9879    }
9880
9881    #[tokio::test]
9882    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9883        for candidate in [false, true] {
9884            let registry = Arc::new(Registry::default());
9885            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9886                [("vault".to_string(), true), ("squatter".to_string(), true)],
9887                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9888            );
9889            let conn = ConnectionId::new(77);
9890            let (ctx, mut rx) = route_ctx(conn);
9891            let initial = capability_manifest("squatter", &[], &[]);
9892            if candidate {
9893                registry
9894                    .register_candidate_with_control_ops(
9895                        initial.clone(),
9896                        PROTOCOL_VERSION,
9897                        conn,
9898                        module_baseline_control_ops(),
9899                    )
9900                    .unwrap();
9901                handler
9902                    .forwarding
9903                    .register_candidate_module_connection(
9904                        conn,
9905                        "squatter".to_string(),
9906                        PROTOCOL_VERSION,
9907                        manifest_concurrency(&initial),
9908                        ctx.egress.clone(),
9909                    )
9910                    .unwrap();
9911            } else {
9912                hello_via_sink(
9913                    &handler,
9914                    &ctx,
9915                    &mut rx,
9916                    hello_frame_with_manifest(initial.clone(), 1),
9917                )
9918                .await;
9919            }
9920            let response = handler
9921                .handle_control_frame(
9922                    &ctx,
9923                    catalog_update_with_capabilities_frame(
9924                        2,
9925                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
9926                            .capabilities
9927                            .unwrap(),
9928                    ),
9929                )
9930                .await
9931                .unwrap();
9932            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9933            assert_eq!(
9934                registry
9935                    .get_module_by_connection(conn)
9936                    .unwrap()
9937                    .unwrap()
9938                    .manifest,
9939                initial
9940            );
9941        }
9942    }
9943
9944    #[test]
9945    fn server_describe_surfaces_required_capability_verdict_fields() {
9946        let registry = Arc::new(Registry::default());
9947        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9948            [
9949                ("consumer".to_string(), true),
9950                ("provider".to_string(), false),
9951            ],
9952            BTreeMap::new(),
9953        );
9954        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9955        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9956            provides: Vec::new(),
9957            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9958                capability: "credentials-provider/v1".to_string(),
9959                need: subc_protocol::manifest::CapabilityNeed::Required,
9960            }],
9961            must_never_reach: Vec::new(),
9962        });
9963        let hello = Frame::build(
9964            FrameType::Hello,
9965            control_flags(),
9966            0,
9967            0,
9968            78,
9969            serde_json::to_vec(&ModuleHelloBody {
9970                manifest: consumer,
9971                protocol_ver: PROTOCOL_VERSION,
9972                control_ops: None,
9973                launch_nonce: None,
9974            })
9975            .expect("HELLO serializes"),
9976        )
9977        .expect("HELLO frame builds");
9978        handler
9979            .handle_control(ConnectionId::new(78), hello)
9980            .expect("consumer registers");
9981        let describe = Frame::build(
9982            FrameType::Request,
9983            control_flags(),
9984            0,
9985            0,
9986            79,
9987            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9988                .expect("request serializes"),
9989        )
9990        .expect("describe frame builds");
9991        let response = handler
9992            .handle_server_describe(describe)
9993            .expect("server.describe succeeds");
9994        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9995        let requirement = &rendered["capability_requirements"][0];
9996        assert_eq!(requirement["consumer"], "consumer");
9997        assert_eq!(requirement["verdict"], "never_provided");
9998        assert_eq!(requirement["episode_seq"], 1);
9999        assert_eq!(requirement["config_satisfiable"], false);
10000        assert_eq!(requirement["runtime_available"], false);
10001        assert!(requirement["detail"]
10002            .as_str()
10003            .expect("detail string")
10004            .contains("credentials-provider/v1"));
10005    }
10006
10007    /// Register a consumer that requires `project-identity/v1` and the provider
10008    /// of it, optionally mark daemon shutdown as begun, then deregister the
10009    /// provider as its connection closes. Returns the `capability.requirement`
10010    /// events that deregistration logged and the verdict the evaluator then
10011    /// holds for the requirement (the one `route.open` refusals read).
10012    fn deregister_required_provider(
10013        daemon_shutdown_begun: bool,
10014    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10015        const CAPABILITY: &str = "project-identity/v1";
10016        let registry = Arc::new(Registry::default());
10017        let daemon_shutdown = DaemonShutdownFlag::default();
10018        let handler = ControlHandler::new(Arc::clone(&registry))
10019            .with_capability_config(
10020                [
10021                    ("consumer".to_string(), true),
10022                    ("provider".to_string(), true),
10023                ],
10024                BTreeMap::new(),
10025            )
10026            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10027        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10028        consumer.capabilities = Some(CapabilityDeclarations {
10029            provides: Vec::new(),
10030            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10031                capability: CAPABILITY.to_string(),
10032                need: subc_protocol::manifest::CapabilityNeed::Required,
10033            }],
10034            must_never_reach: Vec::new(),
10035        });
10036        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10037        let provider_conn = ConnectionId::new(182);
10038        for (conn, manifest, corr) in [
10039            (ConnectionId::new(181), consumer, 181),
10040            (provider_conn, provider, 182),
10041        ] {
10042            let hello = handler
10043                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10044                .expect("module registers");
10045            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10046        }
10047        assert_eq!(
10048            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10049            Some(CapabilityVerdict::Provided),
10050            "the registered provider must satisfy the requirement first"
10051        );
10052        if daemon_shutdown_begun {
10053            daemon_shutdown.set_for_test();
10054        }
10055
10056        let capture = EventCapture::default();
10057        let _subscriber =
10058            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10059        handler
10060            .cleanup_connection(provider_conn)
10061            .expect("provider deregisters");
10062        let events = capture
10063            .events()
10064            .into_iter()
10065            .filter(|event| {
10066                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10067            })
10068            .collect();
10069        (
10070            events,
10071            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10072        )
10073    }
10074
10075    #[test]
10076    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10077        let (events, verdict) = deregister_required_provider(true);
10078        assert_eq!(
10079            verdict,
10080            Some(CapabilityVerdict::NeverProvided),
10081            "shutdown must not change the verdict route.open refusals read"
10082        );
10083        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10084        let event = &events[0];
10085        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10086        assert_eq!(
10087            event.fields.get("cause").map(String::as_str),
10088            Some("\"daemon_shutdown\""),
10089            "{event:?}"
10090        );
10091        assert_eq!(
10092            event.fields.get("verdict").map(String::as_str),
10093            Some("\"never_provided\""),
10094            "{event:?}"
10095        );
10096    }
10097
10098    #[test]
10099    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10100        let (events, verdict) = deregister_required_provider(false);
10101        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10102        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10103        let event = &events[0];
10104        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10105        assert_eq!(event.fields.get("cause"), None, "{event:?}");
10106        assert_eq!(
10107            event.fields.get("verdict").map(String::as_str),
10108            Some("\"never_provided\""),
10109            "{event:?}"
10110        );
10111    }
10112
10113    #[test]
10114    fn catalog_list_omits_capabilities_for_legacy_manifest() {
10115        let registry = Arc::new(Registry::default());
10116        let handler = ControlHandler::new(Arc::clone(&registry));
10117        let hello = handler
10118            .handle_control(
10119                ConnectionId::new(101),
10120                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10121            )
10122            .expect("legacy HELLO registers");
10123        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10124
10125        let request = Frame::build(
10126            FrameType::Request,
10127            control_flags(),
10128            0,
10129            0,
10130            102,
10131            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10132                .expect("catalog request serializes"),
10133        )
10134        .expect("catalog request frame builds");
10135        let response = handler
10136            .handle_catalog_list(request, None)
10137            .expect("catalog list succeeds");
10138        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10139        assert!(
10140            body["modules"][0].get("capabilities").is_none(),
10141            "legacy manifest must retain an absent capabilities field on catalog.list"
10142        );
10143    }
10144
10145    #[test]
10146    fn hello_ack_omits_storage_when_no_storage_config() {
10147        let registry = Arc::new(Registry::default());
10148        let handler = ControlHandler::new(Arc::clone(&registry));
10149        let responses = handler
10150            .handle_control(
10151                ConnectionId::new(1),
10152                hello_frame("aft", PROTOCOL_VERSION, 7),
10153            )
10154            .unwrap();
10155        let ack = parse_ack(&responses[0]);
10156        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10157        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10158    }
10159
10160    #[tokio::test]
10161    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10162        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10163        let registry = Arc::new(Registry::default());
10164        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
10165        let responses = handler
10166            .handle_control(
10167                ConnectionId::new(1),
10168                hello_frame("aft", PROTOCOL_VERSION, 7),
10169            )
10170            .unwrap();
10171        let ack = parse_ack(&responses[0]);
10172        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10173
10174        let described = handler
10175            .handle_control_frame(
10176                &route_ctx(ConnectionId::new(2)).0,
10177                Frame::build(
10178                    FrameType::Request,
10179                    control_flags(),
10180                    0,
10181                    0,
10182                    9,
10183                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10184                )
10185                .unwrap(),
10186            )
10187            .await
10188            .unwrap();
10189        let ClientControlResponse::ServerDescribe { machine_id, .. } =
10190            serde_json::from_slice(&described[0].body).unwrap()
10191        else {
10192            panic!("server.describe answered with another shape");
10193        };
10194        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10195    }
10196
10197    #[test]
10198    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10199        // With a central sqlite storage policy, each registering module gets its
10200        // own resolved descriptor in HELLO_ACK, keyed by its module id.
10201        let registry = Arc::new(Registry::default());
10202        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
10203            crate::daemon_config::StorageConfig::Sqlite {
10204                data_home: std::path::PathBuf::from("/data"),
10205            },
10206        ));
10207
10208        let responses = handler
10209            .handle_control(
10210                ConnectionId::new(1),
10211                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10212            )
10213            .unwrap();
10214        let ack = parse_ack(&responses[0]);
10215        assert_eq!(
10216            ack.storage,
10217            Some(serde_json::json!({
10218                "module_id": "alfonso-routing",
10219                "storage_namespace": "default",
10220                "isolation": { "kind": "module" },
10221                "backend": {
10222                    "backend": "sqlite",
10223                    "path": "/data/cortexkit/alfonso-routing/store.db"
10224                }
10225            })),
10226            "the delivered descriptor is the module's own sqlite store path"
10227        );
10228    }
10229
10230    #[test]
10231    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10232        let registry = Arc::new(Registry::default());
10233        let handler = ControlHandler::new(Arc::clone(&registry));
10234        let conn = ConnectionId::new(1);
10235        let responses = handler
10236            .handle_control(
10237                conn,
10238                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10239            )
10240            .unwrap();
10241        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10242        let registration = registry.get_module("aft").unwrap().unwrap();
10243        assert_eq!(registration.control_ops, module_baseline_control_ops());
10244
10245        let frame =
10246            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10247        assert!(handler
10248            .guard_module_control_op(&frame, "aft", "route.bind")
10249            .unwrap()
10250            .is_none());
10251        let error = handler
10252            .guard_module_control_op(&frame, "aft", "test.synthetic")
10253            .unwrap()
10254            .expect("synthetic ungranted op should be rejected");
10255        assert_eq!(error.header.ty, FrameType::Error);
10256        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10257    }
10258
10259    #[test]
10260    fn hello_control_ops_some_adds_optional_grants() {
10261        let registry = Arc::new(Registry::default());
10262        let handler = ControlHandler::new(Arc::clone(&registry));
10263        handler
10264            .handle_control(
10265                ConnectionId::new(1),
10266                hello_frame_with_control_ops(
10267                    "aft",
10268                    PROTOCOL_VERSION,
10269                    7,
10270                    Some(vec![
10271                        "future.synthetic".to_string(),
10272                        "route.bind".to_string(),
10273                    ]),
10274                ),
10275            )
10276            .unwrap();
10277        let registration = registry.get_module("aft").unwrap().unwrap();
10278        assert_eq!(
10279            registration.control_ops,
10280            vec![
10281                "route.bind".to_string(),
10282                "route.status".to_string(),
10283                "future.synthetic".to_string(),
10284            ]
10285        );
10286        let frame =
10287            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10288        assert!(handler
10289            .guard_module_control_op(&frame, "aft", "future.synthetic")
10290            .unwrap()
10291            .is_none());
10292    }
10293
10294    #[tokio::test]
10295    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10296        let registry = Arc::new(Registry::default());
10297        let forwarding = Arc::new(ForwardingTable::default());
10298        let handler =
10299            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10300        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10301        hello_via_sink(
10302            &handler,
10303            &module_ctx,
10304            &mut module_rx,
10305            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10306        )
10307        .await;
10308
10309        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10310        let responses = handler
10311            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10312            .await
10313            .unwrap();
10314        assert_eq!(responses.len(), 1);
10315        assert_eq!(responses[0].header.ty, FrameType::Error);
10316        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10317        assert!(module_rx.try_recv().is_err());
10318    }
10319
10320    #[tokio::test]
10321    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10322        let registry = Arc::new(Registry::default());
10323        let forwarding = Arc::new(ForwardingTable::default());
10324        let handler =
10325            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10326        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10327        hello_via_sink(
10328            &handler,
10329            &module_ctx,
10330            &mut module_rx,
10331            hello_frame_with_control_ops(
10332                "aft",
10333                PROTOCOL_VERSION,
10334                7,
10335                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10336            ),
10337        )
10338        .await;
10339
10340        let project_root = unique_project_root("demux");
10341        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10342        let route_handler = handler.clone();
10343        let route_task = tokio::spawn(async move {
10344            route_handler
10345                .handle_control_frame(
10346                    &route_client_ctx,
10347                    route_open_frame(100, "aft", project_root),
10348                )
10349                .await
10350                .unwrap()
10351        });
10352        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10353            .await
10354            .unwrap()
10355            .unwrap();
10356        assert!(matches!(
10357            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10358            ModuleControlRequest::RouteBind { .. }
10359        ));
10360
10361        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10362        let health_handler = handler.clone();
10363        let health_task = tokio::spawn(async move {
10364            health_handler
10365                .handle_control_frame(
10366                    &health_client_ctx,
10367                    supervisor_health_probe_frame(101, "aft"),
10368                )
10369                .await
10370                .unwrap()
10371        });
10372        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10373            .await
10374            .unwrap()
10375            .unwrap();
10376        assert_eq!(
10377            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10378            ModuleControlRequest::HealthCheck {}
10379        );
10380
10381        handler
10382            .handle_control_frame(
10383                &module_ctx,
10384                health_response(health_frame.header.corr, HealthStatus::Degraded),
10385            )
10386            .await
10387            .unwrap();
10388        let health_response = health_task.await.unwrap();
10389        assert_eq!(health_response.len(), 1);
10390        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10391            ClientControlResponse::SupervisorHealthProbe {
10392                module_id,
10393                status,
10394                detail,
10395                metrics,
10396            } => {
10397                assert_eq!(module_id, "aft");
10398                assert_eq!(status, HealthStatus::Degraded);
10399                assert_eq!(detail.as_deref(), Some("warming"));
10400                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10401            }
10402            other => panic!("unexpected health response: {other:?}"),
10403        }
10404
10405        handler
10406            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10407            .await
10408            .unwrap();
10409        let route_response = route_task.await.unwrap();
10410        assert!(route_response.is_empty());
10411        let published = route_client_rx.recv().await.unwrap();
10412        assert!(matches!(
10413            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10414            ClientControlResponse::RouteOpen { .. }
10415        ));
10416    }
10417
10418    /// Start one `route.open` on `client_connection` and return its still-running
10419    /// handler task together with the `route.bind` the module received for it.
10420    /// The handler blocks until the module answers, so it has to run as a task
10421    /// while the test drives the module side.
10422    async fn relay_route_open(
10423        handler: &ControlHandler,
10424        client_connection: ConnectionId,
10425        client_egress: &FrameSink,
10426        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10427        corr: u64,
10428        module_id: &str,
10429        project_root_label: &str,
10430    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10431        let ctx = RouteCtx {
10432            connection_id: client_connection,
10433            egress: client_egress.clone(),
10434        };
10435        let handler = handler.clone();
10436        let project_root = unique_project_root(project_root_label);
10437        let module_id = module_id.to_string();
10438        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10439        let task = tokio::spawn(async move {
10440            let _guard = tracing::dispatcher::set_default(&dispatch);
10441            handler
10442                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10443                .await
10444                .unwrap()
10445        });
10446        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10447            .await
10448            .expect("module receives the relayed route.bind")
10449            .expect("module egress is open");
10450        (task, bind.frame)
10451    }
10452
10453    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10454        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10455            ModuleControlRequest::RouteBind {
10456                route_channel,
10457                epoch,
10458                ..
10459            } => (route_channel, epoch),
10460            other => panic!("expected a route.bind request, got {other:?}"),
10461        }
10462    }
10463
10464    fn published_route(frame: &Frame) -> (u16, u32) {
10465        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10466            ClientControlResponse::RouteOpen {
10467                route_channel,
10468                route_epoch,
10469            } => (route_channel, route_epoch),
10470            other => panic!("expected a route.open response, got {other:?}"),
10471        }
10472    }
10473
10474    /// Reproduction of a production outage. A client had `route.open`s in
10475    /// flight to a module and was already marked closing -- its egress had refused a
10476    /// module frame, so the daemon asked its connection to end -- while its sink
10477    /// was still open. When the module acked those binds, the daemon refused to
10478    /// commit a route for a closing client, and that refusal was returned from
10479    /// the MODULE connection's frame handler, where a router error that has no
10480    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
10481    /// the supervisor correctly did not respawn a clean exit, and every seat lost
10482    /// its tools for hours -- one client's teardown took down a connection
10483    /// carrying ~170 other routes.
10484    ///
10485    /// The window is opened here by calling the production path that opens it
10486    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
10487    /// state that matters is "in `closing_connections`, sink still open, relay
10488    /// still pending", and it lasts only from the close request until the
10489    /// connection loop reacts to it; a socket-level test can flood a client into
10490    /// that escalation but cannot pin the module's ack inside the window. Closing
10491    /// the socket instead takes the other path entirely -- connection teardown
10492    /// removes the pending relay under the same lock, so the ack finds nothing.
10493    #[tokio::test]
10494    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10495        let registry = Arc::new(Registry::default());
10496        let forwarding = Arc::new(ForwardingTable::default());
10497        let handler =
10498            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10499
10500        let module_connection = ConnectionId::new(30);
10501        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10502        hello_via_sink(
10503            &handler,
10504            &module_ctx,
10505            &mut module_rx,
10506            hello_frame("aft", PROTOCOL_VERSION, 7),
10507        )
10508        .await;
10509
10510        let dying_client = ConnectionId::new(31);
10511        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10512
10513        // A published route on the dying client. The escalation below only marks
10514        // a connection closing for a route it has already published.
10515        let (first_task, first_bind) = relay_route_open(
10516            &handler,
10517            dying_client,
10518            &dying_ctx.egress,
10519            &mut module_rx,
10520            100,
10521            "aft",
10522            "closing-first",
10523        )
10524        .await;
10525        handler
10526            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10527            .await
10528            .unwrap();
10529        assert!(first_task.await.unwrap().is_empty());
10530        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10531
10532        // A second route.open from the same client, relayed and awaiting its ack.
10533        let (second_task, second_bind) = relay_route_open(
10534            &handler,
10535            dying_client,
10536            &dying_ctx.egress,
10537            &mut module_rx,
10538            101,
10539            "aft",
10540            "closing-second",
10541        )
10542        .await;
10543        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10544
10545        // The window: the client is closing, its sink is still open, and its
10546        // second bind is still pending.
10547        assert!(forwarding
10548            .escalate_client_delivery_failure(
10549                dying_client,
10550                first_channel,
10551                first_epoch,
10552                CloseReason::new(
10553                    "module_to_client_delivery_failed",
10554                    "client egress refused a module frame",
10555                ),
10556                crate::forwarding::UndeliveredFrame {
10557                    module_id: None,
10558                    sink: &dying_ctx.egress,
10559                },
10560            )
10561            .unwrap());
10562        assert!(!dying_ctx.egress.is_closed());
10563
10564        // The frame that used to end the module connection.
10565        let ack = handler
10566            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10567            .await;
10568        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10569        if module_loop_error.is_some() {
10570            // What the server's connection loop does with a router error that has
10571            // no ERROR-frame translation: end the connection, which releases the
10572            // module's registration and every route on it.
10573            handler.cleanup_connection(module_connection).unwrap();
10574        }
10575        // Read the module's next frame before opening the co-tenant's route, so
10576        // the GOODBYE assertion below is about THIS ack and not about later
10577        // traffic. `None` means the module was told nothing.
10578        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10579            .await
10580            .ok()
10581            .flatten();
10582
10583        // 1. The module connection is still registered.
10584        assert!(
10585            registry
10586                .get_module_by_connection(module_connection)
10587                .unwrap()
10588                .is_some(),
10589            "one client's closing connection ended the shared module connection: \
10590             {module_loop_error:?}"
10591        );
10592        // ...and still serving: another client can open and use a route on it.
10593        let cotenant = ConnectionId::new(32);
10594        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10595        let (cotenant_task, cotenant_bind) = relay_route_open(
10596            &handler,
10597            cotenant,
10598            &cotenant_ctx.egress,
10599            &mut module_rx,
10600            102,
10601            "aft",
10602            "closing-cotenant",
10603        )
10604        .await;
10605        handler
10606            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10607            .await
10608            .unwrap();
10609        assert!(cotenant_task.await.unwrap().is_empty());
10610        let (cotenant_channel, cotenant_epoch) =
10611            published_route(&cotenant_rx.recv().await.unwrap());
10612        assert!(matches!(
10613            forwarding
10614                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10615                .unwrap(),
10616            DataRoute::Client(DataRouteState::Bound(_))
10617        ));
10618
10619        // 2. The module was told to drop the binding it created for the route
10620        //    that will never be published.
10621        let goodbye = post_ack_module_frame
10622            .expect("module receives a GOODBYE for the abandoned route channel");
10623        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10624        assert_eq!(goodbye.header.channel, abandoned_channel);
10625        assert_eq!(goodbye.header.epoch, abandoned_epoch);
10626
10627        // 3. The dying client received nothing: no route was ever published to
10628        //    it. Its route.open is answered as unavailable, which the connection
10629        //    loop would write to a socket that is already going away.
10630        assert!(dying_rx.try_recv().is_err());
10631        let second_response = second_task.await.unwrap();
10632        assert_eq!(second_response.len(), 1);
10633        assert_eq!(
10634            parse_error(&second_response[0])["code"],
10635            "target_unavailable"
10636        );
10637    }
10638
10639    /// One `catalog.list` as a client sees it: the generation and the listed
10640    /// module ids with their readiness.
10641    async fn catalog_snapshot(
10642        handler: &ControlHandler,
10643        client_ctx: &RouteCtx,
10644        corr: u64,
10645    ) -> (u64, Vec<(String, bool)>) {
10646        let body =
10647            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10648        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10649        let reply = handler
10650            .handle_control_frame(client_ctx, request)
10651            .await
10652            .unwrap()
10653            .pop()
10654            .unwrap();
10655        let ClientControlResponse::CatalogList {
10656            generation,
10657            modules,
10658            ..
10659        } = serde_json::from_slice(&reply.body).unwrap()
10660        else {
10661            panic!("not a catalog.list reply");
10662        };
10663        (
10664            generation,
10665            modules
10666                .into_iter()
10667                .map(|entry| (entry.module_id, entry.ready))
10668                .collect(),
10669        )
10670    }
10671
10672    /// A handler whose HELLO for module "aft" on connection 40 is parked
10673    /// between reserving its registry entry and installing its forwarding
10674    /// endpoint. Returns the handler, the two-party barrier (wait once to know
10675    /// the HELLO is parked, once more to release it), the HELLO's thread, and
10676    /// the module's context and receiver.
10677    fn hello_held_before_forwarding() -> (
10678        ControlHandler,
10679        Arc<std::sync::Barrier>,
10680        std::thread::JoinHandle<Vec<Frame>>,
10681        RouteCtx,
10682        mpsc::Receiver<crate::router::OutboundFrame>,
10683    ) {
10684        let hold = Arc::new(std::sync::Barrier::new(2));
10685        let handler = ControlHandler::with_forwarding(
10686            Arc::new(Registry::default()),
10687            Arc::new(ForwardingTable::default()),
10688        )
10689        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10690        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10691        let hello = {
10692            let handler = handler.clone();
10693            let module_ctx = module_ctx.clone();
10694            std::thread::spawn(move || {
10695                handler
10696                    .handle_hello(
10697                        module_ctx.connection_id,
10698                        Some(module_ctx.egress.clone()),
10699                        hello_frame("aft", PROTOCOL_VERSION, 7),
10700                    )
10701                    .unwrap()
10702            })
10703        };
10704        (handler, hold, hello, module_ctx, module_rx)
10705    }
10706
10707    /// While a HELLO has reserved its registry entry but its forwarding
10708    /// endpoint is not installed yet, `route.open` to the module can only be
10709    /// refused. `catalog.list` must not list the module in that window: a
10710    /// client that opens a route as soon as it sees a module listed and ready
10711    /// would be refused for a module the daemon had just advertised. The
10712    /// refusal in the window stays the retryable `target_unavailable`, never
10713    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10714    /// Once the HELLO completes, the module is listed and a route to it opens.
10715    #[tokio::test]
10716    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10717        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10718        let client = ConnectionId::new(41);
10719        let (client_ctx, mut client_rx) = route_ctx(client);
10720
10721        hold.wait();
10722        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10723        assert!(handler.registry.get_module("aft").unwrap().is_some());
10724        assert!(!handler
10725            .forwarding
10726            .has_live_module_connection("aft")
10727            .unwrap());
10728        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10729        assert!(
10730            listed.is_empty(),
10731            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10732        );
10733        let refused = handler
10734            .handle_control_frame(
10735                &client_ctx,
10736                route_open_frame(301, "aft", unique_project_root("listed-window")),
10737            )
10738            .await
10739            .unwrap();
10740        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10741        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10742        let duplicate = handler
10743            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10744            .await
10745            .unwrap();
10746        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10747
10748        hold.wait();
10749        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10750        let ack = module_rx.try_recv().unwrap().frame;
10751        assert_eq!(ack.header.ty, FrameType::HelloAck);
10752        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10753        assert_eq!(listed, vec![("aft".to_string(), true)]);
10754
10755        let (open_task, bind) = relay_route_open(
10756            &handler,
10757            client,
10758            &client_ctx.egress,
10759            &mut module_rx,
10760            303,
10761            "aft",
10762            "listed-after",
10763        )
10764        .await;
10765        assert!(handler
10766            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10767            .await
10768            .unwrap()
10769            .is_empty());
10770        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10771        published_route(&client_rx.try_recv().unwrap().frame);
10772    }
10773
10774    /// A catalog watcher keys on the generation: when it changes, it re-reads
10775    /// the listing. So the generation must advance exactly when the module
10776    /// appears. If it advanced while the module was still hidden, a watcher
10777    /// would read the new generation without the module, and the module would
10778    /// later appear under that same generation with nothing telling the
10779    /// watcher to look again.
10780    #[tokio::test]
10781    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10782        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10783        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10784        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10785
10786        hold.wait();
10787        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10788        assert!(listed.is_empty());
10789        assert_eq!(
10790            during, before,
10791            "the generation moved while the module was not listed"
10792        );
10793
10794        hold.wait();
10795        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10796        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10797        assert_eq!(listed, vec![("aft".to_string(), true)]);
10798        assert_eq!(
10799            after,
10800            before + 1,
10801            "the module appeared without exactly one generation bump"
10802        );
10803    }
10804
10805    /// A HELLO whose forwarding endpoint cannot be installed (here the
10806    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
10807    /// refused and leaves nothing behind: no registry entry, no catalog entry,
10808    /// and the catalog generation where it was.
10809    #[tokio::test]
10810    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10811        let registry = Arc::new(Registry::default());
10812        let handler = ControlHandler::with_forwarding(
10813            Arc::clone(&registry),
10814            Arc::new(ForwardingTable::default()),
10815        );
10816        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10817        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10818        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10819        drop(module_rx);
10820
10821        let replies = handler
10822            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10823            .await
10824            .unwrap();
10825        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10826        assert!(registry.get_module("aft").unwrap().is_none());
10827        assert!(registry
10828            .get_module_by_connection(module_ctx.connection_id)
10829            .unwrap()
10830            .is_none());
10831        assert_eq!(
10832            catalog_snapshot(&handler, &client_ctx, 301).await,
10833            (before, Vec::new())
10834        );
10835    }
10836
10837    /// A handler whose `route.open` parks after queueing the route.bind and
10838    /// before marking it enqueued, with module "aft" registered on connection
10839    /// 40. Returns the two-party barrier that parks and releases it.
10840    async fn route_open_held_after_bind_send() -> (
10841        ControlHandler,
10842        Arc<tokio::sync::Barrier>,
10843        RouteCtx,
10844        mpsc::Receiver<crate::router::OutboundFrame>,
10845    ) {
10846        let hold = Arc::new(tokio::sync::Barrier::new(2));
10847        let handler = ControlHandler::with_forwarding(
10848            Arc::new(Registry::default()),
10849            Arc::new(ForwardingTable::default()),
10850        )
10851        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10852        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10853        hello_via_sink(
10854            &handler,
10855            &module_ctx,
10856            &mut module_rx,
10857            hello_frame("aft", PROTOCOL_VERSION, 7),
10858        )
10859        .await;
10860        (handler, hold, module_ctx, module_rx)
10861    }
10862
10863    /// A fast module acks the route.bind, and the reader commits the route,
10864    /// before `route.open` has marked the relay enqueued. The daemon used to
10865    /// read the missing relay as an abandoned bind and send the module a
10866    /// GOODBYE for the route it had just committed: the client was told its
10867    /// route was open, the module unbound the channel, and the client's first
10868    /// request was dropped on the module side as unbound.
10869    #[tokio::test]
10870    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10871        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10872        let client = ConnectionId::new(41);
10873        let (client_ctx, mut client_rx) = route_ctx(client);
10874
10875        let (open_task, bind) = relay_route_open(
10876            &handler,
10877            client,
10878            &client_ctx.egress,
10879            &mut module_rx,
10880            200,
10881            "aft",
10882            "acked-before-mark",
10883        )
10884        .await;
10885        let (module_channel, module_epoch) = route_bind_channel(&bind);
10886        // route.open is now parked between queueing the bind and marking it.
10887        hold.wait().await;
10888
10889        // The module answers inside that window, and the route commits.
10890        assert!(handler
10891            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10892            .await
10893            .unwrap()
10894            .is_empty());
10895        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10896
10897        hold.wait().await;
10898        assert!(
10899            open_task.await.unwrap().is_empty(),
10900            "route.open answers accepted"
10901        );
10902
10903        // Every frame route.open could have sent is already queued, because it
10904        // has returned; the module must have been sent nothing.
10905        match module_rx.try_recv() {
10906            Err(mpsc::error::TryRecvError::Empty) => {}
10907            Ok(outbound) => panic!(
10908                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10909                outbound.frame.header.ty,
10910                outbound.frame.header.channel,
10911                outbound.frame.header.epoch
10912            ),
10913            Err(err) => panic!("module egress closed: {err}"),
10914        }
10915
10916        // The client's first request on the route is the next thing the module
10917        // reads, on the binding it just created.
10918        let router = Router::with_control_handler(Arc::new(handler.clone()));
10919        let request = Frame::build(
10920            FrameType::Request,
10921            Flags::new(true, Priority::Interactive, false),
10922            client_channel,
10923            client_epoch,
10924            201,
10925            b"first request".to_vec(),
10926        )
10927        .unwrap();
10928        router
10929            .route_for_connection(&client_ctx, request)
10930            .await
10931            .unwrap();
10932        let delivered = module_rx.try_recv().unwrap().frame;
10933        assert_eq!(delivered.header.ty, FrameType::Request);
10934        assert_eq!(
10935            (delivered.header.channel, delivered.header.epoch),
10936            (module_channel, module_epoch)
10937        );
10938        assert_eq!(delivered.body, b"first request");
10939    }
10940
10941    /// The client goes away while `route.open` is still queueing the bind, so
10942    /// the relay is dropped with no module reply. The teardown cannot send the
10943    /// GOODBYE itself (it could reach the module ahead of the bind), so
10944    /// `route.open` must send it once the bind is queued; otherwise the module
10945    /// keeps a binding the daemon will never commit.
10946    #[tokio::test]
10947    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10948        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10949        let client = ConnectionId::new(42);
10950        let (client_ctx, _client_rx) = route_ctx(client);
10951
10952        let (open_task, bind) = relay_route_open(
10953            &handler,
10954            client,
10955            &client_ctx.egress,
10956            &mut module_rx,
10957            300,
10958            "aft",
10959            "released-before-mark",
10960        )
10961        .await;
10962        let (module_channel, module_epoch) = route_bind_channel(&bind);
10963        hold.wait().await;
10964
10965        handler.cleanup_connection(client).unwrap();
10966        assert!(
10967            module_rx.try_recv().is_err(),
10968            "client teardown must not GOODBYE a bind route.open is still queueing"
10969        );
10970
10971        hold.wait().await;
10972        let response = open_task.await.unwrap();
10973        assert_eq!(response.len(), 1);
10974        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10975
10976        let goodbye = module_rx
10977            .try_recv()
10978            .expect("module is told to drop the binding the daemon abandoned")
10979            .frame;
10980        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10981        assert_eq!(
10982            (goodbye.header.channel, goodbye.header.epoch),
10983            (module_channel, module_epoch)
10984        );
10985        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10986    }
10987
10988    /// The module acks inside the same window, but the client's egress closed
10989    /// first, so the reader refuses to commit. The ack proves the module holds
10990    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
10991    /// send a second one.
10992    #[tokio::test]
10993    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10994        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10995        let client = ConnectionId::new(43);
10996        let (client_ctx, client_rx) = route_ctx(client);
10997
10998        let (open_task, bind) = relay_route_open(
10999            &handler,
11000            client,
11001            &client_ctx.egress,
11002            &mut module_rx,
11003            400,
11004            "aft",
11005            "gone-client-before-mark",
11006        )
11007        .await;
11008        let (module_channel, module_epoch) = route_bind_channel(&bind);
11009        hold.wait().await;
11010
11011        drop(client_rx);
11012        assert!(client_ctx.egress.is_closed());
11013        handler
11014            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11015            .await
11016            .unwrap();
11017
11018        hold.wait().await;
11019        let response = open_task.await.unwrap();
11020        assert_eq!(response.len(), 1);
11021        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11022
11023        let goodbye = module_rx
11024            .try_recv()
11025            .expect("module is told to drop the binding it acked")
11026            .frame;
11027        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11028        assert_eq!(
11029            (goodbye.header.channel, goodbye.header.epoch),
11030            (module_channel, module_epoch)
11031        );
11032        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11033        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11034    }
11035
11036    /// The fence at the module-loop boundary, stated as its own contract: which
11037    /// forwarding failures are allowed to end the module connection that is being
11038    /// served. A `ConnectionClosing` naming some client is about that client, and
11039    /// a module connection is shared; the same error naming the module's own
11040    /// connection is about this connection and must stay fatal, as must failures
11041    /// that are about the forwarding table itself.
11042    #[test]
11043    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11044        let handler = ControlHandler::default();
11045        let module_connection = ConnectionId::new(30);
11046        let client_connection = ConnectionId::new(31);
11047
11048        handler
11049            .refuse_to_end_module_connection_for_a_client(
11050                module_connection,
11051                77,
11052                ForwardingError::ConnectionClosing {
11053                    connection_id: client_connection,
11054                },
11055            )
11056            .expect("a closing client must never end the module connection");
11057
11058        assert!(matches!(
11059            handler.refuse_to_end_module_connection_for_a_client(
11060                module_connection,
11061                78,
11062                ForwardingError::ConnectionClosing {
11063                    connection_id: module_connection,
11064                },
11065            ),
11066            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11067                connection_id
11068            })) if connection_id == module_connection
11069        ));
11070        assert!(matches!(
11071            handler.refuse_to_end_module_connection_for_a_client(
11072                module_connection,
11073                79,
11074                ForwardingError::Poisoned,
11075            ),
11076            Err(RouterError::Forwarding(ForwardingError::Poisoned))
11077        ));
11078        assert!(matches!(
11079            handler.refuse_to_end_module_connection_for_a_client(
11080                module_connection,
11081                80,
11082                ForwardingError::StaleModuleEndpoint,
11083            ),
11084            Err(RouterError::Forwarding(
11085                ForwardingError::StaleModuleEndpoint
11086            ))
11087        ));
11088    }
11089
11090    /// The spawn-attestation guard is what stops a connected module from claiming
11091    /// another module's identity and being stamped `Reserved` for it. Every other
11092    /// test that supplies a consumer_identity supplies a CORRECT one, because a
11093    /// correct one is what the rest of the flow needs -- so the guard's rejection
11094    /// branch was never the subject of an assertion, only its acceptance branch.
11095    ///
11096    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
11097    /// whole subc-core library suite green; only the forwarding integration tests
11098    /// notice, and they notice for unrelated reasons. This test exists so the
11099    /// refusal itself is asserted where the guard lives: it fails if the identity
11100    /// check stops refusing, which is the direction that matters, since a guard
11101    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
11102    #[tokio::test]
11103    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11104        let registry = Arc::new(Registry::default());
11105        let forwarding = Arc::new(ForwardingTable::default());
11106        let supervisor = SupervisorHandle::new();
11107        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11108        let handler =
11109            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11110                .with_supervisor(supervisor);
11111
11112        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11113        hello_via_sink(
11114            &handler,
11115            &target_ctx,
11116            &mut target_rx,
11117            hello_frame("target", PROTOCOL_VERSION, 1),
11118        )
11119        .await;
11120
11121        // A real supervised module id presenting the wrong nonce. This is the
11122        // impersonation case: the attacker knows a privileged module_id, which is
11123        // public, and guesses at the nonce, which is not.
11124        let wrong_nonce = handler
11125            .handle_control_frame(
11126                &route_ctx(ConnectionId::new(91)).0,
11127                route_open_frame_with_admission_facts(
11128                    20,
11129                    "target",
11130                    unique_project_root("admission-facts"),
11131                    Some(subc_control::ConsumerIdentity {
11132                        module_id: "fed".to_string(),
11133                        launch_nonce: "not-the-real-nonce".to_string(),
11134                    }),
11135                    None,
11136                ),
11137            )
11138            .await
11139            .unwrap();
11140        assert_eq!(
11141            parse_error(&wrong_nonce[0])["code"],
11142            "bad_consumer_identity",
11143            "a mismatched launch nonce must be refused, not stamped Reserved"
11144        );
11145
11146        // A module id the supervisor never spawned at all, so no nonce exists to
11147        // compare against. An implementation that treats "no record" as "nothing
11148        // to check" fails open here while passing the case above.
11149        let never_spawned = handler
11150            .handle_control_frame(
11151                &route_ctx(ConnectionId::new(92)).0,
11152                route_open_frame_with_admission_facts(
11153                    21,
11154                    "target",
11155                    unique_project_root("admission-facts"),
11156                    Some(subc_control::ConsumerIdentity {
11157                        module_id: "never-spawned".to_string(),
11158                        launch_nonce: "any-nonce".to_string(),
11159                    }),
11160                    None,
11161                ),
11162            )
11163            .await
11164            .unwrap();
11165        assert_eq!(
11166            parse_error(&never_spawned[0])["code"],
11167            "bad_consumer_identity",
11168            "an unspawned module_id must be refused rather than accepted for lack of a record"
11169        );
11170    }
11171
11172    /// The refusal test above proves the guard says NO. Nothing proved it can say
11173    /// YES, and the difference is not academic: replacing the whole authorization
11174    /// with `false` -- admitting no consumer identity at all, revoking Reserved
11175    /// standing for every supervised module in the fleet -- leaves 110 of the 111
11176    /// library tests GREEN. The one that notices does so by HANGING, because it
11177    /// waits for a bind that can no longer happen.
11178    ///
11179    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
11180    /// or flaky test, invites a RETRY rather than an investigation, and the retry
11181    /// hangs too and gets blamed on the runner. So a total revocation of the
11182    /// daemon's trust grant would have shipped behind a symptom nobody attributes
11183    /// to code.
11184    ///
11185    /// The bias is structural rather than accidental. A REFUSAL looks like a
11186    /// failure someone writes a test for; a GRANT looks like the happy path. Every
11187    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
11188    /// suite for that reason, and this one is the purest case in the daemon.
11189    ///
11190    /// This test asserts the EFFECT rather than the absence of an error: the module
11191    /// receives a RouteBind and it carries `Reserved` naming the attested module.
11192    /// A guard that admitted nobody would produce no bind at all; one that admitted
11193    /// everybody would stamp the wrong principal, which the refusal test catches.
11194    #[tokio::test]
11195    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11196        let registry = Arc::new(Registry::default());
11197        let forwarding = Arc::new(ForwardingTable::default());
11198        let supervisor = SupervisorHandle::new();
11199        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11200        let handler =
11201            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11202                .with_supervisor(supervisor);
11203
11204        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11205        hello_via_sink(
11206            &handler,
11207            &target_ctx,
11208            &mut target_rx,
11209            hello_frame("target", PROTOCOL_VERSION, 1),
11210        )
11211        .await;
11212
11213        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11214        let route_handler = handler.clone();
11215        let route_task = tokio::spawn(async move {
11216            route_handler
11217                .handle_control_frame(
11218                    &client_ctx,
11219                    route_open_frame_with_admission_facts(
11220                        30,
11221                        "target",
11222                        unique_project_root("admission-facts"),
11223                        Some(subc_control::ConsumerIdentity {
11224                            module_id: "fed".to_string(),
11225                            launch_nonce: "fed-nonce".to_string(),
11226                        }),
11227                        None,
11228                    ),
11229                )
11230                .await
11231                .unwrap()
11232        });
11233
11234        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
11235        // the very mutation it exists to catch -- a guard that admits nobody -- no
11236        // bind is ever sent, so it HUNG rather than failing. That reproduces the
11237        // exact defect being fixed: a total revocation detected only as a stalled
11238        // suite, which reads as flakiness and invites a retry. An acceptance test
11239        // that waits for an effect must bound the wait, or a red becomes a hang.
11240        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11241            .await
11242            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11243            .expect("module control channel closed before route.bind");
11244        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11245        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11246            panic!("expected route.bind")
11247        };
11248        assert_eq!(
11249            principal,
11250            Some(Principal::Reserved {
11251                module_id: "fed".to_string()
11252            }),
11253            "a correctly attested consumer must be stamped Reserved for its own id"
11254        );
11255
11256        handler
11257            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11258            .await
11259            .unwrap();
11260        assert!(route_task.await.unwrap().is_empty());
11261        assert!(
11262            matches!(
11263                serde_json::from_slice::<ClientControlResponse>(
11264                    &client_rx.recv().await.unwrap().body
11265                )
11266                .unwrap(),
11267                ClientControlResponse::RouteOpen { .. }
11268            ),
11269            "the route must actually open, not merely avoid an error"
11270        );
11271    }
11272
11273    #[tokio::test(start_paused = true)]
11274    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11275        let registry = Arc::new(Registry::default());
11276        let forwarding = Arc::new(ForwardingTable::default());
11277        let supervisor = SupervisorHandle::new();
11278        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11279        let handler =
11280            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11281                .with_supervisor(supervisor);
11282
11283        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11284        hello_via_sink(
11285            &handler,
11286            &target_ctx,
11287            &mut target_rx,
11288            hello_frame("target", PROTOCOL_VERSION, 1),
11289        )
11290        .await;
11291
11292        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11293        let direct_handler = handler.clone();
11294        let direct_open = tokio::spawn(async move {
11295            direct_handler
11296                .handle_control_frame(
11297                    &direct_ctx,
11298                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
11299                )
11300                .await
11301                .unwrap()
11302        });
11303        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11304            .await
11305            .expect("no direct route.bind within 5s")
11306            .expect("target control channel closed before direct route.bind");
11307        handler
11308            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11309            .await
11310            .unwrap();
11311        assert!(direct_open.await.unwrap().is_empty());
11312        let _ = direct_rx.recv().await.unwrap();
11313
11314        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11315        let reserved_handler = handler.clone();
11316        let reserved_open = tokio::spawn(async move {
11317            reserved_handler
11318                .handle_control_frame(
11319                    &reserved_ctx,
11320                    route_open_frame_with_admission_facts(
11321                        3,
11322                        "target",
11323                        unique_project_root("admission-facts"),
11324                        Some(ConsumerIdentity {
11325                            module_id: "fed".to_string(),
11326                            launch_nonce: "fed-nonce".to_string(),
11327                        }),
11328                        None,
11329                    ),
11330                )
11331                .await
11332                .unwrap()
11333        });
11334        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11335            .await
11336            .expect("no reserved route.bind within 5s")
11337            .expect("target control channel closed before reserved route.bind");
11338        handler
11339            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11340            .await
11341            .unwrap();
11342        assert!(reserved_open.await.unwrap().is_empty());
11343        let _ = reserved_rx.recv().await.unwrap();
11344
11345        forwarding
11346            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11347            .unwrap();
11348        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11349        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11350            module_id: Some("target".to_string()),
11351        })
11352        .unwrap();
11353        let census_frame =
11354            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11355        let response = handler
11356            .handle_control_frame(&census_ctx, census_frame)
11357            .await
11358            .unwrap()
11359            .pop()
11360            .unwrap();
11361        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11362        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11363        assert!(matches!(
11364            decoded,
11365            ClientControlResponse::SupervisorRoutes { .. }
11366        ));
11367        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11368        assert_eq!(routes.len(), 2);
11369        assert!(routes.iter().all(|route| route["draining"] == true));
11370        // The census carries WHY: the reason the drain was begun with, in the
11371        // route.closing vocabulary, on every draining route this drain marked.
11372        assert!(
11373            routes.iter().all(|route| route["drain_reason"] == "reload"),
11374            "draining routes must name the drain's reason: {routes:?}"
11375        );
11376        assert!(routes.iter().any(|route| {
11377            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11378        }));
11379        assert!(routes.iter().any(|route| {
11380            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11381        }));
11382
11383        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11384            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11385        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11386            std::fs::write(
11387                &golden_path,
11388                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11389            )
11390            .unwrap();
11391        }
11392        let expected: Value =
11393            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11394        assert_eq!(actual, expected);
11395    }
11396
11397    async fn query_live_roots(
11398        handler: &ControlHandler,
11399        module_ctx: &RouteCtx,
11400    ) -> ModuleControlResponseToModule {
11401        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11402        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11403        let response = handler
11404            .handle_control_frame(module_ctx, frame)
11405            .await
11406            .unwrap()
11407            .pop()
11408            .unwrap();
11409        serde_json::from_slice(&response.body).unwrap()
11410    }
11411
11412    #[tokio::test(start_paused = true)]
11413    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11414        let registry = Arc::new(Registry::default());
11415        let forwarding = Arc::new(ForwardingTable::default());
11416        let handler = ControlHandler::with_forwarding(registry, forwarding);
11417        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11418        hello_via_sink(
11419            &handler,
11420            &target_ctx,
11421            &mut target_rx,
11422            hello_frame("target", PROTOCOL_VERSION, 1),
11423        )
11424        .await;
11425        let root = unique_project_root("live-roots-known");
11426        let path = ProjectRootId::from_path_allowing_missing(root.path())
11427            .unwrap()
11428            .as_path()
11429            .to_path_buf();
11430        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11431        let open_handler = handler.clone();
11432        let opened = tokio::spawn(async move {
11433            open_handler
11434                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11435                .await
11436                .unwrap()
11437        });
11438        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11439            .await
11440            .unwrap()
11441            .unwrap();
11442        handler
11443            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11444            .await
11445            .unwrap();
11446        assert!(opened.await.unwrap().is_empty());
11447        let _ = client_rx.recv().await.unwrap();
11448
11449        let root = unique_project_root("live-roots-pending");
11450        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11451            .unwrap()
11452            .as_path()
11453            .to_path_buf();
11454        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11455        let open_handler = handler.clone();
11456        let pending = tokio::spawn(async move {
11457            open_handler
11458                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11459                .await
11460                .unwrap()
11461        });
11462        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11463            .await
11464            .unwrap()
11465            .unwrap();
11466        let actual = query_live_roots(&handler, &target_ctx).await;
11467        let ModuleControlResponseToModule::LiveRoots {
11468            roots,
11469            unknown_root_bindings,
11470            total_bindings,
11471        } = actual
11472        else {
11473            panic!("expected live roots")
11474        };
11475        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11476        assert_eq!(unknown_root_bindings, 0);
11477        assert_eq!(
11478            roots.len(),
11479            2,
11480            "root-known arm must retain each canonical root"
11481        );
11482        assert_eq!(
11483            total_bindings,
11484            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11485        );
11486        let counts = roots
11487            .iter()
11488            .map(|root| (root.project_root.clone(), root.bound, root.pending))
11489            .collect::<Vec<_>>();
11490        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11491        expected.sort_by(|a, b| a.0.cmp(&b.0));
11492        assert_eq!(
11493            counts, expected,
11494            "roots must sort by path and count pending separately"
11495        );
11496        handler
11497            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11498            .await
11499            .unwrap();
11500        assert!(pending.await.unwrap().is_empty());
11501    }
11502
11503    #[tokio::test(start_paused = true)]
11504    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11505        let forwarding = Arc::new(ForwardingTable::default());
11506        let handler =
11507            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11508        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11509        hello_via_sink(
11510            &handler,
11511            &target_ctx,
11512            &mut target_rx,
11513            hello_frame("target", PROTOCOL_VERSION, 1),
11514        )
11515        .await;
11516        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11517        let pending = forwarding
11518            .begin_route_bind_relay_for_test(
11519                client_ctx.connection_id,
11520                client_ctx.egress.clone(),
11521                2,
11522                "target",
11523            )
11524            .unwrap();
11525        forwarding
11526            .complete_pending_relay(
11527                target_ctx.connection_id,
11528                pending.corr,
11529                RouteBindRelayOutcome::Accepted,
11530            )
11531            .unwrap();
11532        let actual = query_live_roots(&handler, &target_ctx).await;
11533        let ModuleControlResponseToModule::LiveRoots {
11534            roots,
11535            unknown_root_bindings,
11536            total_bindings,
11537        } = actual
11538        else {
11539            panic!("expected live roots")
11540        };
11541        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11542        assert_eq!(
11543            unknown_root_bindings, 1,
11544            "unknown-root arm must not read as no bindings"
11545        );
11546        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11547        assert_eq!(
11548            total_bindings,
11549            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11550        );
11551    }
11552
11553    /// A module reads HELLO_ACK as its first frame and exits on anything else,
11554    /// so the ack has to be on its outbound queue before the module is
11555    /// routable. The connection loop writes a handler's replies only after the
11556    /// handler returns; this test stops in exactly that gap, runs a real
11557    /// route.open from another connection, and only then writes whatever the
11558    /// HELLO handler returned, the way the loop would. If the ack were still a
11559    /// reply, the route.bind request would reach the module first.
11560    #[tokio::test(start_paused = true)]
11561    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11562        let forwarding = Arc::new(ForwardingTable::default());
11563        let handler =
11564            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11565        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11566        let replies = handler
11567            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11568            .await
11569            .unwrap();
11570        let queued_by_hello = module_rx.len();
11571
11572        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11573        let open_handler = handler.clone();
11574        let open = tokio::spawn(async move {
11575            open_handler
11576                .handle_control_frame(
11577                    &client_ctx,
11578                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11579                )
11580                .await
11581                .unwrap()
11582        });
11583        // Let the route.open run until its route.bind is on the module's queue.
11584        let mut spins = 0;
11585        while module_rx.len() == queued_by_hello {
11586            spins += 1;
11587            assert!(spins < 10_000, "route.open never queued a route.bind");
11588            tokio::task::yield_now().await;
11589        }
11590
11591        // Now the connection loop's half: write the HELLO handler's replies.
11592        for reply in replies {
11593            module_ctx.egress.send(reply).await.unwrap();
11594        }
11595
11596        let first = module_rx.recv().await.unwrap().frame;
11597        assert_eq!(
11598            first.header.ty,
11599            FrameType::HelloAck,
11600            "the first frame a registering module reads must be its HELLO_ACK"
11601        );
11602        assert_eq!(first.header.corr, 7);
11603        let second = module_rx.recv().await.unwrap().frame;
11604        assert_eq!(second.header.ty, FrameType::Request);
11605        assert!(
11606            matches!(
11607                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11608                ModuleControlRequest::RouteBind { .. }
11609            ),
11610            "the route.bind follows the ack"
11611        );
11612        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11613
11614        handler
11615            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11616            .await
11617            .unwrap();
11618        assert!(open.await.unwrap().is_empty());
11619        let _ = client_rx.recv().await.unwrap();
11620    }
11621
11622    #[tokio::test(start_paused = true)]
11623    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11624        let handler = ControlHandler::with_forwarding(
11625            Arc::new(Registry::default()),
11626            Arc::new(ForwardingTable::default()),
11627        );
11628        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11629        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11630        hello_via_sink(
11631            &handler,
11632            &first_ctx,
11633            &mut first_rx,
11634            hello_frame("first", PROTOCOL_VERSION, 1),
11635        )
11636        .await;
11637        hello_via_sink(
11638            &handler,
11639            &second_ctx,
11640            &mut second_rx,
11641            hello_frame("second", PROTOCOL_VERSION, 2),
11642        )
11643        .await;
11644        let root = unique_project_root("second-only");
11645        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11646        let cloned = handler.clone();
11647        let open = tokio::spawn(async move {
11648            cloned
11649                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11650                .await
11651                .unwrap()
11652        });
11653        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11654            .await
11655            .unwrap()
11656            .unwrap();
11657        let first = query_live_roots(&handler, &first_ctx).await;
11658        let second = query_live_roots(&handler, &second_ctx).await;
11659        assert!(
11660            matches!(
11661                first,
11662                ModuleControlResponseToModule::LiveRoots {
11663                    total_bindings: 0,
11664                    ..
11665                }
11666            ),
11667            "cross-module scope must not expose another module's roots"
11668        );
11669        assert!(
11670            matches!(
11671                second,
11672                ModuleControlResponseToModule::LiveRoots {
11673                    total_bindings: 1,
11674                    ..
11675                }
11676            ),
11677            "second module must see its pending route"
11678        );
11679        handler
11680            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11681            .await
11682            .unwrap();
11683        assert!(open.await.unwrap().is_empty());
11684    }
11685
11686    #[tokio::test(start_paused = true)]
11687    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11688        let handler = ControlHandler::with_forwarding(
11689            Arc::new(Registry::default()),
11690            Arc::new(ForwardingTable::default()),
11691        );
11692        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11693        hello_via_sink(
11694            &handler,
11695            &target_ctx,
11696            &mut target_rx,
11697            hello_frame("target", PROTOCOL_VERSION, 1),
11698        )
11699        .await;
11700        let actual = query_live_roots(&handler, &target_ctx).await;
11701        let ModuleControlResponseToModule::LiveRoots {
11702            roots,
11703            unknown_root_bindings,
11704            total_bindings,
11705        } = actual
11706        else {
11707            panic!("expected live roots")
11708        };
11709        assert!(roots.is_empty());
11710        assert_eq!(unknown_root_bindings, 0);
11711        assert_eq!(total_bindings, 0);
11712        assert_eq!(
11713            total_bindings,
11714            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11715        );
11716    }
11717
11718    /// Read the vendored fed corpus rather than hand-building a package.
11719    ///
11720    /// A hand-built object encodes what the test author believed the carrier
11721    /// emits. These vectors are what it actually emits, and one of them exists
11722    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11723    /// ignores additive unknown fields at the traversal emit terminus."
11724    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11725        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11726            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11727        let text = std::fs::read_to_string(&path)
11728            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11729        let vectors: Vec<(String, Value)> = text
11730            .lines()
11731            .filter(|line| !line.trim().is_empty())
11732            .map(|line| {
11733                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11734                let id = entry["corpus_id"]
11735                    .as_str()
11736                    .expect("every vector carries a corpus_id")
11737                    .to_string();
11738                (id, entry["package"].clone())
11739            })
11740            .collect();
11741        // Pin the count: a corpus that silently shrinks would take its coverage
11742        // with it, and a suite reading N-1 vectors reports the same clean pass
11743        // as one reading N.
11744        assert_eq!(
11745            vectors.len(),
11746            3,
11747            "vendored fed corpus changed size; re-sync from subc-federation"
11748        );
11749
11750        // Pin what makes the corpus DISCRIMINATING, not just present.
11751        //
11752        // The relay test below takes its expected value from the corpus, so the
11753        // corpus supplies the test's power to detect a lossy relay rather than
11754        // its correctness. A relay that dropped unrecognised fields would still
11755        // be caught -- but only by a package carrying fields it does not know.
11756        // Shrink every package to the handful of keys any implementation would
11757        // recognise and the test keeps passing over an input that can no longer
11758        // fail, which is the same clean green as a corpus that shrank away.
11759        //
11760        // So assert the precondition rather than duplicating the packages here:
11761        // at least one vector must carry a field beyond the small common set.
11762        // That is one claim to maintain instead of nine, and it fails loudly if
11763        // a re-sync ever flattens the corpus.
11764        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11765        let richest = vectors
11766            .iter()
11767            .filter_map(|(_, package)| package.as_object())
11768            .map(|object| {
11769                object
11770                    .keys()
11771                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11772                    .count()
11773            })
11774            .max()
11775            .unwrap_or(0);
11776        assert!(
11777            richest >= 2,
11778            "vendored corpus no longer carries a package with unmodelled fields, \
11779             so the relay test can no longer distinguish a verbatim relay from a lossy one"
11780        );
11781
11782        vectors
11783    }
11784
11785    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
11786    ///
11787    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
11788    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
11789    /// three-key object, so a relay that quietly dropped fields it did not
11790    /// recognise would satisfy it. These vectors carry nine keys including ones
11791    /// this crate has no type for, so a typed relay fails here and only here.
11792    #[tokio::test]
11793    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11794        for (corpus_id, package) in fed_admission_facts_vectors() {
11795            let registry = Arc::new(Registry::default());
11796            let forwarding = Arc::new(ForwardingTable::default());
11797            let supervisor = SupervisorHandle::new();
11798            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11799            let handler =
11800                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11801                    .with_supervisor(supervisor)
11802                    .with_admission_facts_config(
11803                        Some("fed".to_string()),
11804                        Some(vec!["target".to_string()]),
11805                    );
11806
11807            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11808            hello_via_sink(
11809                &handler,
11810                &target_ctx,
11811                &mut target_rx,
11812                hello_frame("target", PROTOCOL_VERSION, 1),
11813            )
11814            .await;
11815
11816            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11817            let route_handler = handler.clone();
11818            let expected = package.clone();
11819            let route_task = tokio::spawn(async move {
11820                route_handler
11821                    .handle_control_frame(
11822                        &client_ctx,
11823                        route_open_frame_with_admission_facts(
11824                            20,
11825                            "target",
11826                            unique_project_root("admission-facts"),
11827                            Some(subc_control::ConsumerIdentity {
11828                                module_id: "fed".to_string(),
11829                                launch_nonce: "fed-nonce".to_string(),
11830                            }),
11831                            Some(package),
11832                        ),
11833                    )
11834                    .await
11835                    .unwrap()
11836            });
11837
11838            let bind_frame = target_rx.recv().await.unwrap();
11839            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11840            let ModuleControlRequest::RouteBind {
11841                admission_facts, ..
11842            } = bind
11843            else {
11844                panic!("{corpus_id}: expected route.bind")
11845            };
11846            assert_eq!(
11847                admission_facts,
11848                Some(expected),
11849                "{corpus_id}: relay must not add, drop or reshape any field"
11850            );
11851
11852            handler
11853                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11854                .await
11855                .unwrap();
11856            route_task.await.unwrap();
11857        }
11858    }
11859
11860    #[tokio::test]
11861    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11862        let registry = Arc::new(Registry::default());
11863        let forwarding = Arc::new(ForwardingTable::default());
11864        let supervisor = SupervisorHandle::new();
11865        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11866        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11867        let handler =
11868            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11869                .with_supervisor(supervisor)
11870                .with_admission_facts_config(
11871                    Some("fed".to_string()),
11872                    Some(vec!["target".to_string()]),
11873                );
11874
11875        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11876        hello_via_sink(
11877            &handler,
11878            &target_ctx,
11879            &mut target_rx,
11880            hello_frame("target", PROTOCOL_VERSION, 1),
11881        )
11882        .await;
11883        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11884        hello_via_sink(
11885            &handler,
11886            &other_ctx,
11887            &mut other_rx,
11888            hello_frame("other", PROTOCOL_VERSION, 2),
11889        )
11890        .await;
11891
11892        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11893        let expected_facts = facts.clone();
11894        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11895        let route_handler = handler.clone();
11896        let route_task = tokio::spawn(async move {
11897            route_handler
11898                .handle_control_frame(
11899                    &client_ctx,
11900                    route_open_frame_with_admission_facts(
11901                        10,
11902                        "target",
11903                        unique_project_root("admission-facts"),
11904                        Some(subc_control::ConsumerIdentity {
11905                            module_id: "fed".to_string(),
11906                            launch_nonce: "fed-nonce".to_string(),
11907                        }),
11908                        Some(facts.clone()),
11909                    ),
11910                )
11911                .await
11912                .unwrap()
11913        });
11914        let bind_frame = target_rx.recv().await.unwrap();
11915        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11916        let ModuleControlRequest::RouteBind {
11917            admission_facts, ..
11918        } = bind
11919        else {
11920            panic!("expected route.bind")
11921        };
11922        assert_eq!(admission_facts, Some(expected_facts));
11923        handler
11924            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11925            .await
11926            .unwrap();
11927        assert!(route_task.await.unwrap().is_empty());
11928        assert!(matches!(
11929            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11930                .unwrap(),
11931            ClientControlResponse::RouteOpen { .. }
11932        ));
11933
11934        let direct = handler
11935            .handle_control_frame(
11936                &route_ctx(ConnectionId::new(73)).0,
11937                route_open_frame_with_admission_facts(
11938                    11,
11939                    "target",
11940                    unique_project_root("admission-facts"),
11941                    None,
11942                    Some(json!({"x": 1})),
11943                ),
11944            )
11945            .await
11946            .unwrap();
11947        assert_eq!(
11948            parse_error(&direct[0])["code"],
11949            "admission_facts_not_permitted"
11950        );
11951
11952        let different_reserved = handler
11953            .handle_control_frame(
11954                &route_ctx(ConnectionId::new(77)).0,
11955                route_open_frame_with_admission_facts(
11956                    15,
11957                    "target",
11958                    unique_project_root("admission-facts"),
11959                    Some(subc_control::ConsumerIdentity {
11960                        module_id: "other".to_string(),
11961                        launch_nonce: "other-nonce".to_string(),
11962                    }),
11963                    Some(json!({"x": 1})),
11964                ),
11965            )
11966            .await
11967            .unwrap();
11968        assert_eq!(
11969            parse_error(&different_reserved[0])["code"],
11970            "admission_facts_not_permitted"
11971        );
11972
11973        let other_target = handler
11974            .handle_control_frame(
11975                &route_ctx(ConnectionId::new(74)).0,
11976                route_open_frame_with_admission_facts(
11977                    12,
11978                    "other",
11979                    unique_project_root("admission-facts"),
11980                    Some(subc_control::ConsumerIdentity {
11981                        module_id: "fed".to_string(),
11982                        launch_nonce: "fed-nonce".to_string(),
11983                    }),
11984                    Some(json!({"x": 1})),
11985                ),
11986            )
11987            .await
11988            .unwrap();
11989        assert_eq!(
11990            parse_error(&other_target[0])["code"],
11991            "admission_facts_target_not_allowed"
11992        );
11993
11994        let nonexistent = handler
11995            .handle_control_frame(
11996                &route_ctx(ConnectionId::new(75)).0,
11997                route_open_frame_with_admission_facts(
11998                    13,
11999                    "missing",
12000                    unique_project_root("admission-facts"),
12001                    None,
12002                    Some(json!({"x": 1})),
12003                ),
12004            )
12005            .await
12006            .unwrap();
12007        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12008
12009        let described = handler
12010            .handle_control_frame(
12011                &route_ctx(ConnectionId::new(76)).0,
12012                Frame::build(
12013                    FrameType::Request,
12014                    control_flags(),
12015                    0,
12016                    0,
12017                    14,
12018                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12019                )
12020                .unwrap(),
12021            )
12022            .await
12023            .unwrap();
12024        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12025            serde_json::from_slice(&described[0].body).unwrap()
12026        else {
12027            panic!("expected server.describe response")
12028        };
12029        assert!(capabilities
12030            .iter()
12031            .any(|cap| cap == "admission_facts_relay_v1"));
12032    }
12033
12034    #[tokio::test]
12035    async fn admission_facts_without_configured_carrier_are_rejected() {
12036        let registry = Arc::new(Registry::default());
12037        let forwarding = Arc::new(ForwardingTable::default());
12038        let handler = ControlHandler::with_forwarding(registry, forwarding);
12039        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12040        hello_via_sink(
12041            &handler,
12042            &target_ctx,
12043            &mut target_rx,
12044            hello_frame("target", PROTOCOL_VERSION, 1),
12045        )
12046        .await;
12047
12048        let responses = handler
12049            .handle_control_frame(
12050                &route_ctx(ConnectionId::new(79)).0,
12051                route_open_frame_with_admission_facts(
12052                    16,
12053                    "target",
12054                    unique_project_root("admission-facts"),
12055                    None,
12056                    Some(json!({"x": 1})),
12057                ),
12058            )
12059            .await
12060            .unwrap();
12061        assert_eq!(
12062            parse_error(&responses[0])["code"],
12063            "admission_facts_not_permitted"
12064        );
12065    }
12066
12067    #[tokio::test]
12068    async fn route_open_relays_consumer_capabilities_verbatim() {
12069        let registry = Arc::new(Registry::default());
12070        let forwarding = Arc::new(ForwardingTable::default());
12071        let handler =
12072            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12073        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12074        hello_via_sink(
12075            &handler,
12076            &module_ctx,
12077            &mut module_rx,
12078            hello_frame("aft", PROTOCOL_VERSION, 7),
12079        )
12080        .await;
12081
12082        let expected = vec!["elicitation".to_string(), "roots".to_string()];
12083        let expected_for_request = expected.clone();
12084        let project_root = unique_project_root("consumer-capabilities-present");
12085        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12086        let route_handler = handler.clone();
12087        let route_task = tokio::spawn(async move {
12088            route_handler
12089                .handle_control_frame(
12090                    &client_ctx,
12091                    route_open_frame_with_consumer_capabilities(
12092                        401,
12093                        "aft",
12094                        project_root,
12095                        Some(expected_for_request),
12096                    ),
12097                )
12098                .await
12099                .unwrap()
12100        });
12101        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12102            .await
12103            .unwrap()
12104            .unwrap();
12105        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12106        let ModuleControlRequest::RouteBind {
12107            consumer_capabilities,
12108            ..
12109        } = bind
12110        else {
12111            panic!("expected route.bind request, got {bind:?}");
12112        };
12113        assert_eq!(consumer_capabilities, Some(expected.clone()));
12114
12115        handler
12116            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12117            .await
12118            .unwrap();
12119        let route_response = route_task.await.unwrap();
12120        assert!(route_response.is_empty());
12121        let published = client_rx.recv().await.unwrap();
12122        assert!(matches!(
12123            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12124            ClientControlResponse::RouteOpen { .. }
12125        ));
12126    }
12127
12128    #[tokio::test]
12129    async fn route_open_without_consumer_capabilities_relays_none() {
12130        let registry = Arc::new(Registry::default());
12131        let forwarding = Arc::new(ForwardingTable::default());
12132        let handler =
12133            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12134        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12135        hello_via_sink(
12136            &handler,
12137            &module_ctx,
12138            &mut module_rx,
12139            hello_frame("aft", PROTOCOL_VERSION, 7),
12140        )
12141        .await;
12142
12143        let project_root = unique_project_root("consumer-capabilities-absent");
12144        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12145        let route_handler = handler.clone();
12146        let route_task = tokio::spawn(async move {
12147            route_handler
12148                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12149                .await
12150                .unwrap()
12151        });
12152        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12153            .await
12154            .unwrap()
12155            .unwrap();
12156        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12157        let ModuleControlRequest::RouteBind {
12158            consumer_capabilities,
12159            ..
12160        } = bind
12161        else {
12162            panic!("expected route.bind request, got {bind:?}");
12163        };
12164        assert_eq!(consumer_capabilities, None);
12165
12166        handler
12167            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12168            .await
12169            .unwrap();
12170        let route_response = route_task.await.unwrap();
12171        assert!(route_response.is_empty());
12172        let published = client_rx.recv().await.unwrap();
12173        assert!(matches!(
12174            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12175            ClientControlResponse::RouteOpen { .. }
12176        ));
12177    }
12178
12179    /// Opens a route to a freshly registered `aft` with `sent` as the
12180    /// route.open's role_versions, acks the bind, and returns the role_versions
12181    /// the module's bind carried.
12182    async fn bind_role_versions_for(
12183        sent: Option<BTreeMap<String, String>>,
12184        connection: u64,
12185    ) -> Option<BTreeMap<String, String>> {
12186        let registry = Arc::new(Registry::default());
12187        let forwarding = Arc::new(ForwardingTable::default());
12188        let handler =
12189            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12190        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12191        hello_via_sink(
12192            &handler,
12193            &module_ctx,
12194            &mut module_rx,
12195            hello_frame("aft", PROTOCOL_VERSION, 7),
12196        )
12197        .await;
12198        let project_root = unique_project_root("role-versions");
12199        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12200        let route_handler = handler.clone();
12201        let route_task = tokio::spawn(async move {
12202            route_handler
12203                .handle_control_frame(
12204                    &client_ctx,
12205                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
12206                )
12207                .await
12208                .unwrap()
12209        });
12210        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12211            .await
12212            .expect("a well-formed route.open reaches the module as a bind")
12213            .unwrap();
12214        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12215        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12216            panic!("expected route.bind request, got {bind:?}");
12217        };
12218        handler
12219            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12220            .await
12221            .unwrap();
12222        assert!(route_task.await.unwrap().is_empty());
12223        let published = client_rx.recv().await.unwrap();
12224        assert!(matches!(
12225            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12226            ClientControlResponse::RouteOpen { .. }
12227        ));
12228        role_versions
12229    }
12230
12231    #[tokio::test]
12232    async fn route_open_relays_role_versions_verbatim() {
12233        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12234        assert_eq!(
12235            bind_role_versions_for(Some(sent.clone()), 141).await,
12236            Some(sent)
12237        );
12238    }
12239
12240    /// An empty map declares nothing, so the provider sees no field rather
12241    /// than an empty object it would have to treat as a second "none".
12242    #[tokio::test]
12243    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12244        assert_eq!(bind_role_versions_for(None, 143).await, None);
12245        assert_eq!(
12246            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12247            None
12248        );
12249    }
12250
12251    #[tokio::test]
12252    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12253        let registry = Arc::new(Registry::default());
12254        let forwarding = Arc::new(ForwardingTable::default());
12255        let handler =
12256            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12257        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12258        hello_via_sink(
12259            &handler,
12260            &module_ctx,
12261            &mut module_rx,
12262            hello_frame("aft", PROTOCOL_VERSION, 7),
12263        )
12264        .await;
12265
12266        let nine: BTreeMap<String, String> = (0..9)
12267            .map(|index| (format!("role-{index}"), "v1".to_string()))
12268            .collect();
12269        for (label, malformed) in [
12270            (
12271                "invalid role name",
12272                role_versions(&[("Tool_Provider", "v1")]),
12273            ),
12274            ("invalid version", role_versions(&[("tool-provider", "v0")])),
12275            ("nine entries", nine),
12276        ] {
12277            // A refused open answers at once; one that reached the module
12278            // would wait for its bind ack and trip this timeout.
12279            let responses = tokio::time::timeout(
12280                Duration::from_secs(1),
12281                handler.handle_control_frame(
12282                    &route_ctx(ConnectionId::new(148)).0,
12283                    route_open_frame_with_role_versions(
12284                        404,
12285                        "aft",
12286                        unique_project_root("role-versions-malformed"),
12287                        Some(malformed),
12288                    ),
12289                ),
12290            )
12291            .await
12292            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12293            .unwrap();
12294            assert_eq!(responses.len(), 1, "{label}");
12295            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12296            let error = parse_error(&responses[0]);
12297            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12298            assert_eq!(
12299                error["detail"]["field"], "role_versions",
12300                "{label}: {error}"
12301            );
12302            assert!(
12303                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12304                "{label}: a malformed declaration is terminal"
12305            );
12306            assert!(
12307                module_rx.try_recv().is_err(),
12308                "{label}: the module must never see a bind"
12309            );
12310        }
12311    }
12312
12313    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
12314    /// consumer can tell this daemon forwards the field from one that would
12315    /// drop it.
12316    #[tokio::test]
12317    async fn route_role_versions_capability_is_advertised() {
12318        let handler = ControlHandler::new(Arc::new(Registry::default()));
12319        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12320        let ack = hello_via_sink(
12321            &handler,
12322            &ctx,
12323            &mut rx,
12324            hello_frame("m", PROTOCOL_VERSION, 1),
12325        )
12326        .await;
12327        let ack = parse_ack(&ack);
12328        assert!(
12329            ack.subc_capabilities
12330                .iter()
12331                .any(|c| c == "route-role-versions/v1"),
12332            "{:?}",
12333            ack.subc_capabilities
12334        );
12335
12336        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12337        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12338        let reply = handler
12339            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12340            .await
12341            .unwrap()
12342            .pop()
12343            .unwrap();
12344        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12345            serde_json::from_slice(&reply.body).unwrap()
12346        else {
12347            panic!("not a server.describe reply");
12348        };
12349        assert!(
12350            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12351            "{capabilities:?}"
12352        );
12353    }
12354
12355    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12356    /// catalog entry without `events` (the module declares none) from a daemon
12357    /// that never reports declared events.
12358    #[tokio::test]
12359    async fn catalog_events_capability_is_advertised() {
12360        let handler = ControlHandler::new(Arc::new(Registry::default()));
12361        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12362        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12363        let reply = handler
12364            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12365            .await
12366            .unwrap()
12367            .pop()
12368            .unwrap();
12369        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12370            serde_json::from_slice(&reply.body).unwrap()
12371        else {
12372            panic!("not a server.describe reply");
12373        };
12374        assert!(
12375            capabilities
12376                .iter()
12377                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12378            "{capabilities:?}"
12379        );
12380    }
12381
12382    #[tokio::test]
12383    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12384        let handler = ControlHandler::new(Arc::new(Registry::default()));
12385        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12386        let ack = hello_via_sink(
12387            &handler,
12388            &module,
12389            &mut module_rx,
12390            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12391        )
12392        .await;
12393        let ack = parse_ack(&ack);
12394        assert!(
12395            ack.subc_capabilities
12396                .iter()
12397                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12398            "{:?}",
12399            ack.subc_capabilities
12400        );
12401        assert!(ack
12402            .subc_capabilities
12403            .iter()
12404            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12405
12406        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12407        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12408        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12409        let reply = handler
12410            .handle_control_frame(&client, frame)
12411            .await
12412            .unwrap()
12413            .pop()
12414            .unwrap();
12415        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12416            serde_json::from_slice(&reply.body).unwrap()
12417        else {
12418            panic!("not a server.describe reply");
12419        };
12420        assert!(
12421            capabilities
12422                .iter()
12423                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12424            "{capabilities:?}"
12425        );
12426        assert!(capabilities
12427            .iter()
12428            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12429    }
12430
12431    #[tokio::test]
12432    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12433        let handler = ControlHandler::new(Arc::new(Registry::default()));
12434        let (module, mut rx) = route_ctx(ConnectionId::new(156));
12435        let ack = hello_via_sink(
12436            &handler,
12437            &module,
12438            &mut rx,
12439            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12440        )
12441        .await;
12442        assert!(parse_ack(&ack)
12443            .subc_capabilities
12444            .iter()
12445            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12446        assert!(parse_ack(&ack)
12447            .subc_capabilities
12448            .iter()
12449            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12450        let (client, _rx) = route_ctx(ConnectionId::new(157));
12451        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12452        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12453        let reply = handler
12454            .handle_control_frame(&client, frame)
12455            .await
12456            .unwrap()
12457            .pop()
12458            .unwrap();
12459        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12460            serde_json::from_slice(&reply.body).unwrap()
12461        else {
12462            panic!("server.describe reply")
12463        };
12464        assert!(capabilities
12465            .iter()
12466            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12467        assert!(capabilities
12468            .iter()
12469            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12470    }
12471
12472    #[tokio::test]
12473    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12474        let handler = ControlHandler::new(Arc::new(Registry::default()));
12475        let (module, mut rx) = route_ctx(ConnectionId::new(158));
12476        let ack = hello_via_sink(
12477            &handler,
12478            &module,
12479            &mut rx,
12480            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12481        )
12482        .await;
12483        assert!(parse_ack(&ack)
12484            .subc_capabilities
12485            .iter()
12486            .any(|cap| cap == "scope-workdir/v1"));
12487        let (client, _rx) = route_ctx(ConnectionId::new(159));
12488        let frame = Frame::build(
12489            FrameType::Request,
12490            control_flags(),
12491            0,
12492            0,
12493            5,
12494            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12495        )
12496        .unwrap();
12497        let reply = handler
12498            .handle_control_frame(&client, frame)
12499            .await
12500            .unwrap()
12501            .remove(0);
12502        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12503            serde_json::from_slice(&reply.body).unwrap()
12504        else {
12505            panic!("server.describe reply")
12506        };
12507        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12508    }
12509
12510    #[tokio::test]
12511    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12512        let handler = ControlHandler::new(Arc::new(Registry::default()));
12513        let (module, mut rx) = route_ctx(ConnectionId::new(154));
12514        let ack = hello_via_sink(
12515            &handler,
12516            &module,
12517            &mut rx,
12518            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12519        )
12520        .await;
12521        assert!(parse_ack(&ack)
12522            .subc_capabilities
12523            .iter()
12524            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12525        let (client, _rx) = route_ctx(ConnectionId::new(155));
12526        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12527        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12528        let reply = handler
12529            .handle_control_frame(&client, frame)
12530            .await
12531            .unwrap()
12532            .pop()
12533            .unwrap();
12534        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12535            serde_json::from_slice(&reply.body).unwrap()
12536        else {
12537            panic!("not a server.describe reply")
12538        };
12539        assert!(capabilities
12540            .iter()
12541            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12542    }
12543
12544    #[tokio::test]
12545    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12546        let registry = Arc::new(Registry::default());
12547        let forwarding = Arc::new(ForwardingTable::default());
12548        let handler =
12549            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12550                .with_health_probe_timeout(Duration::from_secs(5));
12551        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12552        hello_via_sink(
12553            &handler,
12554            &module_ctx,
12555            &mut module_rx,
12556            non_routable_hello_frame_with_control_ops(
12557                "mcp",
12558                300,
12559                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12560            ),
12561        )
12562        .await;
12563        assert!(registry
12564            .get_module("mcp")
12565            .unwrap()
12566            .unwrap()
12567            .manifest
12568            .provides
12569            .is_empty());
12570
12571        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12572        let route_response = handler
12573            .handle_control_frame(
12574                &route_client_ctx,
12575                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12576            )
12577            .await
12578            .unwrap();
12579        assert_eq!(route_response[0].header.ty, FrameType::Error);
12580        assert_eq!(
12581            parse_error(&route_response[0])["code"],
12582            "target_unavailable"
12583        );
12584        assert!(parse_error(&route_response[0])["message"]
12585            .as_str()
12586            .unwrap()
12587            .contains("does not provide the requested target"));
12588        assert!(module_rx.try_recv().is_err());
12589
12590        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12591        let health_handler = handler.clone();
12592        let health_task = tokio::spawn(async move {
12593            health_handler
12594                .handle_control_frame(
12595                    &health_client_ctx,
12596                    supervisor_health_probe_frame(302, "mcp"),
12597                )
12598                .await
12599                .unwrap()
12600        });
12601        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12602            .await
12603            .unwrap()
12604            .unwrap();
12605        assert_eq!(
12606            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12607            ModuleControlRequest::HealthCheck {}
12608        );
12609        handler
12610            .handle_control_frame(
12611                &module_ctx,
12612                health_response(health_frame.header.corr, HealthStatus::Ok),
12613            )
12614            .await
12615            .unwrap();
12616        let health_response = health_task.await.unwrap();
12617        assert_eq!(health_response[0].header.ty, FrameType::Response);
12618        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12619            ClientControlResponse::SupervisorHealthProbe {
12620                module_id, status, ..
12621            } => {
12622                assert_eq!(module_id, "mcp");
12623                assert_eq!(status, HealthStatus::Ok);
12624            }
12625            other => panic!("unexpected health response: {other:?}"),
12626        }
12627
12628        // Exercise the forwarding cleanup path directly while leaving the registry
12629        // advertisement in place. If cleanup leaves a stale control sink behind,
12630        // the next probe will enqueue onto it and wait for the long probe timeout
12631        // instead of returning an immediate no-connection error.
12632        forwarding
12633            .cleanup_connection(module_ctx.connection_id)
12634            .unwrap();
12635        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12636        let cleanup_response = tokio::time::timeout(
12637            Duration::from_millis(200),
12638            handler.handle_control_frame(
12639                &cleanup_probe_ctx,
12640                supervisor_health_probe_frame(303, "mcp"),
12641            ),
12642        )
12643        .await
12644        .expect("probe should fail immediately when the control lane is gone")
12645        .unwrap();
12646        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12647        assert_eq!(
12648            parse_error(&cleanup_response[0])["code"],
12649            "target_unavailable"
12650        );
12651        assert!(parse_error(&cleanup_response[0])["message"]
12652            .as_str()
12653            .unwrap()
12654            .contains("no module connection"));
12655
12656        handler
12657            .cleanup_connection(module_ctx.connection_id)
12658            .unwrap();
12659    }
12660
12661    #[tokio::test]
12662    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12663        let registry = Arc::new(Registry::default());
12664        let supervisor_handle = SupervisorHandle::new();
12665        let supervisor =
12666            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12667                .with_handle(supervisor_handle.clone())
12668                .with_connection_file_path(
12669                    std::env::temp_dir()
12670                        .join(format!("subc-route-open-warming-{}", std::process::id())),
12671                );
12672        let module = supervisor
12673            .supervise_configured(
12674                ModuleSpec {
12675                    module_id: "warming".to_string(),
12676                    program: fake_aft_stub_path(),
12677                    args: Vec::new(),
12678                    env: Vec::new(),
12679                    reserved: false,
12680                    reserved_prefixes: Vec::new(),
12681                    protocol: ModuleProtocol::Subc,
12682                    overlap: Default::default(),
12683                },
12684                true,
12685            )
12686            .unwrap();
12687        assert_eq!(module.state().unwrap(), ModuleState::Running);
12688
12689        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12690        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12691        let response = handler
12692            .handle_control_frame(
12693                &ctx,
12694                route_open_frame(304, "warming", unique_project_root("warming")),
12695            )
12696            .await
12697            .unwrap();
12698        module.stop().await.unwrap();
12699
12700        assert_eq!(response[0].header.ty, FrameType::Error);
12701        let error = parse_error(&response[0]);
12702        assert_eq!(error["code"], "module_warming");
12703        assert!(error["message"]
12704            .as_str()
12705            .unwrap()
12706            .contains("state=running, enabled=true, live=false"));
12707    }
12708
12709    #[test]
12710    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12711        let handler = ControlHandler::new(Arc::new(Registry::default()));
12712        let capture = EventCapture::default();
12713        let _subscriber =
12714            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12715        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12716        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12717        let pending = (0..limit).collect::<Vec<_>>();
12718        let response = handler
12719            .route_open_capacity_refusal(
12720                &ctx,
12721                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12722                "busy",
12723                pending.len(),
12724                limit,
12725            )
12726            .unwrap();
12727        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12728        let event = capture
12729            .events()
12730            .into_iter()
12731            .find(|event| {
12732                event.target == "control"
12733                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12734            })
12735            .expect("connection admission refusal event");
12736        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12737        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12738        // Refused before `handle_route_open` parses the request, so the caller
12739        // fields are read from the frame itself.
12740        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
12741        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
12742    }
12743
12744    #[test]
12745    fn route_open_target_cap_logs_admission_reason_and_capacity() {
12746        let handler = ControlHandler::new(Arc::new(Registry::default()));
12747        let capture = EventCapture::default();
12748        let _subscriber =
12749            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12750        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12751        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12752        let guards = (0..limit)
12753            .map(|_| {
12754                handler
12755                    .route_bind_concurrency
12756                    .try_admit("busy", limit)
12757                    .unwrap()
12758            })
12759            .collect::<Vec<_>>();
12760        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12761            Err(in_flight) => in_flight,
12762            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12763        };
12764        let response = handler
12765            .route_open_target_capacity_refusal(
12766                &ctx,
12767                &RouteOpenCaller::unknown(),
12768                &route_open_frame(397, "busy", unique_project_root("target-cap")),
12769                "busy",
12770                in_flight,
12771            )
12772            .unwrap();
12773        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12774        let event = capture
12775            .events()
12776            .into_iter()
12777            .find(|event| {
12778                event.target == "control"
12779                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12780            })
12781            .expect("target admission refusal event");
12782        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12783        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12784        drop(guards);
12785    }
12786
12787    #[tokio::test]
12788    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12789        let registry = Arc::new(Registry::default());
12790        let handle = SupervisorHandle::new();
12791        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12792            .with_handle(handle.clone());
12793        supervisor
12794            .supervise_configured(
12795                crate::ModuleSpec {
12796                    module_id: "drift".into(),
12797                    program: PathBuf::from("disabled-module"),
12798                    args: Vec::new(),
12799                    env: Vec::new(),
12800                    reserved: false,
12801                    reserved_prefixes: Vec::new(),
12802                    protocol: ModuleProtocol::Subc,
12803                    overlap: Default::default(),
12804                },
12805                false,
12806            )
12807            .unwrap();
12808        let handler = ControlHandler::new(registry).with_supervisor(handle);
12809        for configured in [None, Some(false), Some(true)] {
12810            handler.capability_evaluator.configure(
12811                configured.map(|enabled| ("drift".to_string(), enabled)),
12812                BTreeMap::new(),
12813            );
12814            let frame =
12815                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12816            let replies = handler
12817                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12818                .await
12819                .unwrap();
12820            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12821            assert!(body.get("configured_enabled").is_none());
12822            assert_eq!(body["op"], "supervisor.ack");
12823            let frame =
12824                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12825            let replies = handler.handle_supervisor_list(frame).await.unwrap();
12826            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12827            assert_eq!(
12828                body["modules"][0].get("configured_enabled"),
12829                configured.map(Value::Bool).as_ref()
12830            );
12831        }
12832    }
12833
12834    #[tokio::test(flavor = "current_thread")]
12835    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12836        let handler = ControlHandler::new(Arc::new(Registry::default()));
12837        let capture = EventCapture::default();
12838        let _subscriber =
12839            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12840        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12841        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12842
12843        let _ = handler
12844            .handle_client_control_request(
12845                &ctx,
12846                frame,
12847                ClientControlRequest::SupervisorSetEnabled {
12848                    module_id: "broca".to_string(),
12849                    enabled: false,
12850                },
12851                &mut None,
12852            )
12853            .await
12854            .unwrap();
12855
12856        let events = capture
12857            .events()
12858            .into_iter()
12859            .filter(|event| {
12860                event.target == "control"
12861                    && event.fields.get("message").map(String::as_str)
12862                        == Some("supervisor request received")
12863            })
12864            .collect::<Vec<_>>();
12865        assert_eq!(events.len(), 1, "one supervisor request log line");
12866        let event = &events[0];
12867        assert_eq!(event.level, tracing::Level::INFO);
12868        assert_eq!(
12869            event.fields.get("op"),
12870            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12871        );
12872        assert_eq!(
12873            event.fields.get("module_id"),
12874            Some(&"\"broca\"".to_string())
12875        );
12876        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12877        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12878        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12879    }
12880
12881    #[tokio::test(flavor = "current_thread")]
12882    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12883        let registry = Arc::new(Registry::default());
12884        let module_connection = ConnectionId::new(303);
12885        registry
12886            .register_with_control_ops(
12887                manifest("aft", PROTOCOL_VERSION),
12888                PROTOCOL_VERSION,
12889                module_connection,
12890                module_baseline_control_ops(),
12891            )
12892            .unwrap();
12893        let handler = ControlHandler::new(registry);
12894        let capture = EventCapture::default();
12895        let _subscriber =
12896            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12897        let (ctx, _rx) = route_ctx(module_connection);
12898        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12899
12900        let _ = handler
12901            .handle_client_control_request(
12902                &ctx,
12903                frame,
12904                ClientControlRequest::SupervisorSetEnabled {
12905                    module_id: "broca".to_string(),
12906                    enabled: false,
12907                },
12908                &mut None,
12909            )
12910            .await
12911            .unwrap();
12912
12913        let callers = capture
12914            .events()
12915            .into_iter()
12916            .filter(|event| {
12917                event.fields.get("message").map(String::as_str)
12918                    == Some("supervisor request received")
12919            })
12920            .map(|event| event.fields.get("caller").cloned())
12921            .collect::<Vec<_>>();
12922        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12923    }
12924
12925    #[tokio::test(flavor = "current_thread")]
12926    async fn supervisor_list_does_not_log_request_received() {
12927        let handler = ControlHandler::new(Arc::new(Registry::default()));
12928        let capture = EventCapture::default();
12929        let _subscriber =
12930            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12931        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12932        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12933
12934        handler
12935            .handle_client_control_request(
12936                &ctx,
12937                frame,
12938                ClientControlRequest::SupervisorList {},
12939                &mut None,
12940            )
12941            .await
12942            .unwrap();
12943
12944        assert!(capture.events().into_iter().all(|event| {
12945            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12946        }));
12947    }
12948
12949    #[tokio::test(flavor = "current_thread")]
12950    async fn supervisor_rescan_preview_does_not_log_request_received() {
12951        let handler = ControlHandler::new(Arc::new(Registry::default()));
12952        let capture = EventCapture::default();
12953        let _subscriber =
12954            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12955        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12956        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12957
12958        handler
12959            .handle_client_control_request(
12960                &ctx,
12961                frame,
12962                ClientControlRequest::SupervisorRescan { preview: true },
12963                &mut None,
12964            )
12965            .await
12966            .unwrap();
12967
12968        assert!(capture.events().into_iter().all(|event| {
12969            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12970        }));
12971    }
12972
12973    struct ApproveOperator;
12974    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12975        fn prompt(
12976            &self,
12977            _: &str,
12978            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12979        ) -> crate::operator_confirm::ProviderResult {
12980            crate::operator_confirm::ProviderResult::Approved
12981        }
12982    }
12983
12984    #[tokio::test]
12985    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12986        assert_eq!(
12987            ControlHandler::default().privileged_operator_prompt,
12988            PrivilegedOperatorPrompt::Log
12989        );
12990        for (initial, changed, running) in [
12991            ("log", "enforce", PrivilegedOperatorPrompt::Log),
12992            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12993        ] {
12994            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12995            let path = dir.join("subc.jsonc");
12996            std::fs::write(
12997                &path,
12998                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12999            )
13000            .unwrap();
13001            let config = crate::daemon_config::load(&path).unwrap().unwrap();
13002            let registry = Arc::new(Registry::default());
13003            let handle = SupervisorHandle::new();
13004            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13005                .with_handle(handle.clone());
13006            let handler = ControlHandler::new(registry)
13007                .with_supervisor(handle)
13008                .with_privileged_operator_prompt(config.privileged_operator_prompt)
13009                .with_supervisor_rescan(supervisor, &path, config.port);
13010            handler.forwarding.operator_confirms().configure(
13011                Arc::new(ApproveOperator),
13012                crate::operator_confirm::OperatorLimits::default(),
13013            );
13014            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13015
13016            for (disk_mode, needs_restart) in [
13017                (Some(initial), false),
13018                (Some(changed), true),
13019                (Some(changed), true),
13020                (None, running == PrivilegedOperatorPrompt::Enforce),
13021                (Some(initial), false),
13022            ] {
13023                let mut doc = json!({"version":1});
13024                if let Some(mode) = disk_mode {
13025                    doc["privileged_operator_prompt"] = json!(mode);
13026                }
13027                std::fs::write(&path, doc.to_string()).unwrap();
13028                for preview in [true, false] {
13029                    let request = ClientControlRequest::SupervisorRescan { preview };
13030                    let frame = Frame::build(
13031                        FrameType::Request,
13032                        control_flags(),
13033                        0,
13034                        0,
13035                        1,
13036                        serde_json::to_vec(&request).unwrap(),
13037                    )
13038                    .unwrap();
13039                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13040                    assert_eq!(frames.len(), 1);
13041                    let ClientControlResponse::SupervisorRescan { result } =
13042                        serde_json::from_slice(&frames[0].body).unwrap()
13043                    else {
13044                        panic!("expected a rescan response");
13045                    };
13046                    let expected = if needs_restart {
13047                        vec!["privileged_operator_prompt"]
13048                    } else {
13049                        Vec::new()
13050                    };
13051                    assert_eq!(result.restart_required, expected);
13052                    assert_eq!(handler.privileged_operator_prompt, running);
13053                    assert_eq!(
13054                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13055                        running
13056                    );
13057                }
13058            }
13059        }
13060    }
13061
13062    #[tokio::test]
13063    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13064        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13065        let path = dir.join("subc.jsonc");
13066        let registry = Arc::new(Registry::default());
13067        let handle = SupervisorHandle::new();
13068        let supervisor =
13069            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13070        let handler = ControlHandler::new(registry)
13071            .with_supervisor(handle)
13072            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13073            .with_supervisor_rescan(supervisor, &path, None);
13074        handler.forwarding.operator_confirms().configure(
13075            Arc::new(ApproveOperator),
13076            crate::operator_confirm::OperatorLimits::default(),
13077        );
13078        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13079        for preview in [true, false] {
13080            std::fs::write(
13081                &path,
13082                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13083            )
13084            .unwrap();
13085            let request = ClientControlRequest::SupervisorRescan { preview };
13086            let frame = Frame::build(
13087                FrameType::Request,
13088                control_flags(),
13089                0,
13090                0,
13091                1,
13092                serde_json::to_vec(&request).unwrap(),
13093            )
13094            .unwrap();
13095            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13096            assert_eq!(frames.len(), 1);
13097            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13098            assert_eq!(
13099                handler.privileged_operator_prompt,
13100                PrivilegedOperatorPrompt::Enforce
13101            );
13102        }
13103    }
13104
13105    /// One wire code has several senders, so the refusal line names the check
13106    /// that refused. This drives the shared refusal path for ordinary refusals
13107    /// with an unregistered
13108    /// target and requires the branch label on the event.
13109    #[tokio::test]
13110    async fn route_open_refusal_names_the_check_that_refused() {
13111        let handler = ControlHandler::new(Arc::new(Registry::default()));
13112        let capture = EventCapture::default();
13113        let _subscriber =
13114            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13115        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13116        let response = handler
13117            .handle_control_frame(
13118                &ctx,
13119                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13120            )
13121            .await
13122            .unwrap();
13123
13124        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13125        let event = capture
13126            .events()
13127            .into_iter()
13128            .find(|event| {
13129                event.target == "control"
13130                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13131            })
13132            .expect("route.open refusal event");
13133        assert_eq!(
13134            event.fields.get("reason"),
13135            Some(&"\"not_registered\"".to_string())
13136        );
13137    }
13138
13139    #[tokio::test]
13140    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13141        let registry = Arc::new(Registry::default());
13142        let supervisor_handle = SupervisorHandle::new();
13143        let supervisor =
13144            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13145                .with_handle(supervisor_handle.clone())
13146                .with_connection_file_path(std::env::temp_dir().join(format!(
13147                    "subc-route-open-refusal-info-{}",
13148                    std::process::id()
13149                )));
13150        let module = supervisor
13151            .supervise_configured(
13152                ModuleSpec {
13153                    module_id: "warming".to_string(),
13154                    program: fake_aft_stub_path(),
13155                    args: Vec::new(),
13156                    env: Vec::new(),
13157                    reserved: false,
13158                    reserved_prefixes: Vec::new(),
13159                    protocol: ModuleProtocol::Subc,
13160                    overlap: Default::default(),
13161                },
13162                true,
13163            )
13164            .unwrap();
13165        assert_eq!(module.state().unwrap(), ModuleState::Running);
13166
13167        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13168        assert!(handler
13169            .counters()
13170            .snapshot()
13171            .get("route_open_refused_by_code")
13172            .is_none());
13173        let capture = EventCapture::default();
13174        let _subscriber =
13175            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13176        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13177        let response = handler
13178            .handle_control_frame(
13179                &ctx,
13180                route_open_frame(394, "warming", unique_project_root("refusal-info")),
13181            )
13182            .await
13183            .unwrap();
13184        module.stop().await.unwrap();
13185
13186        assert_eq!(parse_error(&response[0])["code"], "module_warming");
13187        let event = capture
13188            .events()
13189            .into_iter()
13190            .find(|event| {
13191                event.target == "control"
13192                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13193            })
13194            .expect("route.open refusal event");
13195        assert_eq!(
13196            event.fields.get("module_id"),
13197            Some(&"\"warming\"".to_string())
13198        );
13199        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13200        assert_eq!(
13201            event.fields.get("reason"),
13202            Some(&"\"supervised_not_registered\"".to_string())
13203        );
13204        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13205        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13206        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13207        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13208        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13209        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13210        assert_eq!(
13211            handler.counters().snapshot()["route_open_refused_by_code"],
13212            json!({ "module_warming": 1 })
13213        );
13214    }
13215
13216    const OUTAGE_START: &str = "route.open refusing module: not serving";
13217    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13218
13219    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13220        capture
13221            .events()
13222            .into_iter()
13223            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13224            .collect()
13225    }
13226
13227    fn supervise_stub(
13228        registry: &Arc<Registry>,
13229        module_id: &str,
13230        enabled: bool,
13231    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13232        let supervisor_handle = SupervisorHandle::new();
13233        let supervisor =
13234            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13235                .with_handle(supervisor_handle.clone())
13236                .with_connection_file_path(std::env::temp_dir().join(format!(
13237                    "subc-route-outage-{module_id}-{}",
13238                    std::process::id()
13239                )));
13240        let module = supervisor
13241            .supervise_configured(
13242                ModuleSpec {
13243                    module_id: module_id.to_string(),
13244                    program: fake_aft_stub_path(),
13245                    args: Vec::new(),
13246                    env: Vec::new(),
13247                    reserved: false,
13248                    reserved_prefixes: Vec::new(),
13249                    protocol: ModuleProtocol::Subc,
13250                    overlap: Default::default(),
13251                },
13252                enabled,
13253            )
13254            .unwrap();
13255        (supervisor_handle, module)
13256    }
13257
13258    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13259        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13260            module_id: module_id.to_string(),
13261            drain_timeout_ms: Some(50),
13262        })
13263        .unwrap();
13264        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13265    }
13266
13267    /// Two handlers built over one forwarding table must share one outage
13268    /// tracker; separate trackers would each log their own opening line for
13269    /// the same outage.
13270    #[test]
13271    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13272        let registry = Arc::new(Registry::default());
13273        let forwarding = Arc::new(ForwardingTable::default());
13274        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13275        let second = ControlHandler::with_forwarding(registry, forwarding);
13276        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13277    }
13278
13279    /// A client can name any module id it likes. Refusing an unknown one,
13280    /// however often, must not create outage state or outage lines, or the
13281    /// tracker would be a memory sink any client could fill.
13282    #[tokio::test(flavor = "current_thread")]
13283    async fn route_open_unknown_module_refusals_add_no_outage_state() {
13284        let handler = ControlHandler::new(Arc::new(Registry::default()));
13285        let capture = EventCapture::default();
13286        let _subscriber =
13287            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13288        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13289        for corr in 0..8 {
13290            let response = handler
13291                .handle_control_frame(
13292                    &ctx,
13293                    route_open_frame(
13294                        380 + corr,
13295                        &format!("nobody-{corr}"),
13296                        unique_project_root("outage-unknown"),
13297                    ),
13298                )
13299                .await
13300                .unwrap();
13301            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13302        }
13303
13304        assert_eq!(handler.route_outages.tracked_module_count(), 0);
13305        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13306        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13307    }
13308
13309    /// Drives the refusal path end to end: a supervised module that served
13310    /// before and stopped being registered with no instruction to stop is a
13311    /// WARN, and the same module refused after an operator `supervisor.restart`
13312    /// is an INFO.
13313    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13314    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13315        let registry = Arc::new(Registry::default());
13316        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
13317        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13318        let capture = EventCapture::default();
13319        let _subscriber =
13320            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13321        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13322        // The stub never registers, so pretend it served once: otherwise every
13323        // refusal would fall in its startup window.
13324        handler.route_outages.record_accepted("outage-restart");
13325
13326        let response = handler
13327            .handle_control_frame(
13328                &ctx,
13329                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13330            )
13331            .await
13332            .unwrap();
13333        assert_eq!(response[0].header.ty, FrameType::Error);
13334        let starts = outage_lines(&capture, OUTAGE_START);
13335        assert_eq!(starts.len(), 1, "{starts:?}");
13336        assert_eq!(starts[0].level, tracing::Level::WARN);
13337        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13338        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13339        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13340        handler.route_outages.record_accepted("outage-restart");
13341        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13342
13343        let restart = handler
13344            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13345            .await
13346            .unwrap();
13347        assert_eq!(
13348            restart[0].header.ty,
13349            FrameType::Response,
13350            "{:?}",
13351            parse_error(&restart[0])
13352        );
13353        handler
13354            .handle_control_frame(
13355                &ctx,
13356                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13357            )
13358            .await
13359            .unwrap();
13360        module.stop().await.unwrap();
13361
13362        let starts = outage_lines(&capture, OUTAGE_START);
13363        assert_eq!(starts.len(), 2, "{starts:?}");
13364        assert_eq!(starts[1].level, tracing::Level::INFO);
13365        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13366    }
13367
13368    /// A restart refused before it touched the module (here: the module is
13369    /// disabled) must clear its operator mark, so the next real outage is
13370    /// still reported as a warning.
13371    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13372    async fn failed_operator_restart_leaves_no_operator_mark() {
13373        let registry = Arc::new(Registry::default());
13374        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
13375        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13376        let capture = EventCapture::default();
13377        let _subscriber =
13378            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13379        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13380        handler.route_outages.record_accepted("outage-disabled");
13381
13382        let restart = handler
13383            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13384            .await
13385            .unwrap();
13386        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13387        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13388
13389        handler
13390            .handle_control_frame(
13391                &ctx,
13392                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13393            )
13394            .await
13395            .unwrap();
13396        let starts = outage_lines(&capture, OUTAGE_START);
13397        assert_eq!(starts.len(), 1, "{starts:?}");
13398        assert_eq!(starts[0].level, tracing::Level::WARN);
13399    }
13400
13401    #[tokio::test(flavor = "current_thread")]
13402    async fn route_open_unknown_module_escapes_target_module_id() {
13403        let handler = ControlHandler::new(Arc::new(Registry::default()));
13404        let capture = EventCapture::default();
13405        let _subscriber =
13406            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13407        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13408        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13409        let response = handler
13410            .handle_control_frame(
13411                &ctx,
13412                route_open_frame(
13413                    395,
13414                    hostile_module_id,
13415                    unique_project_root("hostile-target-module-id"),
13416                ),
13417            )
13418            .await
13419            .unwrap();
13420
13421        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13422        let event = capture
13423            .events()
13424            .into_iter()
13425            .find(|event| {
13426                event.target == "control"
13427                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13428            })
13429            .expect("route.open unknown-module refusal event");
13430        let logged = event.fields.get("module_id").expect("module_id field");
13431        assert!(!logged.bytes().any(|byte| byte < 0x20));
13432        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13433    }
13434
13435    #[tokio::test(flavor = "current_thread")]
13436    async fn route_open_module_rejection_uses_daemon_counter_key() {
13437        let registry = Arc::new(Registry::default());
13438        let forwarding = Arc::new(ForwardingTable::default());
13439        let handler =
13440            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13441        let module_connection = ConnectionId::new(95);
13442        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13443        hello_via_sink(
13444            &handler,
13445            &module_ctx,
13446            &mut module_rx,
13447            hello_frame("aft", PROTOCOL_VERSION, 395),
13448        )
13449        .await;
13450
13451        let client_connection = ConnectionId::new(96);
13452        let (client_ctx, _client_rx) = route_ctx(client_connection);
13453        let capture = EventCapture::default();
13454        let _subscriber =
13455            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13456        let (route_task, bind) = relay_route_open(
13457            &handler,
13458            client_connection,
13459            &client_ctx.egress,
13460            &mut module_rx,
13461            396,
13462            "aft",
13463            "hostile-module-code",
13464        )
13465        .await;
13466        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13467        let rejection = Frame::build(
13468            FrameType::Error,
13469            control_flags(),
13470            0,
13471            0,
13472            bind.header.corr,
13473            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13474        )
13475        .unwrap();
13476        handler
13477            .handle_control_frame(&module_ctx, rejection)
13478            .await
13479            .unwrap();
13480
13481        let response = route_task.await.unwrap();
13482        assert_eq!(parse_error(&response[0])["code"], hostile_code);
13483        let counters = handler.counters().snapshot();
13484        assert_eq!(
13485            counters["route_open_refused_by_code"],
13486            json!({ "module_rejected": 1 })
13487        );
13488        assert!(counters["route_open_refused_by_code"]
13489            .get(hostile_code)
13490            .is_none());
13491
13492        let event = capture
13493            .events()
13494            .into_iter()
13495            .find(|event| {
13496                event.target == "control"
13497                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13498            })
13499            .expect("route.open module-rejection refusal event");
13500        let logged = event.fields.get("module_code").expect("module_code field");
13501        assert!(!logged.bytes().any(|byte| byte < 0x20));
13502        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13503    }
13504
13505    #[tokio::test]
13506    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13507        let registry = Arc::new(Registry::default());
13508        let supervisor_handle = SupervisorHandle::new();
13509        let missing_program = std::env::temp_dir().join(format!(
13510            "subc-route-open-missing-program-{}",
13511            std::process::id()
13512        ));
13513        let supervisor =
13514            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13515                .with_handle(supervisor_handle.clone());
13516        let module = supervisor
13517            .supervise_configured(
13518                ModuleSpec {
13519                    module_id: "failed".to_string(),
13520                    program: missing_program,
13521                    args: Vec::new(),
13522                    env: Vec::new(),
13523                    reserved: false,
13524                    reserved_prefixes: Vec::new(),
13525                    protocol: ModuleProtocol::Subc,
13526                    overlap: Default::default(),
13527                },
13528                true,
13529            )
13530            .unwrap();
13531        assert_eq!(module.state().unwrap(), ModuleState::Failed);
13532
13533        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13534        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13535        let response = handler
13536            .handle_control_frame(
13537                &ctx,
13538                route_open_frame(305, "failed", unique_project_root("failed")),
13539            )
13540            .await
13541            .unwrap();
13542
13543        assert_eq!(response[0].header.ty, FrameType::Error);
13544        let error = parse_error(&response[0]);
13545        assert_eq!(error["code"], "target_unavailable");
13546        assert!(error["message"]
13547            .as_str()
13548            .unwrap()
13549            .contains("state=failed, enabled=true, live=false"));
13550    }
13551
13552    #[tokio::test]
13553    async fn route_open_role_mismatch_remains_target_unavailable() {
13554        let registry = Arc::new(Registry::default());
13555        let handler = ControlHandler::new(Arc::clone(&registry));
13556        handler
13557            .handle_control(
13558                ConnectionId::new(41),
13559                non_routable_hello_frame_with_control_ops("health-only", 306, None),
13560            )
13561            .unwrap();
13562
13563        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13564        let response = handler
13565            .handle_control_frame(
13566                &ctx,
13567                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13568            )
13569            .await
13570            .unwrap();
13571
13572        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13573        assert!(parse_error(&response[0])["message"]
13574            .as_str()
13575            .unwrap()
13576            .contains("does not provide the requested target"));
13577    }
13578
13579    #[tokio::test]
13580    async fn route_open_inactive_registration_remains_target_unavailable() {
13581        let registry = Arc::new(Registry::default());
13582        let handler = ControlHandler::new(Arc::clone(&registry));
13583        handler
13584            .handle_control(
13585                ConnectionId::new(43),
13586                hello_frame("inactive", PROTOCOL_VERSION, 308),
13587            )
13588            .unwrap();
13589        assert!(registry
13590            .set_module_state_for_test("inactive", ChannelState::Closed)
13591            .unwrap());
13592
13593        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13594        let response = handler
13595            .handle_control_frame(
13596                &ctx,
13597                route_open_frame(309, "inactive", unique_project_root("inactive")),
13598            )
13599            .await
13600            .unwrap();
13601
13602        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13603        assert!(parse_error(&response[0])["message"]
13604            .as_str()
13605            .unwrap()
13606            .contains("is not active"));
13607    }
13608
13609    #[tokio::test]
13610    async fn late_health_reply_is_recorded_through_the_module_response_path() {
13611        let registry = Arc::new(Registry::default());
13612        let forwarding = Arc::new(ForwardingTable::default());
13613        let supervisor_handle = SupervisorHandle::new();
13614        let supervisor =
13615            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
13616                .with_forwarding(Arc::clone(&forwarding))
13617                .with_handle(supervisor_handle.clone());
13618        let module = supervisor
13619            .supervise_configured(
13620                crate::ModuleSpec {
13621                    module_id: "late-health-response".to_string(),
13622                    program: PathBuf::from("disabled-module"),
13623                    args: Vec::new(),
13624                    env: Vec::new(),
13625                    reserved: false,
13626                    reserved_prefixes: Vec::new(),
13627                    protocol: ModuleProtocol::Subc,
13628                    overlap: Default::default(),
13629                },
13630                false,
13631            )
13632            .unwrap();
13633        let handler =
13634            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13635                .with_supervisor(supervisor_handle);
13636        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13637        handler
13638            .handle_control_frame(
13639                &module_ctx,
13640                hello_frame_with_control_ops(
13641                    "late-health-response",
13642                    PROTOCOL_VERSION,
13643                    7,
13644                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13645                ),
13646            )
13647            .await
13648            .unwrap();
13649        let probe_started_at = Instant::now() - Duration::from_millis(80);
13650        let pending = forwarding
13651            .begin_health_probe_rpc_for(
13652                "late-health-response",
13653                MODULE_CONTROL_OP_HEALTH_CHECK,
13654                probe_started_at,
13655                Instant::now() - Duration::from_millis(1),
13656            )
13657            .unwrap();
13658        assert!(forwarding
13659            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13660            .unwrap());
13661
13662        let responses = handler
13663            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13664            .await
13665            .unwrap();
13666
13667        assert!(responses.is_empty());
13668        let health = module.status().unwrap().health;
13669        assert_eq!(health.late_answer_count, 1);
13670        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13671    }
13672
13673    #[tokio::test]
13674    async fn health_probe_timeout_and_module_death_are_typed() {
13675        let registry = Arc::new(Registry::default());
13676        let forwarding = Arc::new(ForwardingTable::default());
13677        let handler =
13678            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13679                .with_health_probe_timeout(Duration::from_millis(50));
13680        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13681        hello_via_sink(
13682            &handler,
13683            &module_ctx,
13684            &mut module_rx,
13685            hello_frame_with_control_ops(
13686                "aft",
13687                PROTOCOL_VERSION,
13688                7,
13689                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13690            ),
13691        )
13692        .await;
13693
13694        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13695        let responses = handler
13696            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13697            .await
13698            .unwrap();
13699        assert_eq!(responses[0].header.ty, FrameType::Error);
13700        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13701        let _ = module_rx.try_recv();
13702
13703        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13704        let health_handler = handler.clone();
13705        let death_task = tokio::spawn(async move {
13706            health_handler
13707                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13708                .await
13709                .unwrap()
13710        });
13711        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13712            .await
13713            .unwrap()
13714            .unwrap();
13715        handler
13716            .cleanup_connection(module_ctx.connection_id)
13717            .unwrap();
13718        let responses = death_task.await.unwrap();
13719        assert_eq!(responses[0].header.ty, FrameType::Error);
13720        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13721    }
13722
13723    #[test]
13724    fn hello_requires_exact_protocol_version() {
13725        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13726            let registry = Arc::new(Registry::default());
13727            let handler = ControlHandler::new(Arc::clone(&registry));
13728            let responses = handler
13729                .handle_control(
13730                    ConnectionId::new(connection),
13731                    hello_frame("aft", offered, 9),
13732                )
13733                .unwrap();
13734
13735            assert_eq!(responses.len(), 1);
13736            assert_eq!(responses[0].header.ty, FrameType::Error);
13737            let error = parse_error(&responses[0]);
13738            assert_eq!(error["code"], "version_unsupported");
13739            assert!(registry.get_module("aft").unwrap().is_none());
13740            assert_eq!(registry.active_registration_count().unwrap(), 0);
13741        }
13742    }
13743
13744    #[test]
13745    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13746        let registry = Arc::new(Registry::default());
13747        let forwarding = Arc::new(ForwardingTable::default());
13748        let handler =
13749            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13750        let module_connection = ConnectionId::new(301);
13751        let registration = registry
13752            .register_with_control_ops(
13753                manifest("aft-push", PROTOCOL_VERSION),
13754                PROTOCOL_VERSION,
13755                module_connection,
13756                module_baseline_control_ops(),
13757            )
13758            .unwrap();
13759        let (module_tx, _module_rx) = mpsc::channel(8);
13760        let endpoint = forwarding
13761            .register_module_connection(
13762                module_connection,
13763                "aft-push".to_string(),
13764                PROTOCOL_VERSION,
13765                manifest_concurrency(&registration.manifest),
13766                FrameSink::new(module_tx),
13767            )
13768            .unwrap();
13769
13770        // A push op this version does not know is ignored (forward-compat), not errored.
13771        let unknown = Frame::build(
13772            FrameType::Push,
13773            control_flags(),
13774            0,
13775            0,
13776            5,
13777            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13778        )
13779        .unwrap();
13780        let out = handler.handle_status_update(endpoint, unknown).unwrap();
13781        assert!(
13782            out.is_empty(),
13783            "unknown push op must be ignored, got {out:?}"
13784        );
13785
13786        // A malformed body for a KNOWN op is a real error worth surfacing.
13787        let malformed = Frame::build(
13788            FrameType::Push,
13789            control_flags(),
13790            0,
13791            0,
13792            6,
13793            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13794        )
13795        .unwrap();
13796        let out = handler.handle_status_update(endpoint, malformed).unwrap();
13797        assert_eq!(out.len(), 1);
13798        assert_eq!(out[0].header.ty, FrameType::Error);
13799        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13800    }
13801
13802    #[test]
13803    fn hello_rejected_when_connection_already_owns_client_routes() {
13804        let registry = Arc::new(Registry::default());
13805        let forwarding = Arc::new(ForwardingTable::default());
13806        let handler =
13807            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13808        // Commits a client route on connection 202 (bound to a module on conn 101).
13809        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
13810        let client_connection = ConnectionId::new(202);
13811
13812        // That same connection now tries to register as a module: rejected, so one
13813        // connection never holds both client-route and module-endpoint state.
13814        let responses = handler
13815            .handle_control(
13816                client_connection,
13817                hello_frame("aft-second", PROTOCOL_VERSION, 9),
13818            )
13819            .unwrap();
13820        assert_eq!(responses[0].header.ty, FrameType::Error);
13821        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13822        assert!(registry.get_module("aft-second").unwrap().is_none());
13823    }
13824
13825    #[tokio::test]
13826    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13827        let registry = Arc::new(Registry::default());
13828        let forwarding = Arc::new(ForwardingTable::default());
13829        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13830        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13831        hello_via_sink(
13832            &handler,
13833            &module_ctx,
13834            &mut module_rx,
13835            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13836        )
13837        .await;
13838        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13839        let pending = forwarding
13840            .begin_route_bind_relay_for_test(
13841                client_ctx.connection_id,
13842                client_ctx.egress.clone(),
13843                2,
13844                "alpha",
13845            )
13846            .unwrap();
13847        forwarding
13848            .complete_pending_relay(
13849                module_ctx.connection_id,
13850                pending.corr,
13851                RouteBindRelayOutcome::Accepted,
13852            )
13853            .unwrap();
13854        client_rx.try_recv().unwrap();
13855        for module_id in ["beta", "alpha"] {
13856            let replies = handler
13857                .handle_control_frame(
13858                    &module_ctx,
13859                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13860                )
13861                .await
13862                .unwrap();
13863            assert_eq!(replies.len(), 1, "second HELLO must be refused");
13864            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13865        }
13866        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13867        assert!(registry.get_module("beta").unwrap().is_none());
13868        assert!(matches!(
13869            forwarding
13870                .lookup_data_route(
13871                    client_ctx.connection_id,
13872                    pending.client_channel,
13873                    pending.client_epoch,
13874                )
13875                .unwrap(),
13876            DataRoute::Client(DataRouteState::Bound(_))
13877        ));
13878        assert!(handler
13879            .hello_launch_nonces
13880            .lock()
13881            .unwrap()
13882            .presented(module_ctx.connection_id, Some("alpha-nonce")));
13883        assert!(module_rx.try_recv().is_err());
13884    }
13885
13886    #[test]
13887    fn reserved_module_hello_requires_matching_launch_nonce() {
13888        let registry = Arc::new(Registry::default());
13889        let supervisor = SupervisorHandle::new();
13890        // The supervisor recorded the nonce it injected when it spawned the reserved
13891        // module; the HELLO verifier checks against the same shared handle.
13892        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13893        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13894
13895        // A HELLO with NO nonce is rejected.
13896        let no_nonce = handler
13897            .handle_control(
13898                ConnectionId::new(1),
13899                hello_frame("vault", PROTOCOL_VERSION, 1),
13900            )
13901            .unwrap();
13902        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13903        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13904        assert!(registry.get_module("vault").unwrap().is_none());
13905
13906        // A HELLO with the WRONG nonce is rejected.
13907        let wrong = handler
13908            .handle_control(
13909                ConnectionId::new(2),
13910                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13911            )
13912            .unwrap();
13913        assert_eq!(wrong[0].header.ty, FrameType::Error);
13914        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13915        assert!(registry.get_module("vault").unwrap().is_none());
13916
13917        // A HELLO with the CORRECT nonce registers.
13918        let ok = handler
13919            .handle_control(
13920                ConnectionId::new(3),
13921                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13922            )
13923            .unwrap();
13924        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13925        assert!(registry.get_module("vault").unwrap().is_some());
13926    }
13927
13928    #[test]
13929    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13930        let registry = Arc::new(Registry::default());
13931        let supervisor = SupervisorHandle::new();
13932        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13933        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13934        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13935
13936        let squat = handler
13937            .handle_control(
13938                ConnectionId::new(1),
13939                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13940            )
13941            .unwrap();
13942        assert_eq!(squat[0].header.ty, FrameType::Error);
13943        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13944        assert!(parse_error(&squat[0])["message"]
13945            .as_str()
13946            .unwrap()
13947            .contains("fed:"));
13948
13949        let accepted_peer = handler
13950            .handle_control(
13951                ConnectionId::new(2),
13952                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13953            )
13954            .unwrap();
13955        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13956
13957        let accepted_short = handler
13958            .handle_control(
13959                ConnectionId::new(3),
13960                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13961            )
13962            .unwrap();
13963        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13964
13965        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13966            let response = handler
13967                .handle_control(
13968                    ConnectionId::new(conn),
13969                    hello_frame(module_id, PROTOCOL_VERSION, conn),
13970                )
13971                .unwrap();
13972            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13973        }
13974    }
13975
13976    #[test]
13977    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13978        let registry = Arc::new(Registry::default());
13979        let supervisor = SupervisorHandle::new();
13980        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13981        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13982        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13983        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13984
13985        let owner_nonce = handler
13986            .handle_control(
13987                ConnectionId::new(1),
13988                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13989            )
13990            .unwrap();
13991        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13992        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13993        assert!(registry.get_module("fed:special").unwrap().is_none());
13994
13995        let exact_nonce = handler
13996            .handle_control(
13997                ConnectionId::new(2),
13998                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13999            )
14000            .unwrap();
14001        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14002        assert!(registry.get_module("fed:special").unwrap().is_some());
14003    }
14004
14005    #[test]
14006    fn non_reserved_module_ignores_launch_nonce() {
14007        let registry = Arc::new(Registry::default());
14008        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
14009        // registration succeeds whether a spawned process echoes a nonce or not.
14010        let handler = ControlHandler::new(Arc::clone(&registry));
14011        let no_nonce = handler
14012            .handle_control(
14013                ConnectionId::new(1),
14014                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14015            )
14016            .unwrap();
14017        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14018        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14019
14020        let echoed_nonce = handler
14021            .handle_control(
14022                ConnectionId::new(2),
14023                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14024            )
14025            .unwrap();
14026        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14027        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14028    }
14029
14030    #[test]
14031    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14032        let handler = ControlHandler::default();
14033        let conn = ConnectionId::new(1);
14034        let malformed = Frame::build(
14035            FrameType::Hello,
14036            control_flags(),
14037            0,
14038            0,
14039            3,
14040            b"{not json".to_vec(),
14041        )
14042        .unwrap();
14043
14044        let error = handler.handle_control(conn, malformed).unwrap();
14045        assert_eq!(error[0].header.ty, FrameType::Error);
14046        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14047
14048        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14049        let pong = handler.handle_control(conn, ping).unwrap();
14050        assert_eq!(pong[0].header.ty, FrameType::Pong);
14051        assert_eq!(pong[0].header.corr, 4);
14052    }
14053
14054    #[test]
14055    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14056        let registry = Arc::new(Registry::default());
14057        let handler = ControlHandler::new(Arc::clone(&registry));
14058
14059        handler
14060            .handle_control(
14061                ConnectionId::new(1),
14062                hello_frame("aft", PROTOCOL_VERSION, 1),
14063            )
14064            .unwrap();
14065        let duplicate = handler
14066            .handle_control(
14067                ConnectionId::new(2),
14068                hello_frame("aft", PROTOCOL_VERSION, 2),
14069            )
14070            .unwrap();
14071
14072        assert_eq!(duplicate[0].header.ty, FrameType::Error);
14073        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14074        let registration = registry.get_module("aft").unwrap().unwrap();
14075        assert_eq!(registration.connection_id, ConnectionId::new(1));
14076    }
14077
14078    #[test]
14079    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14080        let registry = Arc::new(Registry::default());
14081        let forwarding = Arc::new(ForwardingTable::default());
14082        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14083        let handler =
14084            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14085                .with_process_liveness(process_liveness);
14086        let (ctx, route_channel, route_epoch) =
14087            bind_liveness_route(&registry, &forwarding, "aft-dead");
14088        let responses = handler
14089            .handle_route_poll(
14090                &ctx,
14091                route_poll_frame(41, PollKind::Liveness, route_channel),
14092                route_channel,
14093                route_epoch,
14094                PollKind::Liveness,
14095            )
14096            .unwrap();
14097
14098        assert_eq!(responses.len(), 1);
14099        assert_eq!(responses[0].header.ty, FrameType::Response);
14100        assert_route_poll_liveness(&responses[0], false);
14101    }
14102
14103    #[test]
14104    fn liveness_poll_without_process_source_uses_bound_route() {
14105        let registry = Arc::new(Registry::default());
14106        let forwarding = Arc::new(ForwardingTable::default());
14107        let handler =
14108            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14109        let (ctx, route_channel, route_epoch) =
14110            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
14111        let responses = handler
14112            .handle_route_poll(
14113                &ctx,
14114                route_poll_frame(42, PollKind::Liveness, route_channel),
14115                route_channel,
14116                route_epoch,
14117                PollKind::Liveness,
14118            )
14119            .unwrap();
14120
14121        assert_route_poll_liveness(&responses[0], true);
14122    }
14123
14124    #[test]
14125    fn liveness_poll_untracked_process_source_uses_bound_route() {
14126        let registry = Arc::new(Registry::default());
14127        let forwarding = Arc::new(ForwardingTable::default());
14128        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14129        let handler =
14130            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14131                .with_process_liveness(process_liveness);
14132        let (ctx, route_channel, route_epoch) =
14133            bind_liveness_route(&registry, &forwarding, "aft-untracked");
14134        let responses = handler
14135            .handle_route_poll(
14136                &ctx,
14137                route_poll_frame(43, PollKind::Liveness, route_channel),
14138                route_channel,
14139                route_epoch,
14140                PollKind::Liveness,
14141            )
14142            .unwrap();
14143
14144        assert_route_poll_liveness(&responses[0], true);
14145    }
14146
14147    #[tokio::test]
14148    async fn unknown_op_returns_unknown_control_op() {
14149        let handler = ControlHandler::default();
14150        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14151        let request = Frame::build(
14152            FrameType::Request,
14153            control_flags(),
14154            0,
14155            0,
14156            55,
14157            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14158        )
14159        .unwrap();
14160
14161        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14162
14163        assert_eq!(response.len(), 1);
14164        assert_eq!(response[0].header.ty, FrameType::Error);
14165        assert_eq!(response[0].header.corr, 55);
14166        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14167    }
14168
14169    #[tokio::test]
14170    async fn supervisor_provenance_rejects_unknown_exact_module() {
14171        let handler = ControlHandler::default();
14172        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14173        let request = Frame::build(
14174            FrameType::Request,
14175            control_flags(),
14176            0,
14177            0,
14178            57,
14179            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14180        )
14181        .unwrap();
14182
14183        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14184
14185        assert_eq!(response.len(), 1);
14186        assert_eq!(response[0].header.ty, FrameType::Error);
14187        assert_eq!(response[0].header.corr, 57);
14188        let error = parse_error(&response[0]);
14189        assert_eq!(error["code"], "unknown_module");
14190        assert_eq!(error["message"], "module_id 'missing' is not supervised");
14191    }
14192
14193    #[test]
14194    fn provenance_probe_override_keeps_handler_tests_deterministic() {
14195        let expected = subc_control::RunningImageAgreement::Unavailable {
14196            reason: subc_control::RunningImageUnavailableReason::HashFailed,
14197        };
14198        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14199        assert_eq!(handler.provenance_probe_override, Some(expected));
14200    }
14201
14202    #[test]
14203    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14204        let verdict = reload_verdict(
14205            std::path::Path::new("/bin/new"),
14206            Some(std::path::Path::new("/bin/old")),
14207            subc_control::RunningImageAgreement::Unavailable {
14208                reason: subc_control::RunningImageUnavailableReason::HashFailed,
14209            },
14210        );
14211        assert!(matches!(
14212            verdict.path,
14213            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14214                if configured == std::path::Path::new("/bin/new")
14215                    && spawned_from == std::path::Path::new("/bin/old")
14216        ));
14217    }
14218
14219    #[test]
14220    fn launch_spec_without_running_digest_is_unavailable_not_match() {
14221        use subc_control::{
14222            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14223        };
14224        let configured = crate::supervise::ModuleSpec {
14225            module_id: "legacy".into(),
14226            program: "/module".into(),
14227            args: Vec::new(),
14228            env: Vec::new(),
14229            reserved: false,
14230            reserved_prefixes: Vec::new(),
14231            protocol: subc_control::ModuleProtocol::Subc,
14232            overlap: Default::default(),
14233        };
14234        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14235        assert_eq!(
14236            launch_spec_agreement(&configured, false, Some(&digest)),
14237            Agreement::Unavailable {
14238                reason: Reason::NotRunning
14239            }
14240        );
14241        assert_eq!(
14242            launch_spec_agreement(&configured, true, None),
14243            Agreement::Unavailable {
14244                reason: Reason::SpawnedSpecUnavailable
14245            }
14246        );
14247        assert_eq!(
14248            launch_spec_agreement(&configured, true, Some(&digest)),
14249            Agreement::Match
14250        );
14251    }
14252
14253    #[cfg(windows)]
14254    #[test]
14255    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14256        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14257        let configured = dir.join("module.exe");
14258        std::fs::write(&configured, b"executable path fixture").unwrap();
14259        let spawned = std::fs::canonicalize(&configured).unwrap();
14260        assert_ne!(
14261            configured, spawned,
14262            "fixture must exercise different path spellings"
14263        );
14264        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14265        let lower_case = configured.to_string_lossy().to_lowercase();
14266        for spelling in [
14267            &configured,
14268            Path::new(&forward_slashes),
14269            Path::new(&lower_case),
14270        ] {
14271            let verdict = reload_verdict(
14272                spelling,
14273                Some(&spawned),
14274                subc_control::RunningImageAgreement::Unavailable {
14275                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14276                },
14277            );
14278            assert_eq!(
14279                verdict.path,
14280                ReloadPathAgreement::Match,
14281                "configured {spelling:?}, spawned {spawned:?}"
14282            );
14283        }
14284    }
14285
14286    #[cfg(windows)]
14287    #[test]
14288    fn windows_reload_path_keeps_different_existing_program_pending() {
14289        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14290        let old = dir.join("old.exe");
14291        let new = dir.join("new.exe");
14292        std::fs::write(&old, b"same executable content").unwrap();
14293        std::fs::copy(&old, &new).unwrap();
14294        let spawned = std::fs::canonicalize(&old).unwrap();
14295        let image = subc_control::RunningImageAgreement::Unavailable {
14296            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14297        };
14298        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14299        assert!(
14300            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14301            if configured == new && spawned_from == spawned)
14302        );
14303        let missing = dir.join("missing.exe");
14304        assert!(matches!(
14305            reload_verdict(&missing, Some(&spawned), image).path,
14306            ReloadPathAgreement::Mismatch { .. }
14307        ));
14308    }
14309
14310    #[test]
14311    fn reload_verdict_detects_replaced_image_at_same_path() {
14312        let image = subc_control::RunningImageAgreement::Mismatch {
14313            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14314                digest: "old".into(),
14315            },
14316            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14317                digest: "new".into(),
14318            },
14319        };
14320        let verdict = reload_verdict(
14321            std::path::Path::new("/bin/same"),
14322            Some(std::path::Path::new("/bin/same")),
14323            image.clone(),
14324        );
14325        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14326        assert_eq!(verdict.image, image);
14327    }
14328
14329    #[test]
14330    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14331        let image = subc_control::RunningImageAgreement::Unavailable {
14332            reason: subc_control::RunningImageUnavailableReason::NotRunning,
14333        };
14334        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14335        assert_eq!(
14336            verdict.path,
14337            subc_control::ReloadPathAgreement::Unavailable {
14338                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14339            }
14340        );
14341        assert_eq!(verdict.image, image);
14342
14343        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14344            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14345        };
14346        let verdict = reload_verdict(
14347            std::path::Path::new("/bin/same"),
14348            Some(std::path::Path::new("/bin/same")),
14349            unconfirmed.clone(),
14350        );
14351        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14352        assert_eq!(verdict.image, unconfirmed);
14353    }
14354
14355    #[test]
14356    fn reload_verdict_preserves_each_image_unavailability_reason() {
14357        use subc_control::RunningImageUnavailableReason as Reason;
14358
14359        for reason in [
14360            Reason::NotRunning,
14361            Reason::UnsupportedPlatform,
14362            Reason::RunningExecutableUnreadable,
14363            Reason::SpawnedPathUnreadable,
14364            Reason::HashFailed,
14365            Reason::ProcessIdentityUnconfirmed,
14366            Reason::Unknown("future_probe_reason".to_string()),
14367        ] {
14368            let image = subc_control::RunningImageAgreement::Unavailable {
14369                reason: reason.clone(),
14370            };
14371            let verdict = reload_verdict(
14372                std::path::Path::new("/bin/same"),
14373                Some(std::path::Path::new("/bin/same")),
14374                image.clone(),
14375            );
14376            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14377            assert_eq!(verdict.image, image, "{reason:?}");
14378        }
14379    }
14380
14381    #[tokio::test]
14382    async fn malformed_control_bodies_return_invalid_control_body() {
14383        let handler = ControlHandler::default();
14384        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14385
14386        for (corr, body) in [
14387            (56, br#"{"route_channel":1}"#.as_slice()),
14388            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14389            (
14390                58,
14391                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14392            ),
14393        ] {
14394            let request = Frame::build(
14395                FrameType::Request,
14396                control_flags(),
14397                0,
14398                0,
14399                corr,
14400                body.to_vec(),
14401            )
14402            .unwrap();
14403            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14404
14405            assert_eq!(response.len(), 1);
14406            assert_eq!(response[0].header.ty, FrameType::Error);
14407            assert_eq!(response[0].header.corr, corr);
14408            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14409        }
14410    }
14411
14412    #[tokio::test]
14413    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14414        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14415        let registry = Arc::new(Registry::default());
14416        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14417        let router = Router::with_control_handler(Arc::clone(&control));
14418        let connection = router.begin_connection();
14419        let (ctx, mut rx) = route_ctx(connection.id());
14420
14421        router
14422            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14423            .await
14424            .unwrap();
14425        let response = rx.recv().await.unwrap();
14426        let ack = parse_ack(&response);
14427        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14428        let channel = 1;
14429
14430        let goodbye =
14431            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14432        router.route_for_connection(&ctx, goodbye).await.unwrap();
14433        assert!(rx.try_recv().is_err());
14434        assert!(registry.get_module("aft").unwrap().is_none());
14435
14436        router
14437            .route_for_connection(&ctx, channel_request(channel, 13))
14438            .await
14439            .unwrap();
14440        let error_frame = rx.recv().await.unwrap();
14441        assert_eq!(error_frame.header.ty, FrameType::Error);
14442        assert_eq!(error_frame.header.channel, channel);
14443        let captured = crate::router::test_log::captured_logs(&logs);
14444        assert_eq!(
14445            captured
14446                .lines()
14447                .filter(|line| {
14448                    line.contains("module_id=aft")
14449                        && line.contains("reason=explicit_goodbye")
14450                        && line.contains("module registration ended")
14451                })
14452                .count(),
14453            1,
14454            "unexpected GOODBYE registry log: {captured}"
14455        );
14456    }
14457
14458    #[tokio::test]
14459    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14460        let registry = Arc::new(Registry::default());
14461        let handler = ControlHandler::new(registry).with_capability_config(
14462            [("prov".to_string(), true), ("cons".to_string(), true)],
14463            BTreeMap::new(),
14464        );
14465        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14466        register_capability_manifest(
14467            &handler,
14468            &provider_ctx,
14469            &mut provider_rx,
14470            capability_manifest("prov", &["thing/v1"], &[]),
14471            1,
14472        )
14473        .await;
14474        let mut consumer = capability_manifest("cons", &[], &[]);
14475        consumer.capabilities.as_mut().unwrap().requires.push(
14476            subc_protocol::manifest::CapabilityRequirement {
14477                capability: "thing/v1".to_string(),
14478                need: subc_protocol::manifest::CapabilityNeed::Required,
14479            },
14480        );
14481        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14482        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14483        assert_eq!(
14484            handler.capability_evaluator.verdict("cons", "thing/v1"),
14485            Some(CapabilityVerdict::Provided)
14486        );
14487        let (mut client_rx, _) = open_route_for_capability_test(
14488            &handler,
14489            &provider_ctx,
14490            &mut provider_rx,
14491            703,
14492            3,
14493            "prov",
14494            None,
14495        )
14496        .await;
14497        let goodbye =
14498            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14499        handler
14500            .handle_control_frame(&provider_ctx, goodbye)
14501            .await
14502            .unwrap();
14503        assert_eq!(
14504            handler.capability_evaluator.verdict("cons", "thing/v1"),
14505            Some(CapabilityVerdict::NeverProvided)
14506        );
14507        let closed = client_rx
14508            .try_recv()
14509            .expect("GOODBYE pushes route.closed before route GOODBYE");
14510        assert!(
14511            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14512            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14513        );
14514        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14515        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14516    }
14517
14518    struct RouteClosedFixture {
14519        handler: ControlHandler,
14520        #[cfg(unix)]
14521        supervisor: Supervisor,
14522        module_ctx: RouteCtx,
14523        #[cfg(unix)]
14524        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14525        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14526        route: Arc<crate::forwarding::RouteBinding>,
14527    }
14528
14529    async fn route_closed_fixture() -> RouteClosedFixture {
14530        let registry = Arc::new(Registry::default());
14531        let handler = ControlHandler::new(Arc::clone(&registry));
14532        #[cfg(unix)]
14533        let supervisor = Supervisor::new(registry, RestartPolicy::default())
14534            .with_forwarding(Arc::clone(&handler.forwarding));
14535        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14536        register_capability_manifest(
14537            &handler,
14538            &module_ctx,
14539            &mut module_rx,
14540            capability_manifest("prov", &[], &[]),
14541            1,
14542        )
14543        .await;
14544        let (client_rx, bind) = open_route_for_capability_test(
14545            &handler,
14546            &module_ctx,
14547            &mut module_rx,
14548            702,
14549            2,
14550            "prov",
14551            None,
14552        )
14553        .await;
14554        let ModuleControlRequest::RouteBind {
14555            route_channel: channel,
14556            epoch: route_epoch,
14557            ..
14558        } = bind
14559        else {
14560            panic!("expected route.bind");
14561        };
14562        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14563            handler
14564                .forwarding
14565                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14566                .unwrap()
14567        else {
14568            panic!("the acknowledged route must be bound");
14569        };
14570        RouteClosedFixture {
14571            handler,
14572            #[cfg(unix)]
14573            supervisor,
14574            module_ctx,
14575            #[cfg(unix)]
14576            module_rx,
14577            client_rx,
14578            route,
14579        }
14580    }
14581
14582    impl RouteClosedFixture {
14583        #[cfg(unix)]
14584        async fn cut(&mut self) {
14585            let notices = async {
14586                let mut draining = self.module_rx.recv().await.unwrap();
14587                assert!(matches!(
14588                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14589                        &draining.body
14590                    )
14591                    .unwrap(),
14592                    subc_protocol::session::ModuleControlCommand::Draining {
14593                        reason: RouteCloseReason::Restart,
14594                        ..
14595                    }
14596                ));
14597                draining.flushed.take().unwrap().send(()).unwrap();
14598                let mut closing = self.client_rx.recv().await.unwrap();
14599                assert!(matches!(
14600                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14601                    ClientControlPush::RouteClosing {
14602                        reason: RouteCloseReason::Restart,
14603                        ..
14604                    }
14605                ));
14606                closing.flushed.take().unwrap().send(()).unwrap();
14607            };
14608            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14609            result.unwrap();
14610        }
14611
14612        fn disconnect(&mut self) -> ClientControlPush {
14613            self.handler
14614                .cleanup_connection(self.module_ctx.connection_id)
14615                .unwrap();
14616            let closed = self.client_rx.try_recv().unwrap();
14617            assert_eq!(closed.header.ty, FrameType::Push);
14618            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14619            assert!(matches!(
14620                &push,
14621                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14622                    if channels == &[self.route.client_channel]
14623            ));
14624            assert_eq!(
14625                self.client_rx.try_recv().unwrap().header.ty,
14626                FrameType::Goodbye
14627            );
14628            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14629            push
14630        }
14631    }
14632
14633    #[cfg(unix)]
14634    #[tokio::test(start_paused = true)]
14635    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14636        let mut fixture = route_closed_fixture().await;
14637        fixture.cut().await;
14638        assert!(matches!(
14639            fixture.disconnect(),
14640            ClientControlPush::RouteClosed {
14641                reason: RouteCloseReason::Restart,
14642                drained: true,
14643                ..
14644            }
14645        ));
14646    }
14647
14648    #[cfg(unix)]
14649    #[tokio::test(start_paused = true)]
14650    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14651        let mut fixture = route_closed_fixture().await;
14652        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14653        let started = tokio::time::Instant::now();
14654        fixture.cut().await;
14655        assert!(started.elapsed() >= Duration::from_secs(2));
14656        assert_eq!(fixture.route.flow.in_flight(), 1);
14657        assert!(matches!(
14658            fixture.disconnect(),
14659            ClientControlPush::RouteClosed {
14660                reason: RouteCloseReason::Restart,
14661                drained: false,
14662                ..
14663            }
14664        ));
14665    }
14666
14667    #[tokio::test]
14668    async fn module_connection_crash_reports_crash_and_not_drained() {
14669        let mut fixture = route_closed_fixture().await;
14670        // The server calls cleanup_connection on module EOF. Without a planned
14671        // daemon shutdown, that disconnect must remain a crash with no prior
14672        // route.closing notice, even when the route has no in-flight calls.
14673        assert!(matches!(
14674            fixture.disconnect(),
14675            ClientControlPush::RouteClosed {
14676                reason: RouteCloseReason::Crash,
14677                drained: false,
14678                ..
14679            }
14680        ));
14681    }
14682
14683    #[tokio::test]
14684    async fn dropping_router_connection_releases_registration() {
14685        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14686        let registry = Arc::new(Registry::default());
14687        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14688        let router = Router::with_control_handler(Arc::clone(&control));
14689        let connection = router.begin_connection();
14690        let connection_id = connection.id();
14691        let (ctx, mut rx) = route_ctx(connection_id);
14692
14693        router
14694            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14695            .await
14696            .unwrap();
14697        let response = rx.recv().await.unwrap();
14698        let ack = parse_ack(&response);
14699        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14700        assert!(registry.get_module("aft").unwrap().is_some());
14701
14702        drop(connection);
14703
14704        assert!(registry.get_module("aft").unwrap().is_none());
14705        assert_eq!(registry.active_registration_count().unwrap(), 0);
14706
14707        control
14708            .cleanup_connection(ConnectionId::new(u64::MAX))
14709            .unwrap();
14710        let captured = crate::router::test_log::captured_logs(&logs);
14711        println!("captured registry lifecycle logs:\n{captured}");
14712        let events: Vec<_> = captured
14713            .lines()
14714            .filter(|line| {
14715                line.contains("module registered module_id=aft ")
14716                    || line.contains("module registration ended")
14717            })
14718            .collect();
14719        assert_eq!(
14720            events.len(),
14721            2,
14722            "unexpected registry lifecycle logs: {captured}"
14723        );
14724        assert!(events[0].contains("module registered module_id=aft "));
14725        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14726        assert!(events[1].contains(&format!(
14727            "module_id=aft connection_id={}",
14728            connection_id.get()
14729        )));
14730        assert!(events[1].contains("reason=connection_closed"));
14731        assert!(events[1].contains("module registration ended"));
14732        assert_eq!(
14733            captured
14734                .lines()
14735                .filter(|line| line.contains("module registered module_id=aft "))
14736                .count(),
14737            1,
14738            "legacy registration admission line must appear once: {captured}"
14739        );
14740    }
14741
14742    #[tokio::test]
14743    async fn hello_registration_keeps_legacy_module_registered_line_once() {
14744        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14745        let registry = Arc::new(Registry::default());
14746        let control = ControlHandler::new(Arc::clone(&registry));
14747        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14748        hello_via_sink(
14749            &control,
14750            &ctx,
14751            &mut rx,
14752            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14753        )
14754        .await;
14755
14756        let captured = crate::router::test_log::captured_logs(&logs);
14757        let admissions: Vec<_> = captured
14758            .lines()
14759            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14760            .collect();
14761        assert_eq!(
14762            admissions.len(),
14763            1,
14764            "legacy admission line must remain exactly once: {captured}"
14765        );
14766        assert!(admissions[0].contains("routable_provider=true"));
14767        assert!(admissions[0].contains("connection_id=777"));
14768    }
14769
14770    fn capability_manifest(
14771        module_id: &str,
14772        provides: &[&str],
14773        must_never_reach: &[&str],
14774    ) -> ModuleManifest {
14775        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14776        manifest.capabilities = Some(CapabilityDeclarations {
14777            provides: provides
14778                .iter()
14779                .map(|capability| (*capability).to_string())
14780                .collect(),
14781            requires: Vec::new(),
14782            must_never_reach: must_never_reach
14783                .iter()
14784                .map(|capability| (*capability).to_string())
14785                .collect(),
14786        });
14787        manifest
14788    }
14789
14790    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14791        Frame::build(
14792            FrameType::Hello,
14793            control_flags(),
14794            0,
14795            0,
14796            corr,
14797            serde_json::to_vec(&ModuleHelloBody {
14798                protocol_ver: manifest.protocol_ver,
14799                manifest,
14800                control_ops: None,
14801                launch_nonce: None,
14802            })
14803            .expect("capability test HELLO serializes"),
14804        )
14805        .expect("capability test HELLO frame builds")
14806    }
14807
14808    fn catalog_update_with_capabilities_frame(
14809        corr: u64,
14810        capabilities: CapabilityDeclarations,
14811    ) -> Frame {
14812        Frame::build(
14813            FrameType::Request,
14814            control_flags(),
14815            0,
14816            0,
14817            corr,
14818            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14819                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14820                capabilities: Some(capabilities),
14821                ready: None,
14822            })
14823            .expect("capability catalog.update serializes"),
14824        )
14825        .expect("capability catalog.update frame builds")
14826    }
14827
14828    async fn register_capability_manifest(
14829        handler: &ControlHandler,
14830        ctx: &RouteCtx,
14831        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14832        manifest: ModuleManifest,
14833        corr: u64,
14834    ) {
14835        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14836    }
14837
14838    async fn open_route_for_capability_test(
14839        handler: &ControlHandler,
14840        target_ctx: &RouteCtx,
14841        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14842        client_connection_id: u64,
14843        corr: u64,
14844        target_module_id: &str,
14845        consumer_identity: Option<ConsumerIdentity>,
14846    ) -> (
14847        mpsc::Receiver<crate::router::OutboundFrame>,
14848        ModuleControlRequest,
14849    ) {
14850        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14851        let route_handler = handler.clone();
14852        let target_module_id = target_module_id.to_string();
14853        let route_task = tokio::spawn(async move {
14854            route_handler
14855                .handle_control_frame(
14856                    &client_ctx,
14857                    route_open_frame_with_admission_facts(
14858                        corr,
14859                        &target_module_id,
14860                        unique_project_root("admission-facts"),
14861                        consumer_identity,
14862                        None,
14863                    ),
14864                )
14865                .await
14866                .expect("capability test route.open succeeds")
14867        });
14868        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14869            .await
14870            .expect("capability test route.open must reach route.bind")
14871            .expect("target control receiver stays open");
14872        let bind_request: ModuleControlRequest =
14873            serde_json::from_slice(&bind.body).expect("route.bind decodes");
14874        handler
14875            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14876            .await
14877            .expect("capability test route.bind ACK succeeds");
14878        assert!(route_task.await.expect("route.open task joins").is_empty());
14879        let opened = client_rx
14880            .recv()
14881            .await
14882            .expect("successful route.open publishes a response");
14883        assert!(matches!(
14884            serde_json::from_slice::<ClientControlResponse>(&opened.body),
14885            Ok(ClientControlResponse::RouteOpen { .. })
14886        ));
14887        (client_rx, bind_request)
14888    }
14889
14890    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14891        assert_eq!(frame.header.ty, FrameType::Push);
14892        assert_eq!(frame.header.channel, 0);
14893        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14894            .expect("route.closed control push decodes");
14895        let ClientControlPush::RouteClosed { channels, .. } = &push else {
14896            panic!("expected route.closed");
14897        };
14898        assert_eq!(channels.len(), 1, "exactly one violating route closed");
14899        let channels = channels.clone();
14900        assert_eq!(
14901            push,
14902            ClientControlPush::RouteClosed {
14903                module_id: target_module_id.to_string(),
14904                channels,
14905                reason: RouteCloseReason::CapabilityDenied,
14906                drained: false,
14907                abandoned: 0,
14908                excluded_subscriptions: 0,
14909                terminal: Some(false),
14910            }
14911        );
14912    }
14913
14914    #[tokio::test]
14915    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14916        let registry = Arc::new(Registry::default());
14917        let forwarding = Arc::new(ForwardingTable::default());
14918        let supervisor = SupervisorHandle::new();
14919        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14920        let handler =
14921            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14922                .with_supervisor(supervisor);
14923        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14924        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14925        register_capability_manifest(
14926            &handler,
14927            &target_ctx,
14928            &mut target_rx,
14929            capability_manifest("target", &["credentials-provider/v1"], &[]),
14930            1,
14931        )
14932        .await;
14933        register_capability_manifest(
14934            &handler,
14935            &opener_ctx,
14936            &mut opener_rx,
14937            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14938            2,
14939        )
14940        .await;
14941
14942        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14943        let replies = handler
14944            .handle_control_frame(
14945                &client_ctx,
14946                route_open_frame_with_admission_facts(
14947                    3,
14948                    "target",
14949                    unique_project_root("admission-facts"),
14950                    Some(ConsumerIdentity {
14951                        module_id: "opener".to_string(),
14952                        launch_nonce: "opener-nonce".to_string(),
14953                    }),
14954                    None,
14955                ),
14956            )
14957            .await
14958            .expect("denied route.open returns a typed frame");
14959        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14960        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14961        assert!(
14962            target_rx.try_recv().is_err(),
14963            "forbidden route.open must not relay route.bind"
14964        );
14965    }
14966
14967    #[tokio::test]
14968    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14969        let registry = Arc::new(Registry::default());
14970        let forwarding = Arc::new(ForwardingTable::default());
14971        let supervisor = SupervisorHandle::new();
14972        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14973        let handler =
14974            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14975                .with_supervisor(supervisor);
14976        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14977        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14978        register_capability_manifest(
14979            &handler,
14980            &target_ctx,
14981            &mut target_rx,
14982            capability_manifest("target", &["credentials-provider/v1"], &[]),
14983            1,
14984        )
14985        .await;
14986        register_capability_manifest(
14987            &handler,
14988            &old_opener_ctx,
14989            &mut old_opener_rx,
14990            capability_manifest("opener", &[], &[]),
14991            2,
14992        )
14993        .await;
14994        let (mut client_rx, _) = open_route_for_capability_test(
14995            &handler,
14996            &target_ctx,
14997            &mut target_rx,
14998            712,
14999            3,
15000            "target",
15001            Some(ConsumerIdentity {
15002                module_id: "opener".to_string(),
15003                launch_nonce: "opener-nonce".to_string(),
15004            }),
15005        )
15006        .await;
15007        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15008
15009        handler
15010            .cleanup_connection(old_opener_ctx.connection_id)
15011            .expect("old opener registration cleans up");
15012        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15013        register_capability_manifest(
15014            &handler,
15015            &new_opener_ctx,
15016            &mut new_opener_rx,
15017            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15018            4,
15019        )
15020        .await;
15021
15022        assert_capability_denied_push(
15023            client_rx
15024                .try_recv()
15025                .expect("HELLO deny addition must emit route.closed")
15026                .frame,
15027            "target",
15028        );
15029        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15030        assert!(matches!(
15031            target_rx.try_recv(),
15032            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15033        ));
15034    }
15035
15036    #[tokio::test]
15037    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15038        let registry = Arc::new(Registry::default());
15039        let forwarding = Arc::new(ForwardingTable::default());
15040        let supervisor = SupervisorHandle::new();
15041        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15042        let handler =
15043            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15044                .with_supervisor(supervisor);
15045        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15046        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15047        register_capability_manifest(
15048            &handler,
15049            &target_ctx,
15050            &mut target_rx,
15051            capability_manifest("target", &[], &[]),
15052            1,
15053        )
15054        .await;
15055        register_capability_manifest(
15056            &handler,
15057            &opener_ctx,
15058            &mut opener_rx,
15059            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15060            2,
15061        )
15062        .await;
15063        let (mut client_rx, _) = open_route_for_capability_test(
15064            &handler,
15065            &target_ctx,
15066            &mut target_rx,
15067            722,
15068            3,
15069            "target",
15070            Some(ConsumerIdentity {
15071                module_id: "opener".to_string(),
15072                launch_nonce: "opener-nonce".to_string(),
15073            }),
15074        )
15075        .await;
15076        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15077
15078        let replies = handler
15079            .handle_control_frame(
15080                &target_ctx,
15081                catalog_update_with_capabilities_frame(
15082                    4,
15083                    CapabilityDeclarations {
15084                        provides: vec!["credentials-provider/v1".to_string()],
15085                        requires: Vec::new(),
15086                        must_never_reach: Vec::new(),
15087                    },
15088                ),
15089            )
15090            .await
15091            .expect("claim catalog.update succeeds");
15092        assert!(matches!(
15093            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15094            Ok(ModuleControlResponseToModule::CatalogUpdate {})
15095        ));
15096        assert_capability_denied_push(
15097            client_rx
15098                .try_recv()
15099                .expect("claim addition must emit route.closed")
15100                .frame,
15101            "target",
15102        );
15103        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15104        assert!(matches!(
15105            target_rx.try_recv(),
15106            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15107        ));
15108    }
15109
15110    #[tokio::test]
15111    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15112        let registry = Arc::new(Registry::default());
15113        let forwarding = Arc::new(ForwardingTable::default());
15114        let supervisor = SupervisorHandle::new();
15115        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15116        let handler =
15117            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15118                .with_supervisor(supervisor);
15119        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15120        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15121        register_capability_manifest(
15122            &handler,
15123            &target_ctx,
15124            &mut target_rx,
15125            capability_manifest("target", &["credentials-provider/v1"], &[]),
15126            1,
15127        )
15128        .await;
15129        register_capability_manifest(
15130            &handler,
15131            &opener_ctx,
15132            &mut opener_rx,
15133            capability_manifest("opener", &[], &[]),
15134            2,
15135        )
15136        .await;
15137        let (mut client_rx, _) = open_route_for_capability_test(
15138            &handler,
15139            &target_ctx,
15140            &mut target_rx,
15141            732,
15142            3,
15143            "target",
15144            Some(ConsumerIdentity {
15145                module_id: "opener".to_string(),
15146                launch_nonce: "opener-nonce".to_string(),
15147            }),
15148        )
15149        .await;
15150        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15151
15152        handler
15153            .handle_control_frame(
15154                &target_ctx,
15155                catalog_update_with_capabilities_frame(
15156                    4,
15157                    CapabilityDeclarations {
15158                        provides: Vec::new(),
15159                        requires: Vec::new(),
15160                        must_never_reach: Vec::new(),
15161                    },
15162                ),
15163            )
15164            .await
15165            .expect("claim removal catalog.update succeeds");
15166        assert_eq!(
15167            forwarding.active_binding_count().unwrap(),
15168            1,
15169            "removing an attested target claim must leave the route census unchanged"
15170        );
15171        assert!(
15172            client_rx.try_recv().is_err(),
15173            "claim removal must not emit route.closed capability_denied"
15174        );
15175        assert!(
15176            target_rx.try_recv().is_err(),
15177            "claim removal must not send the target a route GOODBYE"
15178        );
15179    }
15180
15181    /// A direct client may open a route to a denied capability provider; this
15182    /// policy applies only to attested supervised module origins, not to direct clients.
15183    #[tokio::test]
15184    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15185        let registry = Arc::new(Registry::default());
15186        let forwarding = Arc::new(ForwardingTable::default());
15187        let supervisor = SupervisorHandle::new();
15188        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15189        let handler =
15190            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15191                .with_supervisor(supervisor);
15192        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15193        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15194        register_capability_manifest(
15195            &handler,
15196            &target_ctx,
15197            &mut target_rx,
15198            capability_manifest("target", &["credentials-provider/v1"], &[]),
15199            1,
15200        )
15201        .await;
15202        register_capability_manifest(
15203            &handler,
15204            &opener_ctx,
15205            &mut opener_rx,
15206            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15207            2,
15208        )
15209        .await;
15210
15211        let (_client_rx, bind) = open_route_for_capability_test(
15212            &handler,
15213            &target_ctx,
15214            &mut target_rx,
15215            742,
15216            3,
15217            "target",
15218            None,
15219        )
15220        .await;
15221        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15222            panic!("direct scope-honesty route must bind");
15223        };
15224        assert_eq!(principal, Some(Principal::Direct));
15225        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15226    }
15227
15228    /// A module that denies a capability receives no self-route exemption when it
15229    /// also attestedly provides that capability.
15230    #[tokio::test]
15231    async fn must_never_reach_self_route_is_capability_forbidden() {
15232        let registry = Arc::new(Registry::default());
15233        let forwarding = Arc::new(ForwardingTable::default());
15234        let supervisor = SupervisorHandle::new();
15235        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15236        let handler =
15237            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15238                .with_supervisor(supervisor);
15239        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15240        register_capability_manifest(
15241            &handler,
15242            &self_ctx,
15243            &mut self_rx,
15244            capability_manifest(
15245                "self-provider",
15246                &["credentials-provider/v1"],
15247                &["credentials-provider/v1"],
15248            ),
15249            1,
15250        )
15251        .await;
15252
15253        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15254        let replies = handler
15255            .handle_control_frame(
15256                &client_ctx,
15257                route_open_frame_with_admission_facts(
15258                    2,
15259                    "self-provider",
15260                    unique_project_root("admission-facts"),
15261                    Some(ConsumerIdentity {
15262                        module_id: "self-provider".to_string(),
15263                        launch_nonce: "self-nonce".to_string(),
15264                    }),
15265                    None,
15266                ),
15267            )
15268            .await
15269            .expect("self-route refusal returns a typed frame");
15270        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15271        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15272        assert!(
15273            self_rx.try_recv().is_err(),
15274            "self denial must not relay route.bind"
15275        );
15276    }
15277
15278    #[test]
15279    fn unsupported_channel_zero_frame_returns_error() {
15280        let handler = ControlHandler::default();
15281        let request = Frame::build(
15282            FrameType::Request,
15283            control_flags(),
15284            0,
15285            0,
15286            21,
15287            b"opaque".to_vec(),
15288        )
15289        .unwrap();
15290
15291        let response = handler
15292            .handle_control(ConnectionId::new(1), request)
15293            .unwrap();
15294
15295        assert_eq!(response[0].header.ty, FrameType::Error);
15296        assert_eq!(
15297            parse_error(&response[0])["code"],
15298            "unsupported_control_frame"
15299        );
15300    }
15301
15302    /// Blue/green swap at the control-plane boundary. The supervisor that opens
15303    /// a swap is not wired yet, so the candidate is registered here directly
15304    /// into the registry and forwarding candidate slots, the way the swap's
15305    /// HELLO admission will.
15306    mod swap {
15307        use super::*;
15308
15309        const INCUMBENT: ConnectionId = ConnectionId::new(30);
15310        const CANDIDATE: ConnectionId = ConnectionId::new(40);
15311
15312        struct Swap {
15313            registry: Arc<Registry>,
15314            forwarding: Arc<ForwardingTable>,
15315            handler: ControlHandler,
15316            incumbent_ctx: RouteCtx,
15317            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15318            candidate_ctx: RouteCtx,
15319            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15320        }
15321
15322        async fn swap_with_incumbent() -> Swap {
15323            let registry = Arc::new(Registry::default());
15324            let forwarding = Arc::new(ForwardingTable::default());
15325            let handler =
15326                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15327            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15328            hello_via_sink(
15329                &handler,
15330                &incumbent_ctx,
15331                &mut incumbent_rx,
15332                hello_frame("aft", PROTOCOL_VERSION, 7),
15333            )
15334            .await;
15335            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15336            Swap {
15337                registry,
15338                forwarding,
15339                handler,
15340                incumbent_ctx,
15341                incumbent_rx,
15342                candidate_ctx,
15343                candidate_rx,
15344            }
15345        }
15346
15347        fn register_candidate(swap: &Swap, ready: Option<bool>) {
15348            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15349            candidate_manifest.ready = ready;
15350            let registration = swap
15351                .registry
15352                .register_candidate_with_control_ops(
15353                    candidate_manifest,
15354                    PROTOCOL_VERSION,
15355                    CANDIDATE,
15356                    module_baseline_control_ops(),
15357                )
15358                .unwrap();
15359            swap.forwarding
15360                .register_candidate_module_connection(
15361                    CANDIDATE,
15362                    "aft".to_string(),
15363                    PROTOCOL_VERSION,
15364                    manifest_concurrency(&registration.manifest),
15365                    swap.candidate_ctx.egress.clone(),
15366                )
15367                .unwrap();
15368        }
15369
15370        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15371            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15372            swap.registry.promote_candidate("aft").unwrap().unwrap();
15373            cutover.incumbent.unwrap()
15374        }
15375
15376        /// A swap candidate's HELLO reserves its candidate registry entry
15377        /// before installing its candidate forwarding endpoint. The swap
15378        /// supervisor promotes a ready candidate as soon as it reads it as
15379        /// registered, and promotion cuts forwarding over to the candidate
15380        /// endpoint, so the lookup the supervisor reads
15381        /// (`Registry::published_registration`) must not return the candidate
15382        /// until that endpoint exists. Otherwise the cutover finds no candidate
15383        /// endpoint and the swap fails as if the candidate had disconnected.
15384        #[tokio::test]
15385        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15386            let mut swap = swap_with_incumbent().await;
15387            let hold = Arc::new(std::sync::Barrier::new(2));
15388            let handler = swap
15389                .handler
15390                .clone()
15391                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15392            handler
15393                .supervisor
15394                .open_swap("aft", "candidate-nonce".to_string());
15395            let hello = {
15396                let handler = handler.clone();
15397                let candidate_ctx = swap.candidate_ctx.clone();
15398                std::thread::spawn(move || {
15399                    handler
15400                        .handle_hello(
15401                            candidate_ctx.connection_id,
15402                            Some(candidate_ctx.egress.clone()),
15403                            hello_frame_with_nonce(
15404                                "aft",
15405                                PROTOCOL_VERSION,
15406                                8,
15407                                Some("candidate-nonce"),
15408                            ),
15409                        )
15410                        .unwrap()
15411                })
15412            };
15413
15414            hold.wait();
15415            // Parked: the candidate slot is claimed but not published, and
15416            // forwarding has no candidate endpoint.
15417            assert!(swap
15418                .registry
15419                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15420                .unwrap()
15421                .is_some());
15422            assert!(swap
15423                .registry
15424                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15425                .unwrap()
15426                .is_none());
15427            assert!(swap
15428                .forwarding
15429                .module_endpoint_for_connection(CANDIDATE)
15430                .unwrap()
15431                .is_none());
15432
15433            hold.wait();
15434            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15435            assert_eq!(
15436                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15437                FrameType::HelloAck
15438            );
15439            let published = swap
15440                .registry
15441                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15442                .unwrap()
15443                .expect("candidate is published once its HELLO completes");
15444            assert_eq!(published.connection_id, CANDIDATE);
15445            assert!(swap
15446                .forwarding
15447                .module_endpoint_for_connection(CANDIDATE)
15448                .unwrap()
15449                .is_some());
15450        }
15451
15452        fn keyed_total(counters: &Value, key: &str) -> u64 {
15453            counters[key]
15454                .as_object()
15455                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15456                .unwrap_or(0)
15457        }
15458
15459        #[tokio::test]
15460        async fn replacement_logs_old_end_and_new_admission_once() {
15461            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15462            let mut swap = swap_with_incumbent().await;
15463            swap.handler
15464                .supervisor
15465                .open_swap("aft", "candidate-nonce".to_string());
15466            hello_via_sink(
15467                &swap.handler,
15468                &swap.candidate_ctx,
15469                &mut swap.candidate_rx,
15470                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15471            )
15472            .await;
15473            cutover(&swap);
15474            swap.handler.cleanup_connection(INCUMBENT).unwrap();
15475
15476            let captured = crate::router::test_log::captured_logs(&logs);
15477            println!("captured replacement registry logs:\n{captured}");
15478            let new_admissions: Vec<_> = captured
15479                .lines()
15480                .filter(|line| {
15481                    line.contains("connection_id=40")
15482                        && line.contains("swap candidate registered; not routable until cutover")
15483                })
15484                .collect();
15485            assert_eq!(
15486                new_admissions.len(),
15487                1,
15488                "unexpected admission logs: {captured}"
15489            );
15490            assert!(new_admissions[0].contains("module_id=aft"));
15491            assert!(new_admissions[0].contains("connection_id=40"));
15492            assert!(
15493                new_admissions[0].contains("swap candidate registered; not routable until cutover")
15494            );
15495            assert!(new_admissions[0].contains("ready=true"));
15496
15497            let old_admissions: Vec<_> = captured
15498                .lines()
15499                .filter(|line| {
15500                    line.contains("module registered module_id=aft ")
15501                        && line.contains("connection_id=30")
15502                })
15503                .collect();
15504            assert_eq!(
15505                old_admissions.len(),
15506                1,
15507                "unexpected incumbent admission: {captured}"
15508            );
15509
15510            let old_ends: Vec<_> = captured
15511                .lines()
15512                .filter(|line| {
15513                    line.contains("connection_id=30") && line.contains("module registration ended")
15514                })
15515                .collect();
15516            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15517            assert!(old_ends[0].contains("module_id=aft"));
15518            assert!(old_ends[0].contains("connection_id=30"));
15519            assert!(old_ends[0].contains("reason=replaced"));
15520            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15521            assert_eq!(
15522                captured
15523                    .lines()
15524                    .filter(|line| line.contains("module registration promoted"))
15525                    .count(),
15526                1,
15527                "unexpected promotion logs: {captured}"
15528            );
15529        }
15530
15531        /// An ack from the incumbent for a bind it was sent before cutover,
15532        /// arriving before the incumbent is drained. The incumbent is the live
15533        /// connection carrying every other client's routes, so the ack must
15534        /// not end it: the waiting client is told to retry, the reservation is
15535        /// given back, and the incumbent is told to drop just that binding.
15536        #[tokio::test]
15537        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15538            let mut swap = swap_with_incumbent().await;
15539            let handler = swap.handler.clone();
15540
15541            // A co-tenant route, bound on the incumbent before the swap.
15542            let cotenant = ConnectionId::new(31);
15543            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15544            let (cotenant_task, cotenant_bind) = relay_route_open(
15545                &handler,
15546                cotenant,
15547                &cotenant_ctx.egress,
15548                &mut swap.incumbent_rx,
15549                100,
15550                "aft",
15551                "swap-cotenant",
15552            )
15553            .await;
15554            handler
15555                .handle_control_frame(
15556                    &swap.incumbent_ctx,
15557                    route_bind_ack(cotenant_bind.header.corr),
15558                )
15559                .await
15560                .unwrap();
15561            assert!(cotenant_task.await.unwrap().is_empty());
15562            let (cotenant_channel, cotenant_epoch) =
15563                published_route(&cotenant_rx.recv().await.unwrap());
15564
15565            // A second route.open, relayed to the incumbent and not yet acked.
15566            let caller = ConnectionId::new(32);
15567            let (caller_ctx, mut caller_rx) = route_ctx(caller);
15568            let (caller_task, caller_bind) = relay_route_open(
15569                &handler,
15570                caller,
15571                &caller_ctx.egress,
15572                &mut swap.incumbent_rx,
15573                101,
15574                "aft",
15575                "swap-caller",
15576            )
15577            .await;
15578            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15579
15580            register_candidate(&swap, None);
15581            cutover(&swap);
15582
15583            // The incumbent acks after promotion and before any drain.
15584            let ack = handler
15585                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15586                .await;
15587            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15588            if module_loop_error.is_some() {
15589                // What the connection loop does with an untranslated router
15590                // error: end the connection, releasing every route on it.
15591                handler.cleanup_connection(INCUMBENT).unwrap();
15592            }
15593
15594            // 1. The incumbent's other routes survive.
15595            assert!(
15596                cotenant_rx.try_recv().is_err(),
15597                "the co-tenant route on the incumbent was torn down by one late ack: \
15598                 {module_loop_error:?}"
15599            );
15600            assert!(matches!(
15601                swap.forwarding
15602                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15603                    .unwrap(),
15604                DataRoute::Client(DataRouteState::Bound(_))
15605            ));
15606            assert_eq!(module_loop_error, None);
15607            assert!(swap
15608                .registry
15609                .get_module_by_connection(INCUMBENT)
15610                .unwrap()
15611                .is_some());
15612
15613            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
15614            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15615                .await
15616                .expect("the incumbent is told to drop the abandoned binding")
15617                .unwrap()
15618                .frame;
15619            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15620            assert_eq!(goodbye.header.channel, abandoned_channel);
15621            assert_eq!(goodbye.header.epoch, abandoned_epoch);
15622            assert!(swap.incumbent_rx.try_recv().is_err());
15623
15624            // 3. The waiting client gets a retryable refusal and no route.
15625            let response = caller_task.await.unwrap();
15626            assert_eq!(response.len(), 1);
15627            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15628            assert!(caller_rx.try_recv().is_err());
15629
15630            // 4. The reservation pair is given back, and the pending bind
15631            //    settled exactly once: one accepted open (the co-tenant) and one
15632            //    refused open (the caller), nothing counted twice.
15633            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15634            let counters = handler.counters().snapshot();
15635            assert_eq!(
15636                keyed_total(&counters, "route_open_accepted_by_principal"),
15637                1
15638            );
15639            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15640            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15641        }
15642
15643        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
15644        /// id would resolve to the promoted candidate and every new route.open
15645        /// would be refused as reloading, leaving neither process routable.
15646        #[tokio::test]
15647        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15648            let mut swap = swap_with_incumbent().await;
15649            register_candidate(&swap, None);
15650            let incumbent = cutover(&swap);
15651            swap.forwarding
15652                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15653                .unwrap()
15654                .expect("the incumbent is still registered");
15655
15656            let client = ConnectionId::new(33);
15657            let (client_ctx, mut client_rx) = route_ctx(client);
15658            let route_handler = swap.handler.clone();
15659            let open_ctx = RouteCtx {
15660                connection_id: client,
15661                egress: client_ctx.egress.clone(),
15662            };
15663            let mut route_task = tokio::spawn(async move {
15664                route_handler
15665                    .handle_control_frame(
15666                        &open_ctx,
15667                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15668                    )
15669                    .await
15670                    .unwrap()
15671            });
15672            let bind = tokio::select! {
15673                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15674                response = &mut route_task => {
15675                    let response = response.unwrap();
15676                    panic!(
15677                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
15678                        parse_error(&response[0])["code"]
15679                    );
15680                }
15681            };
15682            swap.handler
15683                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15684                .await
15685                .unwrap();
15686            assert!(route_task.await.unwrap().is_empty());
15687            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15688            match swap
15689                .forwarding
15690                .lookup_data_route(client, channel, epoch)
15691                .unwrap()
15692            {
15693                DataRoute::Client(DataRouteState::Bound(route)) => {
15694                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15695                }
15696                other => panic!("expected a bound route on the candidate, got {other:?}"),
15697            }
15698            assert!(swap.incumbent_rx.try_recv().is_err());
15699        }
15700
15701        /// A candidate declares itself ready with `catalog.update` on its own
15702        /// connection. If the connection-keyed registry lookups searched only the
15703        /// active slot, this would answer `not_registered` and the candidate
15704        /// would never become ready.
15705        #[tokio::test]
15706        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15707            let swap = swap_with_incumbent().await;
15708            register_candidate(&swap, Some(false));
15709            let update = Frame::build(
15710                FrameType::Request,
15711                control_flags(),
15712                0,
15713                0,
15714                55,
15715                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15716                    provides: manifest("aft", PROTOCOL_VERSION).provides,
15717                    capabilities: None,
15718                    ready: Some(true),
15719                })
15720                .unwrap(),
15721            )
15722            .unwrap();
15723
15724            let replies = swap
15725                .handler
15726                .handle_control_frame(&swap.candidate_ctx, update)
15727                .await
15728                .unwrap();
15729
15730            assert_eq!(replies.len(), 1);
15731            assert_eq!(
15732                replies[0].header.ty,
15733                FrameType::Response,
15734                "candidate catalog.update was refused: {:?}",
15735                serde_json::from_slice::<Value>(&replies[0].body).ok()
15736            );
15737            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15738            assert_eq!(
15739                swap.registry
15740                    .get_module("aft")
15741                    .unwrap()
15742                    .unwrap()
15743                    .connection_id,
15744                INCUMBENT
15745            );
15746        }
15747    }
15748
15749    /// The HELLO gate while the supervisor has a swap open: only the nonce it
15750    /// minted for the candidate admits a second process, into the candidate
15751    /// slot, and that check runs ahead of the reserved-module gate.
15752    mod swap_admission {
15753        use super::*;
15754
15755        const INCUMBENT_NONCE: &str = "incumbent-nonce";
15756        const CANDIDATE_NONCE: &str = "candidate-nonce";
15757
15758        fn handler_with_incumbent(
15759            module_id: &str,
15760            reserved: bool,
15761        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15762            let registry = Arc::new(Registry::default());
15763            let supervisor = SupervisorHandle::new();
15764            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15765            if reserved {
15766                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15767            }
15768            let handler =
15769                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
15770            let incumbent = handler
15771                .handle_control(
15772                    ConnectionId::new(1),
15773                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15774                )
15775                .unwrap();
15776            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15777            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15778            (registry, supervisor, handler)
15779        }
15780
15781        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
15782        /// admits every nonce, so while a swap is open the swap gate is the only
15783        /// thing between a key-holder and the candidate slot. A nonce the
15784        /// supervisor did not mint, or none at all, is refused, and neither the
15785        /// incumbent's registration nor the candidate slot moves.
15786        #[test]
15787        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15788            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15789
15790            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15791                let replies = handler
15792                    .handle_control(
15793                        ConnectionId::new(connection),
15794                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15795                    )
15796                    .unwrap();
15797                assert_eq!(replies[0].header.ty, FrameType::Error);
15798                assert_eq!(
15799                    parse_error(&replies[0])["code"],
15800                    "swap_token_invalid",
15801                    "nonce {nonce:?}"
15802                );
15803            }
15804            assert!(registry.get_candidate("aft").unwrap().is_none());
15805            assert_eq!(
15806                registry.get_module("aft").unwrap().unwrap().connection_id,
15807                ConnectionId::new(1)
15808            );
15809
15810            // Control: the minted token is admitted, into the candidate slot,
15811            // and only once.
15812            let admitted = handler
15813                .handle_control(
15814                    ConnectionId::new(4),
15815                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15816                )
15817                .unwrap();
15818            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15819            assert_eq!(
15820                registry
15821                    .get_candidate("aft")
15822                    .unwrap()
15823                    .unwrap()
15824                    .connection_id,
15825                ConnectionId::new(4)
15826            );
15827            assert_eq!(
15828                registry.get_module("aft").unwrap().unwrap().connection_id,
15829                ConnectionId::new(1),
15830                "the candidate must not take the active slot"
15831            );
15832            let replayed = handler
15833                .handle_control(
15834                    ConnectionId::new(5),
15835                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15836                )
15837                .unwrap();
15838            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15839
15840            // The case only this gate covers: the incumbent has died mid-swap,
15841            // so its duplicate refusal is gone too, and without the gate a
15842            // key-holder would take the id's ACTIVE slot.
15843            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15844            let squatter = handler
15845                .handle_control(
15846                    ConnectionId::new(6),
15847                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15848                )
15849                .unwrap();
15850            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15851            assert!(
15852                registry.get_module("aft").unwrap().is_none(),
15853                "a squatter took the active slot of an id being swapped"
15854            );
15855        }
15856
15857        /// Design mutation arm (iii). A reserved module's candidate presents a
15858        /// nonce the reserved gate has never seen (that gate holds the
15859        /// incumbent's), so the swap gate must run first or the candidate is
15860        /// refused `reserved_module` and a reserved module can never be swapped.
15861        #[test]
15862        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15863            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15864
15865            let replies = handler
15866                .handle_control(
15867                    ConnectionId::new(2),
15868                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15869                )
15870                .unwrap();
15871
15872            assert_eq!(
15873                replies[0].header.ty,
15874                FrameType::HelloAck,
15875                "reserved candidate refused: {:?}",
15876                serde_json::from_slice::<Value>(&replies[0].body).ok()
15877            );
15878            assert_eq!(
15879                registry
15880                    .get_candidate("vault")
15881                    .unwrap()
15882                    .unwrap()
15883                    .connection_id,
15884                ConnectionId::new(2)
15885            );
15886        }
15887
15888        /// With no swap open the gate is inert: the incumbent's reserved gate
15889        /// and duplicate refusal behave exactly as before.
15890        #[test]
15891        fn without_an_open_swap_the_ordinary_gates_decide() {
15892            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15893            supervisor.close_swap("vault");
15894
15895            let candidate = handler
15896                .handle_control(
15897                    ConnectionId::new(2),
15898                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15899                )
15900                .unwrap();
15901            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15902            let duplicate = handler
15903                .handle_control(
15904                    ConnectionId::new(3),
15905                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15906                )
15907                .unwrap();
15908            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15909            assert!(registry.get_candidate("vault").unwrap().is_none());
15910        }
15911    }
15912
15913    /// `scope.sync` and `scope.describe` through the real control handler: who
15914    /// may sync is decided by the registration and launch nonce of the module
15915    /// connection, never by the request body.
15916    mod scopes {
15917        mod idle_tests {
15918            use super::*;
15919            include!("scope_idle_tests.rs");
15920        }
15921        mod route_open_timings {
15922            use super::*;
15923            include!("route_open_timing_tests.rs");
15924        }
15925        mod route_open_refusal_log {
15926            use super::*;
15927            include!("route_open_refusal_log_tests.rs");
15928        }
15929        mod activity {
15930            include!("activity_scope_tests.rs");
15931        }
15932        mod delta {
15933            include!("scope_handler_delta_tests.rs");
15934        }
15935        mod call_modes {
15936            include!("scope_call_mode_tests.rs");
15937        }
15938        #[tokio::test]
15939        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15940            let rig = rig().await;
15941            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15942                "s".into(),
15943                1,
15944                "tool".into(),
15945            ))
15946            .unwrap();
15947            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15948            let (direct, _rx) = wide_ctx(999);
15949            for (ctx, expected) in [
15950                (&rig.owner, FrameType::Response),
15951                (&direct, FrameType::Error),
15952            ] {
15953                let frame =
15954                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15955                        .unwrap();
15956                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15957                assert_eq!(replies[0].header.ty, expected);
15958                if expected == FrameType::Response {
15959                    let reply: subc_control::ScopeServedByReply =
15960                        serde_json::from_slice(&replies[0].body).unwrap();
15961                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15962                } else {
15963                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15964                }
15965            }
15966            let reply = call(
15967                &rig.handler,
15968                &rig.owner,
15969                &ModuleControlRequestFromModule::ScopeDescribe {
15970                    owner: Principal::Reserved {
15971                        module_id: OWNER.into(),
15972                    },
15973                    scope_ref: "s".into(),
15974                },
15975            )
15976            .await;
15977            assert!(matches!(
15978                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15979                ModuleControlResponseToModule::ScopeDescribe { .. }
15980            ));
15981        }
15982
15983        use subc_protocol::scope::{
15984            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15985            ScopeStatus,
15986        };
15987
15988        use super::*;
15989
15990        const OWNER: &str = "prefrontal-core";
15991
15992        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15993            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15994        }
15995
15996        async fn call(
15997            handler: &ControlHandler,
15998            ctx: &RouteCtx,
15999            request: &ModuleControlRequestFromModule,
16000        ) -> Frame {
16001            let body = serde_json::to_vec(request).unwrap();
16002            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16003            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16004            assert_eq!(replies.len(), 1, "{replies:?}");
16005            replies.pop().unwrap()
16006        }
16007
16008        async fn sync(
16009            handler: &ControlHandler,
16010            ctx: &RouteCtx,
16011            generation: u64,
16012            scopes: Vec<ScopeRecord>,
16013        ) -> Result<ModuleControlResponseToModule, String> {
16014            let reply = call(
16015                handler,
16016                ctx,
16017                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16018            )
16019            .await;
16020            match reply.header.ty {
16021                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16022                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16023            }
16024        }
16025
16026        async fn describe(
16027            handler: &ControlHandler,
16028            ctx: &RouteCtx,
16029            owner: &str,
16030            scope_ref: &str,
16031        ) -> ModuleControlResponseToModule {
16032            let reply = call(
16033                handler,
16034                ctx,
16035                &ModuleControlRequestFromModule::ScopeDescribe {
16036                    owner: Principal::Reserved {
16037                        module_id: owner.to_string(),
16038                    },
16039                    scope_ref: scope_ref.to_string(),
16040                },
16041            )
16042            .await;
16043            assert_eq!(
16044                reply.header.ty,
16045                FrameType::Response,
16046                "{:?}",
16047                parse_error(&reply)
16048            );
16049            serde_json::from_slice(&reply.body).unwrap()
16050        }
16051
16052        #[tokio::test]
16053        async fn scope_describe_never_echoes_bind_proof_or_hash() {
16054            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16055            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16056            let mut rig = rig().await;
16057            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16058                .await;
16059            let _bound = rig
16060                .bound(
16061                    Some(AFT),
16062                    PLEXUS,
16063                    Some(
16064                        rig.selector("s", Some(1))
16065                            .with_bind_proof(Some(PROOF.into())),
16066                    ),
16067                )
16068                .await;
16069            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16070            let output = serde_json::to_string(&reply).unwrap();
16071            assert!(output.contains("\"status\":\"live\""));
16072            assert!(!output.contains(PROOF));
16073            assert!(!output.contains(HASH));
16074        }
16075
16076        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
16077        async fn module(
16078            handler: &ControlHandler,
16079            connection: u64,
16080            module_id: &str,
16081            nonce: Option<&str>,
16082        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16083            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16084            hello_via_sink(
16085                handler,
16086                &ctx,
16087                &mut rx,
16088                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16089            )
16090            .await;
16091            (ctx, rx)
16092        }
16093
16094        /// `direct` and every other client connection has no registration, so
16095        /// it can neither sync nor own a scope.
16096        #[tokio::test]
16097        async fn a_client_connection_cannot_sync_or_describe() {
16098            let handler = ControlHandler::new(Arc::new(Registry::default()));
16099            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16100            for request in [
16101                ModuleControlRequestFromModule::ScopeSync {
16102                    generation: 1,
16103                    scopes: vec![head("s", 1)],
16104                },
16105                ModuleControlRequestFromModule::ScopeDescribe {
16106                    owner: Principal::Direct,
16107                    scope_ref: "s".to_string(),
16108                },
16109            ] {
16110                let reply = call(&handler, &ctx, &request).await;
16111                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16112            }
16113            assert!(
16114                !handler
16115                    .scopes
16116                    .read()
16117                    .unwrap()
16118                    .describe(
16119                        &Principal::Reserved {
16120                            module_id: OWNER.to_string()
16121                        },
16122                        "s"
16123                    )
16124                    .owner_synced
16125            );
16126        }
16127
16128        /// A module the supervisor did not spawn registers without a launch
16129        /// nonce, so it is never an owner's current launch.
16130        #[tokio::test]
16131        async fn a_module_without_a_supervised_launch_cannot_sync() {
16132            let handler = ControlHandler::new(Arc::new(Registry::default()));
16133            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16134            assert_eq!(
16135                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16136                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16137            );
16138        }
16139
16140        #[tokio::test]
16141        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16142            let supervisor = SupervisorHandle::new();
16143            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16144            let handler = ControlHandler::new(Arc::new(Registry::default()))
16145                .with_supervisor(supervisor.clone());
16146            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16147            sync(&handler, &incumbent, 1, vec![head("s", 1)])
16148                .await
16149                .expect("the current launch syncs");
16150
16151            // A swap candidate registers with the swap token and is refused
16152            // while the incumbent keeps syncing.
16153            supervisor.open_swap(OWNER, "n2".to_string());
16154            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16155            assert_eq!(
16156                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16157                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16158            );
16159            sync(&handler, &incumbent, 2, vec![head("s", 1)])
16160                .await
16161                .expect("the serving owner syncs during the swap");
16162
16163            // The swap fails and is rolled back. The candidate never held sync
16164            // authority, and still cannot sync.
16165            supervisor.close_swap(OWNER);
16166            assert_eq!(
16167                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16168                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16169            );
16170            sync(&handler, &incumbent, 3, vec![head("s", 1)])
16171                .await
16172                .expect("the serving owner syncs after the rollback");
16173            handler.cleanup_connection(candidate.connection_id).unwrap();
16174
16175            // A swap that cuts over. Promotion records the candidate's nonce as
16176            // the module's spawn nonce, which is what `set_spawn_nonce` does
16177            // here; the promoted connection then takes authority at any
16178            // generation and the superseded incumbent is refused.
16179            supervisor.open_swap(OWNER, "n3".to_string());
16180            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16181            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16182            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16183                .await
16184                .expect("the promoted launch takes authority");
16185            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16186                panic!("unexpected reply {reply:?}");
16187            };
16188            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16189            assert_eq!(
16190                sync(&handler, &incumbent, 4, Vec::new()).await,
16191                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16192            );
16193        }
16194
16195        /// Authority dies with its connection: the cleanup path releases it,
16196        /// so the owner's next connection takes it at any generation.
16197        #[tokio::test]
16198        async fn closing_the_authority_connection_frees_sync_authority() {
16199            let supervisor = SupervisorHandle::new();
16200            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16201            let handler = ControlHandler::new(Arc::new(Registry::default()))
16202                .with_supervisor(supervisor.clone());
16203            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16204            sync(&handler, &first, 10, vec![head("s", 1)])
16205                .await
16206                .unwrap();
16207            handler.cleanup_connection(first.connection_id).unwrap();
16208
16209            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16210            sync(&handler, &second, 1, vec![head("s", 1)])
16211                .await
16212                .expect("the next connection takes the released authority");
16213        }
16214
16215        #[tokio::test]
16216        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16217            let supervisor = SupervisorHandle::new();
16218            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16219            let handler =
16220                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16221            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16222            sync(&handler, &first, 10, vec![head("s", 1)])
16223                .await
16224                .unwrap();
16225            handler
16226                .handle_control_frame(
16227                    &first,
16228                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16229                )
16230                .await
16231                .unwrap();
16232            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16233            sync(&handler, &second, 1, vec![head("s", 1)])
16234                .await
16235                .expect("GOODBYE releases authority even if the old socket remains open");
16236        }
16237
16238        #[tokio::test]
16239        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16240            let registry = Arc::new(Registry::default());
16241            let supervisor_handle = SupervisorHandle::new();
16242            let supervisor =
16243                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16244                    .with_handle(supervisor_handle.clone())
16245                    .with_daemon_incarnation("incarnation-7".to_string());
16246            // Configured with enabled: false, so the supervisor lists the
16247            // module without spawning a process for it.
16248            supervisor
16249                .supervise_configured(
16250                    ModuleSpec {
16251                        module_id: OWNER.to_string(),
16252                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16253                        args: Vec::new(),
16254                        env: Vec::new(),
16255                        reserved: false,
16256                        reserved_prefixes: Vec::new(),
16257                        protocol: ModuleProtocol::Subc,
16258                        overlap: Default::default(),
16259                    },
16260                    false,
16261                )
16262                .unwrap();
16263            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16264            let handler =
16265                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
16266            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16267
16268            // Configured but not yet synced: a reader waits for the owner.
16269            let ModuleControlResponseToModule::ScopeDescribe {
16270                status,
16271                daemon_incarnation,
16272                owner_synced,
16273                owner_configured,
16274                scope,
16275                ..
16276            } = describe(&handler, &reader, OWNER, "s").await
16277            else {
16278                panic!("not a describe reply");
16279            };
16280            assert_eq!(status, ScopeStatus::NotLive);
16281            assert_eq!(daemon_incarnation, "incarnation-7");
16282            assert!(!owner_synced);
16283            assert!(owner_configured);
16284            assert!(scope.is_none());
16285
16286            // Not a supervised module: the owner will never sync, and a reader
16287            // refuses rather than waits.
16288            let ModuleControlResponseToModule::ScopeDescribe {
16289                status,
16290                owner_configured,
16291                ..
16292            } = describe(&handler, &reader, "ghost", "s").await
16293            else {
16294                panic!("not a describe reply");
16295            };
16296            assert_eq!(status, ScopeStatus::NotLive);
16297            assert!(!owner_configured);
16298
16299            // Live, with the stamp fields and the computed owner_authorized.
16300            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16301            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16302            let ModuleControlResponseToModule::ScopeDescribe {
16303                status,
16304                scope_epoch,
16305                owner_synced,
16306                scope,
16307                ..
16308            } = describe(&handler, &reader, OWNER, "s").await
16309            else {
16310                panic!("not a describe reply");
16311            };
16312            assert_eq!(status, ScopeStatus::Live);
16313            assert_eq!(scope_epoch, Some(4));
16314            assert!(owner_synced);
16315            let stamp = scope.expect("a live scope carries its stamp");
16316            assert!(
16317                stamp.owner_authorized,
16318                "prefrontal-core is the default authority"
16319            );
16320            assert_eq!(stamp.kind, ScopeKind::Head);
16321        }
16322
16323        #[tokio::test]
16324        async fn scope_authority_owners_decides_owner_authorized() {
16325            let supervisor = SupervisorHandle::new();
16326            supervisor.set_spawn_nonce("broca", "b1".to_string());
16327            let handler = ControlHandler::new(Arc::new(Registry::default()))
16328                .with_supervisor(supervisor)
16329                .with_scope_authority_owners(vec!["broca".to_string()]);
16330            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16331            let mut gated = head("s", 1);
16332            gated.attributes.agent_id = Some("agent".to_string());
16333            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16334            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16335                describe(&handler, &broca, "broca", "s").await
16336            else {
16337                panic!("not a describe reply");
16338            };
16339            assert!(scope.unwrap().owner_authorized);
16340        }
16341
16342        /// With route admission, the stamp, the commit re-check and drains in
16343        /// place, the feature is advertised: the module ops in HELLO_ACK, and
16344        /// `scopes/v1` in HELLO_ACK and `server.describe`.
16345        #[tokio::test]
16346        async fn scope_ops_and_the_scopes_capability_are_advertised() {
16347            let handler = ControlHandler::new(Arc::new(Registry::default()));
16348            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16349            let ack = hello_via_sink(
16350                &handler,
16351                &ctx,
16352                &mut rx,
16353                hello_frame("m", PROTOCOL_VERSION, 1),
16354            )
16355            .await;
16356            let ack = parse_ack(&ack);
16357            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16358                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16359            }
16360            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16361
16362            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16363            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16364            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16365            let reply = handler
16366                .handle_control_frame(&client, frame)
16367                .await
16368                .unwrap()
16369                .pop()
16370                .unwrap();
16371            let ClientControlResponse::ServerDescribe { capabilities, .. } =
16372                serde_json::from_slice(&reply.body).unwrap()
16373            else {
16374                panic!("not a server.describe reply");
16375            };
16376            assert!(
16377                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16378                "{capabilities:?}"
16379            );
16380        }
16381
16382        // ---- route admission, stamps, commit re-check and drains ----------
16383
16384        const PLEXUS: &str = "plexus";
16385        const OTHER: &str = "other";
16386        const AFT: &str = "aft";
16387        const BROCA: &str = "broca";
16388        const MAGIC: &str = "magic-context";
16389
16390        fn nonce(module_id: &str) -> String {
16391            format!("nonce-{module_id}")
16392        }
16393
16394        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16395            let (tx, rx) = mpsc::channel(64);
16396            (
16397                RouteCtx {
16398                    connection_id: ConnectionId::new(connection),
16399                    egress: FrameSink::new(tx),
16400                },
16401                rx,
16402            )
16403        }
16404
16405        /// A daemon with a configured owner (prefrontal-core) registered on its
16406        /// own module connection, two routable targets (plexus, other), and
16407        /// launch nonces minted for the modules that open routes as carriers.
16408        struct Rig {
16409            handler: ControlHandler,
16410            forwarding: Arc<ForwardingTable>,
16411            owner: RouteCtx,
16412            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16413            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16414            generation: u64,
16415            next_connection: u64,
16416            _supervisor: Supervisor,
16417        }
16418
16419        async fn rig() -> Rig {
16420            rig_with_flow_support(true).await
16421        }
16422
16423        async fn rig_with_flow_support(flow_support: bool) -> Rig {
16424            rig_with_scope_capabilities(flow_support, false).await
16425        }
16426
16427        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16428            let mut provides = Vec::new();
16429            if flow_support {
16430                provides.push("flow-scopes/v1");
16431            }
16432            if run_support {
16433                provides.push("agent-run-scopes/v1");
16434            }
16435            rig_with_target_capabilities(&provides).await
16436        }
16437
16438        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16439            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16440            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16441            // Decoder versions alone cannot admit scoped calls: only a capability
16442            // declaration promises the required run or flow behaviour.
16443            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16444            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16445            Frame::build(
16446                FrameType::Hello,
16447                control_flags(),
16448                0,
16449                0,
16450                connection,
16451                serde_json::to_vec(&body).unwrap(),
16452            )
16453            .unwrap()
16454        }
16455
16456        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16457            let registry = Arc::new(Registry::default());
16458            let forwarding = Arc::new(ForwardingTable::default());
16459            let supervisor_handle = SupervisorHandle::new();
16460            let supervisor =
16461                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16462                    .with_handle(supervisor_handle.clone());
16463            supervisor
16464                .supervise_configured(
16465                    ModuleSpec {
16466                        module_id: OWNER.to_string(),
16467                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16468                        args: Vec::new(),
16469                        env: Vec::new(),
16470                        reserved: false,
16471                        reserved_prefixes: Vec::new(),
16472                        protocol: ModuleProtocol::Subc,
16473                        overlap: Default::default(),
16474                    },
16475                    false,
16476                )
16477                .unwrap();
16478            for module_id in [OWNER, AFT, BROCA, MAGIC] {
16479                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16480            }
16481            let handler =
16482                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16483                    .with_supervisor(supervisor_handle);
16484            let (owner, mut owner_rx) = wide_ctx(1);
16485            hello_via_sink(
16486                &handler,
16487                &owner,
16488                &mut owner_rx,
16489                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16490            )
16491            .await;
16492            let mut modules = BTreeMap::new();
16493            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16494                let (ctx, mut rx) = wide_ctx(connection);
16495                let hello = hello_with_capabilities(module_id, connection, provides);
16496                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16497                modules.insert(module_id.to_string(), (ctx, rx));
16498            }
16499            Rig {
16500                handler,
16501                forwarding,
16502                owner,
16503                _owner_rx: owner_rx,
16504                modules,
16505                generation: 0,
16506                next_connection: 100,
16507                _supervisor: supervisor,
16508            }
16509        }
16510
16511        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16512            ScopeCarrier::new(Principal::Reserved {
16513                module_id: module_id.to_string(),
16514            })
16515            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16516        }
16517
16518        /// The scope most tests open under: aft carries to any module, broca
16519        /// only to plexus and other, and the owner delegates as agent-1.
16520        fn session(scope_epoch: u64) -> ScopeRecord {
16521            let mut record = head("s", scope_epoch);
16522            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16523            record.attributes.agent_id = Some("agent-1".to_string());
16524            record.attributes.delegates = true;
16525            record
16526        }
16527
16528        impl Rig {
16529            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16530                self.generation += 1;
16531                sync(&self.handler, &self.owner, self.generation, scopes)
16532                    .await
16533                    .expect("the owner's sync is accepted");
16534            }
16535
16536            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16537                ScopeSelector::new(
16538                    Principal::Reserved {
16539                        module_id: OWNER.to_string(),
16540                    },
16541                    scope_ref.to_string(),
16542                    scope_epoch,
16543                )
16544            }
16545
16546            fn open_frame(
16547                &mut self,
16548                opener: Option<&str>,
16549                target: &str,
16550                scope: Option<ScopeSelector>,
16551            ) -> (
16552                RouteCtx,
16553                mpsc::Receiver<crate::router::OutboundFrame>,
16554                Frame,
16555            ) {
16556                self.next_connection += 1;
16557                let (ctx, rx) = wide_ctx(self.next_connection);
16558                let root = unique_project_root("scoped-open");
16559                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16560                    target: RouteTarget::ToolProvider {
16561                        module_id: target.to_string(),
16562                    },
16563                    identity: BindIdentity::new(
16564                        root.path().to_path_buf(),
16565                        "unit".to_string(),
16566                        "session".to_string(),
16567                    ),
16568                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
16569                        module_id: module_id.to_string(),
16570                        launch_nonce: nonce(module_id),
16571                    }),
16572                    consumer_capabilities: None,
16573                    role_versions: None,
16574                    admission_facts: None,
16575                    scope,
16576                })
16577                .unwrap();
16578                let frame = Frame::build(
16579                    FrameType::Request,
16580                    control_flags(),
16581                    0,
16582                    0,
16583                    self.next_connection,
16584                    body,
16585                )
16586                .unwrap();
16587                (ctx, rx, frame)
16588            }
16589
16590            /// Open and expect a refusal before anything is relayed.
16591            async fn refused(
16592                &mut self,
16593                opener: Option<&str>,
16594                target: &str,
16595                scope: Option<ScopeSelector>,
16596            ) -> String {
16597                self.refusal_body(opener, target, scope).await["code"]
16598                    .as_str()
16599                    .unwrap()
16600                    .to_string()
16601            }
16602
16603            async fn refusal_body(
16604                &mut self,
16605                opener: Option<&str>,
16606                target: &str,
16607                scope: Option<ScopeSelector>,
16608            ) -> Value {
16609                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16610                let replies = tokio::time::timeout(
16611                    Duration::from_secs(2),
16612                    self.handler.handle_control_frame(&ctx, frame),
16613                )
16614                .await
16615                .expect("the open must be refused before waiting for a bind ack")
16616                .unwrap();
16617                assert_eq!(replies.len(), 1, "{replies:?}");
16618                assert_eq!(replies[0].header.ty, FrameType::Error);
16619                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16620                assert!(
16621                    module_rx.try_recv().is_err(),
16622                    "a refused open relays nothing"
16623                );
16624                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16625                parse_error(&replies[0])
16626            }
16627
16628            /// Start an open and return its task and the bind the target got.
16629            async fn relayed(
16630                &mut self,
16631                opener: Option<&str>,
16632                target: &str,
16633                scope: Option<ScopeSelector>,
16634            ) -> Relayed {
16635                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16636                let handler = self.handler.clone();
16637                let task_ctx = ctx.clone();
16638                let task = tokio::spawn(async move {
16639                    handler
16640                        .handle_control_frame(&task_ctx, frame)
16641                        .await
16642                        .unwrap()
16643                });
16644                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16645                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16646                    .await
16647                    .expect("the target receives the relayed route.bind")
16648                    .unwrap()
16649                    .frame;
16650                Relayed {
16651                    target: target.to_string(),
16652                    client: ctx,
16653                    client_rx: rx,
16654                    task,
16655                    bind,
16656                }
16657            }
16658
16659            async fn ack(&self, relayed: &Relayed) {
16660                let (module, _) = &self.modules[&relayed.target];
16661                self.handler
16662                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16663                    .await
16664                    .unwrap();
16665            }
16666
16667            /// Open, ack and return the bound route.
16668            async fn bound(
16669                &mut self,
16670                opener: Option<&str>,
16671                target: &str,
16672                scope: Option<ScopeSelector>,
16673            ) -> Bound {
16674                let relayed = self.relayed(opener, target, scope).await;
16675                self.ack(&relayed).await;
16676                let Relayed {
16677                    target,
16678                    client,
16679                    mut client_rx,
16680                    task,
16681                    bind,
16682                } = relayed;
16683                assert!(
16684                    task.await.unwrap().is_empty(),
16685                    "the open is answered by commit"
16686                );
16687                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16688                Bound {
16689                    target,
16690                    client,
16691                    client_rx,
16692                    channel,
16693                    epoch,
16694                    bind,
16695                }
16696            }
16697
16698            fn live(&self, route: &Bound) -> bool {
16699                matches!(
16700                    self.forwarding
16701                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16702                        .unwrap(),
16703                    DataRoute::Client(DataRouteState::Bound(_))
16704                )
16705            }
16706        }
16707
16708        struct Relayed {
16709            target: String,
16710            client: RouteCtx,
16711            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16712            task: tokio::task::JoinHandle<Vec<Frame>>,
16713            bind: Frame,
16714        }
16715
16716        struct Bound {
16717            target: String,
16718            client: RouteCtx,
16719            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16720            channel: u16,
16721            epoch: u32,
16722            bind: Frame,
16723        }
16724
16725        impl Bound {
16726            /// The reason of the `route.closed` this client was sent, after
16727            /// checking it also got a GOODBYE on exactly this route.
16728            fn closed_reason(&mut self) -> RouteCloseReason {
16729                let mut reason = None;
16730                let mut goodbye = false;
16731                while let Ok(outbound) = self.client_rx.try_recv() {
16732                    let frame = outbound.frame;
16733                    match frame.header.ty {
16734                        FrameType::Goodbye => {
16735                            assert_eq!(
16736                                (frame.header.channel, frame.header.epoch),
16737                                (self.channel, self.epoch)
16738                            );
16739                            goodbye = true;
16740                        }
16741                        FrameType::Push => {
16742                            let ClientControlPush::RouteClosed {
16743                                reason: r,
16744                                module_id,
16745                                ..
16746                            } = serde_json::from_slice(&frame.body).unwrap()
16747                            else {
16748                                panic!("unexpected push");
16749                            };
16750                            assert_eq!(module_id, self.target);
16751                            reason = Some(r);
16752                        }
16753                        other => panic!("unexpected frame {other:?}"),
16754                    }
16755                }
16756                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16757                reason.expect("the client is told why the route closed")
16758            }
16759
16760            fn untouched(&mut self) -> bool {
16761                self.client_rx.try_recv().is_err()
16762            }
16763
16764            fn stamp(&self) -> Option<ScopeStamp> {
16765                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16766                    ModuleControlRequest::RouteBind { scope, .. } => scope,
16767                    other => panic!("expected a route.bind, got {other:?}"),
16768                }
16769            }
16770        }
16771
16772        #[tokio::test]
16773        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16774        ) {
16775            let mut rig = rig().await;
16776            let mut record = session(1);
16777            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16778            record.child_owners = vec![Principal::Reserved {
16779                module_id: MAGIC.to_string(),
16780            }];
16781            rig.sync(vec![record]).await;
16782            let scope = || Some(rig_selector("s", Some(1)));
16783
16784            // Admitted: the owner, a bare carrier to any module, a targeted
16785            // carrier to its listed module.
16786            rig.bound(Some(OWNER), PLEXUS, scope()).await;
16787            rig.bound(Some(AFT), OTHER, scope()).await;
16788            rig.bound(Some(BROCA), PLEXUS, scope()).await;
16789
16790            // Refused scope_not_carrier: a targeted carrier to an unlisted
16791            // module, a module that is not listed at all (a child owner is not
16792            // a carrier), and a direct key-holder.
16793            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16794                assert_eq!(
16795                    rig.refused(opener, target, scope()).await,
16796                    error_codes::SCOPE_NOT_CARRIER,
16797                    "{opener:?} -> {target}"
16798                );
16799            }
16800        }
16801
16802        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16803            ScopeSelector::new(
16804                Principal::Reserved {
16805                    module_id: OWNER.to_string(),
16806                },
16807                scope_ref.to_string(),
16808                scope_epoch,
16809            )
16810        }
16811
16812        #[tokio::test]
16813        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16814            let mut rig = rig().await;
16815            rig.sync(vec![session(1)]).await;
16816            for opener in [OWNER, AFT] {
16817                assert_eq!(
16818                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16819                        .await,
16820                    error_codes::SCOPE_EPOCH_REQUIRED,
16821                    "{opener}"
16822                );
16823            }
16824        }
16825
16826        #[tokio::test]
16827        async fn admission_separates_not_synced_not_live_and_ended() {
16828            let mut rig = rig().await;
16829            // Before the configured owner's first sync: retryable.
16830            let code = rig
16831                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16832                .await;
16833            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16834            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16835
16836            // An owner that is not configured will never sync: terminal.
16837            let ghost = ScopeSelector::new(
16838                Principal::Reserved {
16839                    module_id: "ghost".to_string(),
16840                },
16841                "s".to_string(),
16842                Some(1),
16843            );
16844            assert_eq!(
16845                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16846                error_codes::SCOPE_NOT_LIVE
16847            );
16848
16849            rig.sync(vec![session(2)]).await;
16850            assert_eq!(
16851                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16852                    .await,
16853                error_codes::SCOPE_NOT_LIVE
16854            );
16855            for epoch in [1, 3] {
16856                assert_eq!(
16857                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16858                        .await,
16859                    error_codes::SCOPE_ENDED,
16860                    "epoch {epoch}"
16861                );
16862            }
16863            // Control: the live epoch is admitted.
16864            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16865                .await;
16866        }
16867
16868        #[tokio::test]
16869        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16870            let mut rig = rig().await;
16871            rig.sync(vec![session(1)]).await;
16872            let route = rig
16873                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16874                .await;
16875            let stamp = route.stamp().expect("a scoped bind carries the stamp");
16876            assert_eq!(stamp.scope_ref, "s");
16877            assert_eq!(stamp.scope_epoch, 1);
16878            assert_eq!(stamp.kind, ScopeKind::Head);
16879            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16880            assert!(stamp.attributes.delegates);
16881            assert!(stamp.owner_authorized);
16882            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16883            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16884
16885            // broca owns a scope of its own on its own module connection; it is
16886            // not in scope_authority_owners, so its stamp is not authorized.
16887            let (broca, mut broca_rx) = wide_ctx(50);
16888            hello_via_sink(
16889                &rig.handler,
16890                &broca,
16891                &mut broca_rx,
16892                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16893            )
16894            .await;
16895            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16896                .await
16897                .unwrap();
16898            let own = ScopeSelector::new(
16899                Principal::Reserved {
16900                    module_id: BROCA.to_string(),
16901                },
16902                "b".to_string(),
16903                Some(1),
16904            );
16905            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16906            assert!(!route.stamp().unwrap().owner_authorized);
16907        }
16908
16909        #[tokio::test]
16910        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16911            let mut rig = rig().await;
16912            let mut record = head("s", 1);
16913            record.carriers = vec![carrier(AFT, None)];
16914            let flow_id = "Flow:run-7/step_2!~";
16915            record.attributes.flow_id = Some(flow_id.to_string());
16916            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16917                .await
16918                .unwrap();
16919            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16920                panic!("not a sync reply");
16921            };
16922            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16923            let route = rig
16924                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16925                .await;
16926            assert!(rig.live(&route), "the stamped bind committed");
16927            let stamp = route.stamp().expect("a flow scope carries a stamp");
16928            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16929            assert_eq!(stamp.attributes.agent_id, None);
16930            assert!(!stamp.attributes.delegates);
16931            assert!(stamp.owner_authorized);
16932            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16933            assert_eq!(unscoped.stamp(), None);
16934        }
16935
16936        #[tokio::test]
16937        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16938            let mut rig = rig_with_flow_support(false).await;
16939            let mut record = session(1);
16940            record.attributes.flow_id = Some("flow:7".to_string());
16941            rig.sync(vec![record]).await;
16942            for opener in [OWNER, AFT] {
16943                let body = rig
16944                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16945                    .await;
16946                assert_eq!(body["code"], "target_flow_unsupported");
16947                let message = body["message"].as_str().unwrap();
16948                for required in [PLEXUS, "flow-scopes/v1"] {
16949                    assert!(message.contains(required), "{message}");
16950                }
16951            }
16952        }
16953
16954        #[tokio::test]
16955        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16956            let mut rig = rig_with_flow_support(true).await;
16957            let mut record = session(1);
16958            record.attributes.flow_id = Some("flow:7".to_string());
16959            rig.sync(vec![record]).await;
16960            for opener in [OWNER, AFT] {
16961                let route = rig
16962                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16963                    .await;
16964                assert!(rig.live(&route));
16965                assert_eq!(
16966                    route.stamp().unwrap().attributes.flow_id.as_deref(),
16967                    Some("flow:7")
16968                );
16969            }
16970        }
16971
16972        #[tokio::test]
16973        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16974            use std::future::Future;
16975
16976            let mut rig = rig_with_flow_support(true).await;
16977            let mut record = session(1);
16978            record.attributes.flow_id = Some("flow:7".to_string());
16979            rig.sync(vec![record]).await;
16980            let (client, mut client_rx, frame) =
16981                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16982            // Hold the route-open response permit so admission sees the first
16983            // target but relay reservation cannot capture an endpoint yet.
16984            for _ in 0..64 {
16985                client.egress.try_send(route_bind_ack(1)).unwrap();
16986            }
16987            let handler = rig.handler.clone();
16988            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16989            std::future::poll_fn(|cx| {
16990                assert!(open.as_mut().poll(cx).is_pending());
16991                std::task::Poll::Ready(())
16992            })
16993            .await;
16994            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16995
16996            let old_connection = rig.modules[PLEXUS].0.connection_id;
16997            rig.handler.cleanup_connection(old_connection).unwrap();
16998            let (replacement, mut replacement_rx) = wide_ctx(200);
16999            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17000            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17001            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17002            let hello = Frame::build(
17003                FrameType::Hello,
17004                control_flags(),
17005                0,
17006                0,
17007                200,
17008                serde_json::to_vec(&body).unwrap(),
17009            )
17010            .unwrap();
17011            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17012
17013            client_rx.try_recv().unwrap();
17014            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17015                .await
17016                .expect("the replacement is refused without waiting for a bind ack")
17017                .unwrap();
17018            assert_eq!(replies.len(), 1);
17019            let body = parse_error(&replies[0]);
17020            assert_eq!(body["code"], "target_flow_unsupported");
17021            for required in [PLEXUS, "flow-scopes/v1"] {
17022                assert!(body["message"].as_str().unwrap().contains(required));
17023            }
17024            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17025            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17026            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17027            assert!(rig
17028                .handler
17029                .registry
17030                .get_module_by_connection(replacement.connection_id)
17031                .unwrap()
17032                .is_some());
17033        }
17034
17035        #[tokio::test]
17036        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17037            use std::future::Future;
17038            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17039            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17040            const NEXT_HASH: &str =
17041                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17042            let mut rig = rig().await;
17043            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17044            record.carriers.push(ScopeCarrier::new(Principal::Direct));
17045            rig.sync(vec![record.clone()]).await;
17046            let (client, mut rx, frame) = rig.open_frame(
17047                None,
17048                PLEXUS,
17049                Some(
17050                    rig.selector("s", Some(1))
17051                        .with_bind_proof(Some(PROOF.into())),
17052                ),
17053            );
17054            // Hold egress reservation while the scope owner changes its bind-proof
17055            // hash, so a proof valid at admission must be checked again at relay.
17056            for _ in 0..64 {
17057                client.egress.try_send(route_bind_ack(1)).unwrap();
17058            }
17059            let handler = rig.handler.clone();
17060            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17061            std::future::poll_fn(|cx| {
17062                assert!(open.as_mut().poll(cx).is_pending());
17063                std::task::Poll::Ready(())
17064            })
17065            .await;
17066            record.bind_proof_sha256 = Some(NEXT_HASH.into());
17067            rig.sync(vec![record]).await;
17068            rx.try_recv().unwrap();
17069            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17070                .await
17071                .unwrap()
17072                .unwrap();
17073            assert_eq!(
17074                parse_error(&replies[0])["code"],
17075                error_codes::SCOPE_BIND_PROOF_MISMATCH
17076            );
17077            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17078            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17079            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17080        }
17081
17082        #[tokio::test]
17083        async fn scope_bind_proof_reserved_openers_ignore_proof() {
17084            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17085            let mut rig = rig().await;
17086            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17087                .await;
17088            for opener in [OWNER, AFT] {
17089                for proof in [None, Some("malformed+proof=")] {
17090                    let route = rig
17091                        .bound(
17092                            Some(opener),
17093                            PLEXUS,
17094                            Some(
17095                                rig.selector("s", Some(1))
17096                                    .with_bind_proof(proof.map(str::to_string)),
17097                            ),
17098                        )
17099                        .await;
17100                    assert!(rig.live(&route));
17101                }
17102            }
17103        }
17104
17105        #[tokio::test]
17106        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17107            use std::future::Future;
17108            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17109            const NEXT_HASH: &str =
17110                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17111            let mut rig = rig().await;
17112            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17113                .await;
17114            let (client, mut rx, frame) =
17115                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17116            for _ in 0..64 {
17117                client.egress.try_send(route_bind_ack(1)).unwrap();
17118            }
17119            let handler = rig.handler.clone();
17120            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17121            std::future::poll_fn(|cx| {
17122                assert!(open.as_mut().poll(cx).is_pending());
17123                std::task::Poll::Ready(())
17124            })
17125            .await;
17126            rig.sync(vec![
17127                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17128            ])
17129            .await;
17130            rx.try_recv().unwrap();
17131            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17132            let bind = std::future::poll_fn(|cx| {
17133                assert!(open.as_mut().poll(cx).is_pending());
17134                module_rx.poll_recv(cx)
17135            })
17136            .await
17137            .unwrap()
17138            .frame;
17139            handler
17140                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17141                .await
17142                .unwrap();
17143            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17144                .await
17145                .unwrap()
17146                .unwrap();
17147            assert_eq!(replies.len(), 1);
17148            // Proof rechecking allowed the relay. Independent tag settlement
17149            // still asks the opener to retry after any same-epoch content change.
17150            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17151            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17152            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17153            let route = rig
17154                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17155                .await;
17156            assert!(rig.live(&route));
17157        }
17158
17159        #[tokio::test]
17160        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17161        {
17162            let mut rig = rig_with_flow_support(false).await;
17163            rig.sync(vec![session(1)]).await;
17164            let route = rig
17165                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17166                .await;
17167            assert!(rig.live(&route));
17168            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17169            let mut record = session(1);
17170            record.attributes.flow_id = Some("flow:7".to_string());
17171            rig.sync(vec![record]).await;
17172            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17173            assert!(rig.live(&unscoped));
17174            assert_eq!(unscoped.stamp(), None);
17175        }
17176
17177        #[tokio::test]
17178        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17179            let mut rig = rig().await;
17180            let mut record = session(1);
17181            record.attributes.flow_id = Some("flow:7".to_string());
17182            rig.sync(vec![record.clone()]).await;
17183            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17184            let mut owner_route = rig
17185                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17186                .await;
17187            let mut carrier_route = rig
17188                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17189                .await;
17190            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17191            rig.sync(vec![record.clone()]).await;
17192            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17193            assert!(rig.live(&owner_route) && owner_route.untouched());
17194            assert!(rig.live(&carrier_route) && carrier_route.untouched());
17195
17196            record.attributes.flow_id = Some("flow:8".to_string());
17197            rig.sync(vec![record]).await;
17198            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17199            let before = before.unwrap();
17200            assert_eq!(after.scope_epoch, before.scope_epoch);
17201            assert!(after.version > before.version);
17202            for route in [&mut owner_route, &mut carrier_route] {
17203                assert!(!rig.live(route));
17204                assert_eq!(
17205                    route.closed_reason(),
17206                    RouteCloseReason::ScopeDelegationChanged
17207                );
17208            }
17209            assert!(rig.live(&unscoped) && unscoped.untouched());
17210            // Each provider also receives a GOODBYE for its drained route;
17211            // consume it before expecting the next route.bind on that sink.
17212            for target in [PLEXUS, OTHER] {
17213                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17214                let goodbye = module_rx
17215                    .try_recv()
17216                    .expect("the provider sees the drain")
17217                    .frame;
17218                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17219                assert!(module_rx.try_recv().is_err());
17220            }
17221            let rebound = rig
17222                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17223                .await;
17224            assert_eq!(
17225                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17226                Some("flow:8")
17227            );
17228        }
17229
17230        /// The owner's sync lands between admission and the module's ack. The
17231        /// open is refused by name, the module's other routes stay up, and the
17232        /// reserved pair is released. Changed content is retryable; an ended
17233        /// scope is not.
17234        #[tokio::test]
17235        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17236            let mut rig = rig().await;
17237            rig.sync(vec![session(1)]).await;
17238            let mut cotenant = rig
17239                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17240                .await;
17241
17242            let mut changed = session(1);
17243            changed.child_owners.push(Principal::Reserved {
17244                module_id: MAGIC.to_string(),
17245            });
17246            let mut ended = None;
17247            for (code, next) in [
17248                (error_codes::SCOPE_CHANGED, vec![changed]),
17249                (error_codes::SCOPE_ENDED, Vec::new()),
17250            ] {
17251                let relayed = rig
17252                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17253                    .await;
17254                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17255                ended = Some(next.is_empty());
17256                rig.sync(next).await;
17257                rig.ack(&relayed).await;
17258                let replies = relayed.task.await.unwrap();
17259                assert_eq!(replies.len(), 1, "{replies:?}");
17260                assert_eq!(parse_error(&replies[0])["code"], code);
17261                assert_eq!(
17262                    subc_protocol::error_codes::is_retryable_route_open(code),
17263                    code == error_codes::SCOPE_CHANGED
17264                );
17265                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17266                // The module is told to drop just the binding it created.
17267                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17268                // Collected, because ending the scope also closes the co-tenant
17269                // route, whose GOODBYE comes first.
17270                let mut goodbyes = Vec::new();
17271                while let Ok(outbound) = plexus_rx.try_recv() {
17272                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17273                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17274                }
17275                assert!(
17276                    goodbyes.contains(&(bind_channel, bind_epoch)),
17277                    "{goodbyes:?}"
17278                );
17279                assert!(rig
17280                    .handler
17281                    .registry
17282                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17283                    .unwrap()
17284                    .is_some());
17285            }
17286            assert_eq!(ended, Some(true));
17287            // The co-tenant stayed up through the change, and closed only when
17288            // the scope ended, by the drain rule rather than by the commit.
17289            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17290        }
17291
17292        /// Each row of the drain table on one set of routes: the owner's, a
17293        /// bare carrier's, and a targeted carrier's to each of its targets.
17294        #[tokio::test]
17295        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17296            struct Case {
17297                name: &'static str,
17298                change: fn(&mut ScopeRecord),
17299                /// Closed routes by index: owner->plexus, aft->plexus,
17300                /// broca->plexus, broca->other.
17301                closed: [Option<RouteCloseReason>; 4],
17302            }
17303            use RouteCloseReason::*;
17304            let cases = [
17305                Case {
17306                    name: "a carrier entry removed",
17307                    change: |r| {
17308                        r.carriers.retain(|c| {
17309                            c.principal
17310                                != Principal::Reserved {
17311                                    module_id: AFT.to_string(),
17312                                }
17313                        })
17314                    },
17315                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17316                },
17317                Case {
17318                    name: "a target removed from a carrier",
17319                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17320                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
17321                },
17322                Case {
17323                    name: "a bare carrier narrowed to targets",
17324                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17325                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17326                },
17327                Case {
17328                    name: "delegates turned off",
17329                    change: |r| r.attributes.delegates = false,
17330                    closed: [Some(ScopeDelegationChanged); 4],
17331                },
17332                Case {
17333                    name: "agent_id changed",
17334                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17335                    closed: [Some(ScopeDelegationChanged); 4],
17336                },
17337                Case {
17338                    name: "a carrier added, child owners changed, the record re-sent",
17339                    change: |r| {
17340                        r.carriers.push(carrier(MAGIC, None));
17341                        r.child_owners.push(Principal::Reserved {
17342                            module_id: MAGIC.to_string(),
17343                        });
17344                    },
17345                    closed: [None; 4],
17346                },
17347                Case {
17348                    name: "a target added",
17349                    change: |r| {
17350                        r.carriers[1]
17351                            .targets
17352                            .as_mut()
17353                            .unwrap()
17354                            .push("third".to_string())
17355                    },
17356                    closed: [None; 4],
17357                },
17358                Case {
17359                    name: "delegates turned on",
17360                    change: |r| r.attributes.delegates = true,
17361                    closed: [None; 4],
17362                },
17363            ];
17364            for case in cases {
17365                let mut rig = rig().await;
17366                rig.sync(vec![session(1)]).await;
17367                let scope = || Some(rig_selector("s", Some(1)));
17368                let mut routes = [
17369                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
17370                    rig.bound(Some(AFT), PLEXUS, scope()).await,
17371                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
17372                    rig.bound(Some(BROCA), OTHER, scope()).await,
17373                ];
17374                let mut record = session(1);
17375                (case.change)(&mut record);
17376                rig.sync(vec![record]).await;
17377                for (index, expected) in case.closed.iter().enumerate() {
17378                    let route = &mut routes[index];
17379                    match expected {
17380                        Some(reason) => {
17381                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
17382                            assert_eq!(
17383                                route.closed_reason(),
17384                                *reason,
17385                                "{}: route {index}",
17386                                case.name
17387                            );
17388                        }
17389                        None => {
17390                            assert!(rig.live(route), "{}: route {index} closed", case.name);
17391                            assert!(
17392                                route.untouched(),
17393                                "{}: route {index} was told something",
17394                                case.name
17395                            );
17396                        }
17397                    }
17398                }
17399            }
17400        }
17401
17402        #[tokio::test]
17403        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17404            // Removed, and replaced by a higher epoch.
17405            for next in [Vec::new(), vec![session(2)]] {
17406                let mut rig = rig().await;
17407                rig.sync(vec![session(1)]).await;
17408                let mut route = rig
17409                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17410                    .await;
17411                rig.sync(next).await;
17412                assert!(!rig.live(&route));
17413                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17414            }
17415
17416            // A child whose parent ends: its routes close as parent-ended, the
17417            // child stays live, and routes under the parent close as ended.
17418            let mut rig = rig().await;
17419            let mut child = session(1);
17420            child.scope_ref = "child".to_string();
17421            child.kind = ScopeKind::Worker;
17422            child.parent = Some(ScopeParent::new(
17423                Principal::Reserved {
17424                    module_id: OWNER.to_string(),
17425                },
17426                "s".to_string(),
17427                1,
17428            ));
17429            rig.sync(vec![session(1), child.clone()]).await;
17430            let mut child_route = rig
17431                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17432                .await;
17433            assert_eq!(
17434                child_route.stamp().unwrap().parent_state,
17435                Some(ParentState::Linked)
17436            );
17437            rig.sync(vec![child]).await;
17438            assert!(!rig.live(&child_route));
17439            assert_eq!(
17440                child_route.closed_reason(),
17441                RouteCloseReason::ScopeParentEnded
17442            );
17443        }
17444
17445        #[tokio::test]
17446        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17447        ) {
17448            let mut rig = rig().await;
17449            rig.sync(vec![session(1)]).await;
17450            let mut route = rig
17451                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17452                .await;
17453            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17454            rig.sync(vec![session(1)]).await;
17455            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17456            assert!(rig.live(&route) && route.untouched());
17457
17458            // A call in flight on the route when another carrier is added. A
17459            // forwarded REQUEST holds one credit on the route's flow until the
17460            // module answers; the router takes it exactly like this.
17461            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17462                .forwarding
17463                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17464                .unwrap()
17465            else {
17466                panic!("the route is bound");
17467            };
17468            binding.flow.acquire_tagged(9, false).await.unwrap();
17469            let mut widened = session(1);
17470            widened.carriers.push(carrier(MAGIC, None));
17471            rig.sync(vec![widened]).await;
17472            assert!(rig.live(&route) && route.untouched());
17473            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17474            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17475            // The call's credit is still held on an open flow, so its answer
17476            // will be delivered: closing the route would have closed the flow.
17477            assert_eq!(binding.flow.in_flight(), 1);
17478            binding
17479                .flow
17480                .acquire_tagged(10, false)
17481                .await
17482                .expect("the flow is still open");
17483        }
17484
17485        /// A swap's superseded endpoint keeps its routes until drained; ending
17486        /// the scope closes them there too.
17487        #[tokio::test]
17488        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17489            let mut rig = rig().await;
17490            rig.sync(vec![session(1)]).await;
17491            let mut on_incumbent = rig
17492                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17493                .await;
17494
17495            // Swap plexus: register a candidate and cut over, leaving the
17496            // incumbent superseded with the route still on it.
17497            let (candidate, _candidate_rx) = wide_ctx(9);
17498            let registration = rig
17499                .handler
17500                .registry
17501                .register_candidate_with_control_ops(
17502                    manifest(PLEXUS, PROTOCOL_VERSION),
17503                    PROTOCOL_VERSION,
17504                    candidate.connection_id,
17505                    module_baseline_control_ops(),
17506                )
17507                .unwrap();
17508            rig.forwarding
17509                .register_candidate_module_connection(
17510                    candidate.connection_id,
17511                    PLEXUS.to_string(),
17512                    PROTOCOL_VERSION,
17513                    manifest_concurrency(&registration.manifest),
17514                    candidate.egress.clone(),
17515                )
17516                .unwrap();
17517            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17518            rig.handler
17519                .registry
17520                .promote_candidate(PLEXUS)
17521                .unwrap()
17522                .unwrap();
17523            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17524
17525            rig.sync(Vec::new()).await;
17526            assert!(!rig.live(&on_incumbent));
17527            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17528            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17529            let goodbye = incumbent_rx
17530                .try_recv()
17531                .expect("the superseded endpoint is told")
17532                .frame;
17533            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17534        }
17535    }
17536}
17537
17538#[cfg(test)]
17539mod concurrency_default_exposure_tests {
17540    use super::*;
17541
17542    fn hello_body(role_json: &str) -> Vec<u8> {
17543        format!(
17544            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17545        )
17546        .into_bytes()
17547    }
17548
17549    fn manifest_from(body: &[u8]) -> ModuleManifest {
17550        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17551        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17552            .expect("manifest parses")
17553    }
17554
17555    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17556
17557    #[test]
17558    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17559        let body = hello_body(&format!(
17560            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17561        ));
17562        let manifest = manifest_from(&body);
17563        // Precondition: serde really resolved it to the default, so the typed
17564        // manifest alone cannot answer the question this probe exists for.
17565        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17566        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17567    }
17568
17569    #[test]
17570    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17571        let body = hello_body(&format!(
17572            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17573        ));
17574        let manifest = manifest_from(&body);
17575        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17576        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17577    }
17578
17579    #[test]
17580    fn non_management_roles_are_never_reported() {
17581        let body = hello_body(
17582            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17583        );
17584        let manifest = manifest_from(&body);
17585        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17586    }
17587}