Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
26        validate_hello_role_declarations, validate_hello_self_signal_declarations,
27        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
28        ProviderRole,
29    },
30    scope::{
31        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
32        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
33        SCOPE_SYNC_OP,
34    },
35    session::{
36        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
37        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
38        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
39        ROLE_VERSIONS_FIELD,
40    },
41    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
42    Priority, RouteTarget, PROTOCOL_VERSION,
43};
44use tokio::time::{timeout_at, Instant};
45use tracing::{debug, info, warn};
46
47use crate::{
48    caller_identity::CallerIdentity,
49    capability_requirements::{
50        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
51        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
52        RuntimeModule,
53    },
54    child_roster::DaemonShutdownFlag,
55    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
56    forwarding::{
57        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
58        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
59        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
60    },
61    observability::{
62        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
63    },
64    privileged_prompt::{prompt_text, PrivilegedOperation},
65    provenance::{
66        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
67    },
68    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
69    router::{RouteCtx, RouterError},
70    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
71    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
72    stderr_tail::{CaptureState, TailEntry},
73    supervise::{
74        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
75        SupervisorHandle, SwapHelloAdmission,
76    },
77    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
78};
79
80/// Lowest envelope version this subc build will negotiate.
81///
82/// Module HELLO negotiation is exact: peers must use the daemon's locked
83/// protocol version. Older and newer peers receive `version_unsupported` and
84/// are not registered.
85pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
86
87const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
88const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
89const CAP_PING_PONG: &str = "ping_pong_v1";
90const CAP_SESSION_ATTACH: &str = "session_attach_v1";
91const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
92
93const SUBC_CONTROL_OPS: &[&str] = &[
94    ops::ACTIVITY_RECORD,
95    ops::ACTIVITY_QUERY,
96    ops::ACTIVITY_SUBSCRIBE,
97    ops::SERVER_DESCRIBE,
98    ops::CATALOG_LIST,
99    ops::ROUTE_OPEN,
100    ops::ROUTE_POLL,
101    ops::ROUTE_CLOSING,
102    ops::ROUTE_CLOSED,
103    ops::SUPERVISOR_LIST,
104    ops::SUPERVISOR_RESTART,
105    ops::SUPERVISOR_SWAP,
106    ops::SUPERVISOR_RELOAD,
107    ops::SUPERVISOR_RESCAN,
108    ops::SUPERVISOR_RELEASE_RESERVED,
109    ops::SUPERVISOR_SET_ENABLED,
110    ops::SUPERVISOR_HEALTH_PROBE,
111    ops::SUPERVISOR_HEALTH,
112    ops::SUPERVISOR_STDERR_TAIL,
113    ops::SUPERVISOR_TERMINALS,
114    ops::SUPERVISOR_ROUTES,
115    ops::SUPERVISOR_PROVENANCE,
116    ops::SUPERVISOR_SPAWN_SNAPSHOT,
117    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
118];
119
120/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
121/// announced by a capability in `server.describe` instead, and clients check
122/// that capability before sending it. `ck daemon --json` prints `subc_ops`
123/// verbatim and scripts read that output, so a new name there would change it.
124/// A malformed body for one of these is still refused `invalid_control_body`
125/// rather than as an unknown op.
126const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
127
128mod rescan_checked;
129use rescan_checked::{RescanMode, RescanPlan};
130
131const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
132    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
133    "supervisor.live_roots",
134    SCOPE_SYNC_OP,
135    SCOPE_APPLY_OP,
136    SCOPE_DESCRIBE_OP,
137    "operator.confirm",
138];
139
140/// Module-originated ops the daemon answers but does not advertise in
141/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
142/// belongs to is incomplete, so no module is told it works before it does.
143const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
144
145const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
146
147/// How long subc waits for a module to ack a relayed route.bind before returning
148/// `module_timeout`. The ack waits on the module's own configure, which for AFT
149/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
150/// and DB-open work — on a cold page cache or a large repo that legitimately
151/// exceeds a couple of seconds. The default is generous because rejecting a VALID
152/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
153/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
154pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
155
156/// How many CONSECUTIVE full-budget relay timeouts against one target module
157/// open that module's bind-relay breaker.
158///
159/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
160/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
161/// so three consecutive full-budget timeouts take ~36s to observe: every client
162/// whose open contributed to opening the breaker had already given up on its
163/// own. That is what makes opening the breaker unable to turn a call that would
164/// have succeeded into a refusal — it can only make an already-failing module
165/// fail faster.
166///
167/// Two would be reachable inside one default deadline. One would convict a
168/// module on a single cold-cache bind, which is exactly the valid-but-slow case
169/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
170pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
171
172/// How long a module's bind-relay breaker stays open before exactly one
173/// `route.open` is let through as a probe.
174///
175/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
176/// re-pays a full-budget stall almost continuously, and the breaker stops being
177/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
178/// deadline: a client that starts retrying after the module recovers has to get
179/// a probe opportunity inside its own deadline, or the breaker converts a
180/// recovered module into a failed call — the failure it exists to prevent,
181/// pointed the other way.
182///
183/// 20s sits between those with room on both sides, and it caps what a wedged
184/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
185/// rather than one per `route.open` per connection. The stall that motivated
186/// this, with its measurements, is written up in
187/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
188/// waited the whole budget out.
189pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
190
191const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
192const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
193
194fn reload_verdict(
195    configured: &Path,
196    spawned_from: Option<&Path>,
197    image: subc_control::RunningImageAgreement,
198) -> PendingReloadVerdict {
199    let path = match spawned_from {
200        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
201            ReloadPathAgreement::Match
202        }
203        Some(spawned_from) => ReloadPathAgreement::Mismatch {
204            configured: configured.to_path_buf(),
205            spawned_from: spawned_from.to_path_buf(),
206        },
207        None => ReloadPathAgreement::Unavailable {
208            reason: if matches!(
209                image,
210                subc_control::RunningImageAgreement::Unavailable {
211                    reason: subc_control::RunningImageUnavailableReason::NotRunning
212                }
213            ) {
214                ReloadPathUnavailableReason::NotRunning
215            } else {
216                ReloadPathUnavailableReason::SpawnedPathUnavailable
217            },
218        },
219    };
220    PendingReloadVerdict {
221        path,
222        image,
223        spec: None,
224    }
225}
226
227fn launch_spec_agreement(
228    configured: &crate::supervise::ModuleSpec,
229    running: bool,
230    spawned: Option<&crate::supervise::LaunchSpecDigest>,
231) -> subc_control::LaunchSpecAgreement {
232    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
233    if !running {
234        return Agreement::Unavailable {
235            reason: Reason::NotRunning,
236        };
237    }
238    match spawned {
239        None => Agreement::Unavailable {
240            reason: Reason::SpawnedSpecUnavailable,
241        },
242        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
243            Agreement::Match
244        }
245        Some(_) => Agreement::Mismatch,
246    }
247}
248
249fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
250    if configured == spawned_from {
251        return true;
252    }
253    #[cfg(windows)]
254    {
255        // Windows spawn records the resolved verbatim path (\\?\...), while
256        // configuration keeps its original spelling. Resolve both with the OS
257        // instead of stripping prefixes or folding case, which could conflate
258        // distinct names in case-sensitive directories. Different resolved
259        // paths still differ even if their files have identical content.
260        match (
261            std::fs::canonicalize(configured),
262            std::fs::canonicalize(spawned_from),
263        ) {
264            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
265            _ => false,
266        }
267    }
268    #[cfg(not(windows))]
269    {
270        false
271    }
272}
273
274#[derive(Clone)]
275struct DaemonProvenanceFacts {
276    build: DaemonBuildProvenance,
277    pid: Option<u32>,
278    started_at_ms: Option<u64>,
279    start_clock: Option<crate::clock::StartClock>,
280    executable_path: Option<PathBuf>,
281    executable_identity: Option<SpawnedFileIdentity>,
282    process_start_time: Option<u64>,
283    probe: ExecutableIdentityProbe,
284}
285
286impl Default for DaemonProvenanceFacts {
287    fn default() -> Self {
288        Self {
289            build: DaemonBuildProvenance {
290                build_git_sha: None,
291                build_lock_digest: None,
292            },
293            pid: None,
294            started_at_ms: None,
295            start_clock: None,
296            executable_path: None,
297            executable_identity: None,
298            process_start_time: None,
299            probe: ExecutableIdentityProbe::default(),
300        }
301    }
302}
303
304#[derive(Debug, Clone)]
305struct SupervisorRescanContext {
306    supervisor: Supervisor,
307    config_path: PathBuf,
308    configured_port: Option<u16>,
309    storage_config: Option<crate::daemon_config::StorageConfig>,
310    admission_facts_carrier_module_id: Option<String>,
311    admission_facts_targets: Option<Vec<String>>,
312    scope_authority_owners: Vec<String>,
313    privileged_operator_prompt: PrivilegedOperatorPrompt,
314    /// Counts executed rescans that set out to change the supervised modules'
315    /// staged or applied configuration, counted before the first change so a
316    /// rescan that fails partway still counts. With the daemon incarnation it
317    /// names the state a `supervisor.rescan_checked` preview was computed
318    /// against.
319    generation: Arc<std::sync::atomic::AtomicU64>,
320}
321
322/// Refusal labels passed to `observe_route_open_refusal` that mean the target
323/// module is not serving right now, and so open or extend an outage in the
324/// route outage tracker. Every one of them is only reachable after the target
325/// was found in the registry, which is what keeps an arbitrary client-chosen
326/// id from ever creating tracker state.
327///
328/// Deliberately absent: `not_registered` and `removed` (the id may be
329/// anything a client sent, and a removed module is gone on purpose),
330/// `protocol_none` (such a module never serves routes, so nothing is out),
331/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
332/// capability and admission-facts refusals (they refuse the caller, not a
333/// module outage), and `relay_reservation_failed` (its code ranges over
334/// capacity limits as well as a vanished connection). Capacity, breaker,
335/// relay-timeout and module-rejection refusals do not pass through that
336/// function at all; the breaker logs its own transitions.
337///
338/// The two not-serving refusals that bypass that function record themselves
339/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
340/// `required_capability_unprovided` is not tracked: the module itself is up,
341/// and the outage belongs to the missing provider.
342const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
343    "reloading",
344    "supervisor_not_live",
345    "registration_not_active",
346    "no_forwarding_connection",
347    "relay_send_failed",
348];
349
350/// Real channel-0 control handler for subc itself.
351#[derive(Clone)]
352pub struct ControlHandler {
353    registry: Arc<Registry>,
354    activity: Option<crate::activity::ActivityLedger>,
355    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
356    forwarding: Arc<ForwardingTable>,
357    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
358    supervisor: SupervisorHandle,
359    subc_capabilities: Arc<[String]>,
360    /// Daemon-wide route.bind relay budget. Used as the fallback when the
361    /// target module has no per-module override in
362    /// `route_bind_relay_timeouts`.
363    route_bind_relay_timeout: Duration,
364    /// Per-module route.bind relay budget overrides, keyed by module id. When
365    /// `handle_route_open` resolves the deadline for a target module, a
366    /// per-module entry wins over the daemon-wide value above.
367    route_bind_relay_timeouts: BTreeMap<String, Duration>,
368    /// Per-target-module bind-relay breaker state. Shared with the forwarding
369    /// table, which is where a new module connection resets it.
370    route_bind_breakers: RouteBindBreakers,
371    /// Live relay admissions keyed by target module. Shared through the
372    /// forwarding table so cloned or separately built handlers enforce one cap.
373    route_bind_concurrency: RouteBindConcurrency,
374    /// Start and end of each module's not-serving period as seen by
375    /// `route.open`, so an outage gets one line at each edge instead of only
376    /// the per-refusal INFO lines. Taken from the forwarding table, so every
377    /// handler built over one table shares it.
378    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
379    /// Consecutive relay timeouts that open a module's breaker.
380    route_bind_breaker_threshold: u32,
381    /// How long a breaker stays open before one probe is admitted.
382    route_bind_breaker_cooldown: Duration,
383    health_probe_timeout: Duration,
384    /// Central storage policy. When set, each registering module receives its
385    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
386    storage_config: Option<crate::daemon_config::StorageConfig>,
387    /// The machine id established at boot, served on every HELLO_ACK and on
388    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
389    /// changes the file, never this value. `None` serves no id.
390    machine_id: Option<crate::machine_id::MachineId>,
391    admission_facts_carrier_module_id: Option<String>,
392    admission_facts_targets: Option<Vec<String>>,
393    /// Scope records with their sync authorities and tombstones; see
394    /// `crate::scopes`. Shared by clones of this handler, so every connection
395    /// reads and writes one table.
396    scopes: Arc<RwLock<ScopeTable>>,
397    /// Wakes the expiry loop when an accepted scope change may have added a
398    /// deadline, so the loop can wait without ticking while no live scope has
399    /// one. A permit is stored when nothing waits, so no wake is lost.
400    scope_deadline_added: Arc<tokio::sync::Notify>,
401    /// The configured `scope_authority_owners`, kept so a rescan can report a
402    /// changed value as needing a daemon restart; rescan never applies it.
403    scope_authority_owners: Vec<String>,
404    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
405    privileged_operator_prompt: PrivilegedOperatorPrompt,
406    /// The local and peer address of each accepted TCP client connection, read
407    /// from the daemon's own socket, so a privileged request can name its caller.
408    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
409    #[cfg(test)]
410    caller_lookup: Option<TestCallerLookup>,
411    /// The launch nonce each module connection presented at HELLO, which is how
412    /// a `scope.sync` is matched to the owner's current launch.
413    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
414    rescan: Option<SupervisorRescanContext>,
415    connected_clients: ConnectedClients,
416    counters: DaemonCounters,
417    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
418    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
419    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
420    /// their usual severity (see [`log_requirement_events`]). A handler built
421    /// without one has a flag that is never set.
422    daemon_shutdown: DaemonShutdownFlag,
423    daemon_provenance: DaemonProvenanceFacts,
424    #[cfg(test)]
425    control_dispatch_delay: Option<Duration>,
426    #[cfg(test)]
427    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
428    /// Holds `route.open` after the route.bind is queued to the module and
429    /// before it is marked enqueued, so a test can make the module's reply (or
430    /// a client teardown) land exactly inside that window.
431    #[cfg(test)]
432    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
433    /// Holds a HELLO after its registry entry is reserved and before its
434    /// forwarding endpoint is installed, so a test can read the catalog and
435    /// open routes while the module is registered but not yet routable. A
436    /// blocking barrier because the HELLO handler is synchronous; the test
437    /// runs the HELLO on its own thread.
438    #[cfg(test)]
439    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
440}
441
442impl fmt::Debug for ControlHandler {
443    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
444        f.debug_struct("ControlHandler")
445            .field("registry", &self.registry)
446            .field("forwarding", &self.forwarding)
447            .field("process_liveness", &self.process_liveness.is_some())
448            .field("supervisor", &self.supervisor)
449            .field("subc_capabilities", &self.subc_capabilities)
450            .finish()
451    }
452}
453
454struct RouteOpenRequest {
455    target: RouteTarget,
456    identity: BindIdentity,
457    consumer_identity: Option<ConsumerIdentity>,
458    consumer_capabilities: Option<Vec<String>>,
459    role_versions: Option<BTreeMap<String, String>>,
460    admission_facts: Option<serde_json::Value>,
461    scope: Option<ScopeSelector>,
462}
463
464#[derive(Clone, Copy)]
465enum RouteOpenPhase {
466    Admission,
467    ProjectRoot,
468    EgressReservation,
469    BindSend,
470    BindAckWait,
471}
472
473/// Store elapsed time for each route-open phase, including attempts that end
474/// in an early refusal or backend error. Admission also includes rechecks and
475/// commit bookkeeping performed between the other timed phases.
476struct RouteOpenTimings {
477    module_id: String,
478    principal: String,
479    admission: Duration,
480    project_root: Duration,
481    egress_reservation: Duration,
482    bind_send: Duration,
483    bind_ack_wait: Duration,
484    phase: Option<RouteOpenPhase>,
485    phase_started_at: Instant,
486}
487
488impl RouteOpenTimings {
489    fn new() -> Self {
490        Self {
491            module_id: String::new(),
492            // Refusals before attestation must not claim a verified principal.
493            principal: "unresolved".to_string(),
494            admission: Duration::ZERO,
495            project_root: Duration::ZERO,
496            egress_reservation: Duration::ZERO,
497            bind_send: Duration::ZERO,
498            bind_ack_wait: Duration::ZERO,
499            phase: Some(RouteOpenPhase::Admission),
500            phase_started_at: Instant::now(),
501        }
502    }
503
504    fn finish(&mut self) {
505        let elapsed = self.phase_started_at.elapsed();
506        match self.phase.take() {
507            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
508            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
509            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
510            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
511            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
512            None => {}
513        }
514    }
515
516    fn enter(&mut self, phase: RouteOpenPhase) {
517        self.finish();
518        self.phase = Some(phase);
519        self.phase_started_at = Instant::now();
520    }
521}
522
523struct RouteBindReservationGuard {
524    forwarding: Arc<ForwardingTable>,
525    endpoint: ModuleEndpointId,
526    relay_corr: u64,
527    armed: bool,
528}
529
530struct ModuleControlRpcGuard {
531    forwarding: Arc<ForwardingTable>,
532    endpoint: ModuleEndpointId,
533    corr: u64,
534    armed: bool,
535}
536
537impl ModuleControlRpcGuard {
538    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
539        Self {
540            forwarding,
541            endpoint,
542            corr,
543            armed: true,
544        }
545    }
546
547    fn disarm(&mut self) {
548        self.armed = false;
549    }
550}
551
552impl Drop for ModuleControlRpcGuard {
553    fn drop(&mut self) {
554        if self.armed {
555            let _ = self
556                .forwarding
557                .cancel_module_control_rpc(self.endpoint, self.corr);
558        }
559    }
560}
561
562impl RouteBindReservationGuard {
563    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
564        Self {
565            forwarding,
566            endpoint,
567            relay_corr,
568            armed: true,
569        }
570    }
571
572    fn release_and_disarm(&mut self) {
573        if !self.armed {
574            return;
575        }
576        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
577            self.endpoint,
578            self.relay_corr,
579            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
580        ) {
581            send_goodbye_target_best_effort(
582                &self.forwarding.counters(),
583                &target,
584                "canceled route.bind",
585            );
586        }
587        self.armed = false;
588    }
589
590    fn disarm(&mut self) {
591        self.armed = false;
592    }
593}
594
595impl Drop for RouteBindReservationGuard {
596    fn drop(&mut self) {
597        self.release_and_disarm();
598    }
599}
600
601/// Per-target-module circuit breaker around the `route.bind` relay.
602///
603/// The connection reader is serial per connection, so a module whose `on_bind`
604/// sits on the ack blocks every LATER frame on the connections that call it,
605/// including calls to unrelated modules. This does not make any module's bind
606/// fast; it stops the daemon paying the full budget again and again for a
607/// condition it has already observed.
608///
609/// State is keyed by TARGET MODULE and shared by every connection: a wedged
610/// module wedges everyone, so what one connection learned should protect the
611/// rest.
612///
613/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
614/// relay to that module has actually timed out, and is removed again when a
615/// relay is accepted or the module reconnects, so it cannot grow with traffic
616/// or with modules that behave.
617///
618/// # Why a `std` mutex here is not the head-of-line defect again
619///
620/// Acquisition never awaits. The critical section is a hash lookup plus a few
621/// integer updates, with no I/O and no `.await` inside it, so a reader task
622/// cannot be descheduled behind it the way it can behind
623/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
624/// primitive, held for the same kind of work, as the refusal counter this very
625/// path already increments.
626///
627/// It is also NOT on the data-plane splice path: only `route.open` and module
628/// registration touch it, so bound-route frames gain no state check and no
629/// contention.
630#[derive(Debug, Clone, Default)]
631pub(crate) struct RouteBindBreakers {
632    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
633}
634
635#[derive(Debug, Clone, Default)]
636pub(crate) struct RouteBindConcurrency {
637    modules: Arc<Mutex<HashMap<String, usize>>>,
638}
639
640struct RouteBindConcurrencyGuard {
641    concurrency: RouteBindConcurrency,
642    module_id: String,
643}
644
645impl RouteBindConcurrency {
646    /// Admit without waiting. Waiting here would move the bind stall from the
647    /// module reply to a semaphore and restore reader head-of-line blocking.
648    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
649        let mut modules = self
650            .modules
651            .lock()
652            .expect("route.bind concurrency mutex poisoned");
653        let in_flight = modules.entry(module_id.to_string()).or_default();
654        if *in_flight >= limit {
655            return Err(*in_flight);
656        }
657        *in_flight += 1;
658        Ok(RouteBindConcurrencyGuard {
659            concurrency: self.clone(),
660            module_id: module_id.to_string(),
661        })
662    }
663}
664
665impl Drop for RouteBindConcurrencyGuard {
666    fn drop(&mut self) {
667        let mut modules = self
668            .concurrency
669            .modules
670            .lock()
671            .expect("route.bind concurrency mutex poisoned");
672        let remove = {
673            let in_flight = modules
674                .get_mut(&self.module_id)
675                .expect("admitted route.bind has a concurrency entry");
676            *in_flight -= 1;
677            *in_flight == 0
678        };
679        if remove {
680            modules.remove(&self.module_id);
681        }
682    }
683}
684
685#[derive(Debug, Default)]
686struct ModuleBreakerState {
687    /// Relay timeouts observed with no accepted relay in between.
688    consecutive_timeouts: u32,
689    /// `Some` while the breaker is open: the instant the cooldown expires and
690    /// the next arrival may probe. `None` means closed.
691    cooldown_until: Option<Instant>,
692    /// A half-open probe has been admitted and has not settled yet. This is
693    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
694    /// that read the cooldown, so concurrent opens arriving at the moment the
695    /// cooldown expires cannot all decide that they are the probe.
696    probe_in_flight: Option<Arc<()>>,
697}
698
699/// What the breaker decided for one `route.open`, before any relay work.
700enum RouteBindAdmission<'a> {
701    Admitted {
702        guard: RouteBindBreakerGuard<'a>,
703        /// This open is the single half-open probe, so the transition is worth
704        /// one log line.
705        probe: bool,
706    },
707    Refused {
708        consecutive_timeouts: u32,
709        /// What is left of the cooldown. Zero when the refusal is because the
710        /// one probe is already in flight rather than because the cooldown has
711        /// not elapsed.
712        retry_in: Duration,
713        probe_in_flight: bool,
714    },
715}
716
717/// An outstanding admission, which must be told how its relay settled.
718///
719/// `Drop` settles it as inconclusive, so an early return between admission and
720/// the relay -- or the whole handler being cancelled when the client
721/// disconnects -- releases a half-open probe slot instead of leaving the
722/// breaker wedged half-open with no further probes.
723struct RouteBindBreakerGuard<'a> {
724    breakers: RouteBindBreakers,
725    module_id: &'a str,
726    probe_token: Option<Arc<()>>,
727    settled: bool,
728}
729
730impl RouteBindBreakerGuard<'_> {
731    /// The module answered within the budget and took the bind. THE ONLY
732    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
733    /// breaker, which is a transition worth logging.
734    fn record_accepted(&mut self) -> bool {
735        self.settled = true;
736        self.breakers.record_accepted(self.module_id)
737    }
738
739    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
740    /// COUNTS TOWARD OPENING.
741    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
742        self.settled = true;
743        self.breakers.record_timeout(
744            self.module_id,
745            self.probe_token.as_ref(),
746            threshold,
747            cooldown,
748        )
749    }
750
751    /// Everything else: the module REJECTED the bind, its connection went away
752    /// mid-relay, or the waiter was cancelled.
753    ///
754    /// None of these is evidence that a module is slow, and each already has
755    /// its own refusal with its own code. A module that rejects a bind in
756    /// microseconds is healthy and must never be convicted for it; a module
757    /// that died has said nothing about the module that replaces it. So these
758    /// neither increment nor reset the count -- they only release a probe slot.
759    fn record_inconclusive(&mut self) {
760        self.settled = true;
761        self.breakers
762            .record_inconclusive(self.module_id, self.probe_token.as_ref());
763    }
764}
765
766impl Drop for RouteBindBreakerGuard<'_> {
767    fn drop(&mut self) {
768        if !self.settled {
769            self.breakers
770                .record_inconclusive(self.module_id, self.probe_token.as_ref());
771        }
772    }
773}
774
775/// The breaker moved to open, reported so the caller can log it outside the
776/// lock. Opening is rare and load-bearing; the refusals that follow are
777/// frequent and are counted rather than logged.
778struct BreakerOpened {
779    consecutive_timeouts: u32,
780    /// True when a failed probe re-opened an already-open breaker, which reads
781    /// very differently in a log from a first opening.
782    reopened_after_probe: bool,
783}
784
785impl RouteBindBreakers {
786    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
787        self.modules
788            .lock()
789            .expect("route.bind breaker mutex poisoned")
790    }
791
792    /// Decide whether this `route.open` may attempt its relay. Takes the map
793    /// lock and nothing else, and never awaits.
794    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
795        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
796            probe: probe_token.is_some(),
797            guard: RouteBindBreakerGuard {
798                breakers: self.clone(),
799                module_id,
800                probe_token,
801                settled: false,
802            },
803        };
804
805        let mut modules = self.lock();
806        let Some(state) = modules.get_mut(module_id) else {
807            return admitted(None);
808        };
809        let Some(cooldown_until) = state.cooldown_until else {
810            return admitted(None);
811        };
812        if state.probe_in_flight.is_some() {
813            return RouteBindAdmission::Refused {
814                consecutive_timeouts: state.consecutive_timeouts,
815                retry_in: Duration::ZERO,
816                probe_in_flight: true,
817            };
818        }
819        let now = Instant::now();
820        if now < cooldown_until {
821            return RouteBindAdmission::Refused {
822                consecutive_timeouts: state.consecutive_timeouts,
823                retry_in: cooldown_until - now,
824                probe_in_flight: false,
825            };
826        }
827        let token = Arc::new(());
828        state.probe_in_flight = Some(Arc::clone(&token));
829        admitted(Some(token))
830    }
831
832    fn record_accepted(&self, module_id: &str) -> bool {
833        self.lock()
834            .remove(module_id)
835            .is_some_and(|state| state.cooldown_until.is_some())
836    }
837
838    fn record_timeout(
839        &self,
840        module_id: &str,
841        probe_token: Option<&Arc<()>>,
842        threshold: u32,
843        cooldown: Duration,
844    ) -> Option<BreakerOpened> {
845        let mut modules = self.lock();
846        let state = modules.entry(module_id.to_string()).or_default();
847        let was_open = state.cooldown_until.is_some();
848        let was_probe = Self::owns_probe(state, probe_token);
849        if was_probe {
850            state.probe_in_flight = None;
851        }
852        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
853        if state.consecutive_timeouts < threshold {
854            return None;
855        }
856        state.cooldown_until = Some(Instant::now() + cooldown);
857        Some(BreakerOpened {
858            consecutive_timeouts: state.consecutive_timeouts,
859            reopened_after_probe: was_open && was_probe,
860        })
861    }
862
863    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
864        // A relay can finish after the breaker was reset or after it opened
865        // again and started a new probe. Only the guard whose token matches the
866        // active probe may release it, so a late relay never frees a newer probe.
867        state
868            .probe_in_flight
869            .as_ref()
870            .zip(token)
871            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
872    }
873
874    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
875        if let Some(state) = self.lock().get_mut(module_id) {
876            if Self::owns_probe(state, probe_token) {
877                state.probe_in_flight = None;
878            }
879        }
880    }
881
882    /// Discard what was learned about a module, because the process it was
883    /// learned about is gone. Returns the discarded count when it was non-zero.
884    ///
885    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
886    /// `module_id` is a configuration identity that outlives any particular
887    /// child; what the breaker observed was the process behind the module
888    /// connection of the moment. When a new connection registers under that id
889    /// the verdict's subject no longer exists, so the verdict is stale by
890    /// construction rather than merely likely to be wrong. Keeping it would
891    /// apply a dead process's record to a live one, which is the same defect
892    /// class this breaker exists to stop the daemon committing.
893    ///
894    /// A half-open probe in flight is discarded with the rest: it was a
895    /// question about the old process.
896    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
897        self.lock()
898            .remove(module_id)
899            .map(|state| state.consecutive_timeouts)
900            .filter(|discarded| *discarded > 0)
901    }
902
903    /// Open breakers, for the `server.describe` counters object. `None` when
904    /// none is open, so the key stays absent rather than present-and-empty.
905    ///
906    /// This is the operator's answer to "is this module refusing instantly or
907    /// is it fine?", which look identical from a client that retries and then
908    /// succeeds.
909    fn open_snapshot(&self) -> Option<serde_json::Value> {
910        let now = Instant::now();
911        let modules = self.lock();
912        let open = modules
913            .iter()
914            .filter_map(|(module_id, state)| {
915                let cooldown_until = state.cooldown_until?;
916                Some((
917                    module_id.clone(),
918                    serde_json::json!({
919                        "consecutive_timeouts": state.consecutive_timeouts,
920                        "cooldown_remaining_ms":
921                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
922                        "probe_in_flight": state.probe_in_flight.is_some(),
923                    }),
924                ))
925            })
926            .collect::<serde_json::Map<String, serde_json::Value>>();
927        (!open.is_empty()).then_some(serde_json::Value::Object(open))
928    }
929}
930
931impl ControlHandler {
932    pub fn new(registry: Arc<Registry>) -> Self {
933        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
934    }
935
936    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
937        let counters = forwarding.counters();
938        // Taken from the forwarding table rather than created here, so that the
939        // breaker a `route.open` consults is the same one a module's
940        // registration resets, however many handlers are built over one table.
941        let route_bind_breakers = forwarding.route_bind_breakers();
942        let route_bind_concurrency = forwarding.route_bind_concurrency();
943        let route_outages = forwarding.route_outages();
944        Self {
945            registry,
946            activity: None,
947            activity_streams: Arc::new(Mutex::new(HashMap::new())),
948            forwarding,
949            process_liveness: None,
950            supervisor: SupervisorHandle::new(),
951            subc_capabilities: Arc::from([
952                CAP_MANIFEST_REGISTRATION.to_string(),
953                CAP_CHANNEL_LIFECYCLE.to_string(),
954                CAP_PING_PONG.to_string(),
955                CAP_SESSION_ATTACH.to_string(),
956                CAP_ADMISSION_FACTS_RELAY.to_string(),
957                CAP_SCOPES_V1.to_string(),
958                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
959                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
961                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
962                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
963                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
964                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
965                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
966                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
967                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
968                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
969            ]),
970            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
971            route_bind_relay_timeouts: BTreeMap::new(),
972            route_bind_breakers,
973            route_bind_concurrency,
974            route_outages,
975            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
976            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
977            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
978            storage_config: None,
979            machine_id: None,
980            admission_facts_carrier_module_id: None,
981            admission_facts_targets: None,
982            scopes: Arc::new(RwLock::new(ScopeTable::new(
983                crate::daemon_config::default_scope_authority_owners(),
984            ))),
985            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
986            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
987            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
988            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
989            #[cfg(test)]
990            caller_lookup: None,
991            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
992            rescan: None,
993            connected_clients: ConnectedClients::new(),
994            counters,
995            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
996            daemon_shutdown: DaemonShutdownFlag::default(),
997            daemon_provenance: DaemonProvenanceFacts::default(),
998            #[cfg(test)]
999            control_dispatch_delay: None,
1000            #[cfg(test)]
1001            provenance_probe_override: None,
1002            #[cfg(test)]
1003            route_bind_relay_sent_hold: None,
1004            #[cfg(test)]
1005            hello_registry_to_forwarding_hold: None,
1006        }
1007    }
1008
1009    /// Set the central storage policy: registering modules then receive their
1010    /// resolved storage descriptor in HELLO_ACK.
1011    pub fn with_storage_config(
1012        mut self,
1013        storage_config: Option<crate::daemon_config::StorageConfig>,
1014    ) -> Self {
1015        self.storage_config = storage_config;
1016        self
1017    }
1018
1019    /// Open the ledger on its own writer thread; an unavailable sink does not
1020    /// prevent the daemon from serving routing or supervisor requests.
1021    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1022        self.activity = Some(crate::activity::ActivityLedger::open(
1023            run_dir.join("activity.jsonl"),
1024            incarnation,
1025        ));
1026        self.supervisor.set_activity(self.activity.clone());
1027        self
1028    }
1029
1030    /// Set the machine id served to every registering module (HELLO_ACK) and on
1031    /// `server.describe`.
1032    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1033        self.machine_id = machine_id;
1034        self
1035    }
1036
1037    /// Configure the exact reserved module and target ids permitted to relay
1038    /// opaque admission facts. Config-file loading validates this authority;
1039    /// this builder keeps the same policy available to embedded test daemons.
1040    pub fn with_admission_facts_config(
1041        mut self,
1042        carrier_module_id: Option<String>,
1043        targets: Option<Vec<String>>,
1044    ) -> Self {
1045        self.admission_facts_carrier_module_id = carrier_module_id;
1046        self.admission_facts_targets = targets;
1047        self
1048    }
1049
1050    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1051    /// Replaces the scope table with an empty one under the new list, so call it
1052    /// while building the handler, before any module can sync.
1053    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1054        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1055        self.scope_authority_owners = owners;
1056        self
1057    }
1058
1059    /// Set the startup policy before installing the rescan context or serving requests.
1060    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1061        self.privileged_operator_prompt = mode;
1062        self
1063    }
1064
1065    /// Override the route.bind relay timeout. Used by tests that assert the
1066    /// timeout path so they don't block on the production-safe default.
1067    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1068        self.route_bind_relay_timeout = timeout;
1069        self
1070    }
1071
1072    /// Install per-module route.bind relay budget overrides. A module id
1073    /// listed here wins over the daemon-wide default set via
1074    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1075    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1076    /// the same `Duration` the bind path will use.
1077    pub fn with_route_bind_relay_timeouts(
1078        mut self,
1079        timeouts: impl IntoIterator<Item = (String, Duration)>,
1080    ) -> Self {
1081        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1082        self
1083    }
1084
1085    /// Resolve the route.bind relay budget for a specific target module id.
1086    /// Per-module overrides win; the daemon-wide value (set via
1087    /// `with_route_bind_relay_timeout` or the built-in default) is the
1088    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1089    /// the same resolution `handle_route_open` will use.
1090    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1091        self.route_bind_relay_timeouts
1092            .get(module_id)
1093            .copied()
1094            .unwrap_or(self.route_bind_relay_timeout)
1095    }
1096
1097    /// Override the per-module bind-relay breaker policy.
1098    ///
1099    /// Used by tests, which cannot spend three production budgets opening a
1100    /// breaker or twenty seconds waiting for its cooldown. The production
1101    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1102    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1103    /// reasoning for the numbers.
1104    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1105        self.route_bind_breaker_threshold = threshold.max(1);
1106        self.route_bind_breaker_cooldown = cooldown;
1107        self
1108    }
1109
1110    #[cfg(test)]
1111    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1112        self.health_probe_timeout = timeout;
1113        self
1114    }
1115
1116    #[cfg(test)]
1117    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1118        self.control_dispatch_delay = Some(delay);
1119        self
1120    }
1121
1122    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1123    /// announce it is parked, once to be released. Build the barrier for two
1124    /// parties, the test being the other one.
1125    #[cfg(test)]
1126    pub(crate) fn with_route_bind_relay_sent_hold(
1127        mut self,
1128        hold: Arc<tokio::sync::Barrier>,
1129    ) -> Self {
1130        self.route_bind_relay_sent_hold = Some(hold);
1131        self
1132    }
1133
1134    /// A HELLO waits on `hold` twice between reserving its registry entry and
1135    /// installing its forwarding endpoint: once to announce it is parked, once
1136    /// to be released. Build the barrier for two parties, the test being the
1137    /// other one, and run the HELLO on a thread of its own.
1138    #[cfg(test)]
1139    pub(crate) fn with_hello_registry_to_forwarding_hold(
1140        mut self,
1141        hold: Arc<std::sync::Barrier>,
1142    ) -> Self {
1143        self.hello_registry_to_forwarding_hold = Some(hold);
1144        self
1145    }
1146
1147    #[cfg(test)]
1148    fn hold_between_registry_and_forwarding(&self) {
1149        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1150            hold.wait();
1151            hold.wait();
1152        }
1153    }
1154
1155    pub fn with_process_liveness(
1156        mut self,
1157        process_liveness: Arc<dyn ModuleProcessLiveness>,
1158    ) -> Self {
1159        self.process_liveness = Some(process_liveness);
1160        self
1161    }
1162
1163    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1164        self.supervisor = supervisor;
1165        self.supervisor.set_activity(self.activity.clone());
1166        self
1167    }
1168
1169    pub fn with_daemon_provenance(
1170        mut self,
1171        pid: u32,
1172        started_at_ms: u64,
1173        executable_path: Option<PathBuf>,
1174        build_git_sha: Option<String>,
1175        build_lock_digest: Option<String>,
1176    ) -> Self {
1177        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1178        let process_start_time = process_start_time(pid);
1179        self.daemon_provenance = DaemonProvenanceFacts {
1180            build: DaemonBuildProvenance {
1181                build_git_sha,
1182                build_lock_digest,
1183            },
1184            pid: Some(pid),
1185            started_at_ms: Some(started_at_ms),
1186            start_clock: None,
1187            executable_path,
1188            executable_identity,
1189            process_start_time,
1190            probe: ExecutableIdentityProbe::default(),
1191        };
1192        self
1193    }
1194
1195    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1196        self.daemon_provenance.start_clock = Some(clock);
1197        self
1198    }
1199
1200    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1201    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1202        self.scopes
1203            .write()
1204            .unwrap_or_else(|poisoned| poisoned.into_inner())
1205            .set_wall_clock(Arc::new(clock));
1206        self
1207    }
1208
1209    #[cfg(test)]
1210    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1211        self.provenance_probe_override = Some(result);
1212        self
1213    }
1214
1215    /// Install the configured module set and its reserved capability bindings.
1216    /// Bindings are configuration-scoped and may point at a provider that has not
1217    /// been installed yet, so this does not require the bound module to exist.
1218    pub fn with_capability_config(
1219        self,
1220        modules: impl IntoIterator<Item = (String, bool)>,
1221        reserved_capabilities: BTreeMap<String, String>,
1222    ) -> Self {
1223        self.capability_evaluator
1224            .configure(modules, reserved_capabilities);
1225        self
1226    }
1227
1228    /// Share the supervisor's daemon-shutdown flag with this handler, so
1229    /// requirement changes after the daemon has begun its announced shutdown
1230    /// are not reported as faults.
1231    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1232        self.daemon_shutdown = flag;
1233        self
1234    }
1235
1236    pub fn with_supervisor_rescan(
1237        mut self,
1238        supervisor: Supervisor,
1239        config_path: impl Into<PathBuf>,
1240        configured_port: Option<u16>,
1241    ) -> Self {
1242        self.rescan = Some(SupervisorRescanContext {
1243            supervisor,
1244            config_path: config_path.into(),
1245            configured_port,
1246            storage_config: self.storage_config.clone(),
1247            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1248            admission_facts_targets: self.admission_facts_targets.clone(),
1249            scope_authority_owners: self.scope_authority_owners.clone(),
1250            privileged_operator_prompt: self.privileged_operator_prompt,
1251            generation: Arc::default(),
1252        });
1253        self
1254    }
1255
1256    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1257        self.connected_clients = connected_clients;
1258        self
1259    }
1260
1261    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1262        Arc::clone(&self.forwarding)
1263    }
1264
1265    pub(crate) fn counters(&self) -> DaemonCounters {
1266        self.counters.clone()
1267    }
1268
1269    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1270    /// requirement event without depending on an operator polling a status command.
1271    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1272        tokio::spawn(async move {
1273            loop {
1274                self.capability_evaluator
1275                    .wait_for_change_or_deadline()
1276                    .await;
1277                self.refresh_capability_requirements();
1278            }
1279        });
1280    }
1281
1282    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1283    /// deadline passed during sleep is enforced on the first tick after wake.
1284    /// While no live scope carries a deadline the loop waits for an accepted
1285    /// scope change instead of ticking, so a daemon without deadlines never
1286    /// wakes for expiry.
1287    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1288        tokio::spawn(async move {
1289            loop {
1290                let has_deadlines = self
1291                    .scopes
1292                    .read()
1293                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1294                    .has_deadlines();
1295                if !has_deadlines {
1296                    self.scope_deadline_added.notified().await;
1297                    continue;
1298                }
1299                tokio::time::sleep(Duration::from_secs(1)).await;
1300                if let Err(error) = self.sweep_expired_scopes() {
1301                    warn!(%error, "scope expiry sweep failed");
1302                }
1303            }
1304        });
1305    }
1306
1307    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1308        let mut table = self
1309            .scopes
1310            .write()
1311            .unwrap_or_else(|poisoned| poisoned.into_inner());
1312        let swept = table.sweep_expired();
1313        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1314        drop(table);
1315        self.close_scope_drained_routes(drained);
1316        Ok(())
1317    }
1318
1319    fn publish_scope_changes(
1320        &self,
1321        changes: &[crate::scopes::ScopeTagChange],
1322        expired: &[crate::scopes::ScopeExpired],
1323    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1324        if changes.is_empty() {
1325            return Ok(Vec::new());
1326        }
1327        let drained = self
1328            .forwarding
1329            .publish_scope_changes(changes)
1330            .map_err(RouterError::Forwarding)?;
1331        let mut counts = HashMap::new();
1332        for route in &drained {
1333            *counts
1334                .entry((
1335                    route.scope.owner.as_str(),
1336                    route.scope.scope_ref.as_str(),
1337                    route.scope.tag.scope_epoch,
1338                ))
1339                .or_insert(0usize) += 1;
1340        }
1341        for scope in expired {
1342            let routes_closed = counts
1343                .get(&(
1344                    scope.owner.as_str(),
1345                    scope.scope_ref.as_str(),
1346                    scope.scope_epoch,
1347                ))
1348                .copied()
1349                .unwrap_or(0);
1350            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1351                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1352                routes_closed, "scope expired");
1353        }
1354        Ok(drained)
1355    }
1356
1357    fn runtime_capability_snapshot(
1358        &self,
1359    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1360        let runtime = self
1361            .supervisor
1362            .list()
1363            .into_iter()
1364            .map(|module| {
1365                let status = module.status().map_err(|err| {
1366                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1367                })?;
1368                Ok(RuntimeModule {
1369                    module_id: status.module_id,
1370                    state: status.state,
1371                    enabled: status.enabled,
1372                })
1373            })
1374            .collect::<Result<Vec<_>, RouterError>>()?;
1375        // Include every registration, published or not. The capability
1376        // evaluator counts a module as providing its capabilities as soon as it
1377        // has registered, even before `catalog.list` shows it, so a HELLO in
1378        // progress does not change other modules' required-capability
1379        // verdicts. See `Registry::list_registered_modules`.
1380        let registrations = self.registry.list_registered_modules().map_err(|err| {
1381            RouterError::backend(
1382                0,
1383                0,
1384                format!("failed to list capability registrations: {err}"),
1385            )
1386        })?;
1387        let registrations = registrations
1388            .into_iter()
1389            .map(|registration| RegisteredModule {
1390                module_id: registration.manifest.module_id,
1391                module_version: registration.manifest.module_version,
1392                capabilities: registration.manifest.capabilities,
1393            })
1394            .collect();
1395        Ok((runtime, registrations))
1396    }
1397
1398    /// The capability side effects of a module becoming the active registration
1399    /// for its id: cache its manifest (warning if its claims drifted), run the
1400    /// deny census when its declarations call for one, and recompute the
1401    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1402    /// candidate's does not, and the supervisor does it at promotion instead,
1403    /// through [`crate::supervise::SwapPromotionObserver`].
1404    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1405        let cached_registration = RegisteredModule {
1406            module_id: registration.manifest.module_id.clone(),
1407            module_version: registration.manifest.module_version.clone(),
1408            capabilities: registration.manifest.capabilities.clone(),
1409        };
1410        if self.capability_evaluator.record_hello(&cached_registration) {
1411            warn!(
1412                module_id = %cached_registration.module_id,
1413                "capability claims drifted from the cached manifest"
1414            );
1415        }
1416        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1417            self.enforce_capability_denies();
1418        }
1419        self.refresh_capability_requirements();
1420    }
1421
1422    /// Point the shared supervisor handle at this handler for swap promotions.
1423    /// Called wherever a handler is put behind the `Arc` the router serves, so
1424    /// it can be held weakly.
1425    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1426        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1427            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1428        self.supervisor.set_swap_promotion_observer(observer);
1429    }
1430
1431    pub fn refresh_capability_requirements(&self) {
1432        match self.runtime_capability_snapshot() {
1433            Ok((runtime, registrations)) => {
1434                log_requirement_events(
1435                    self.capability_evaluator
1436                        .evaluate_now(&runtime, &registrations),
1437                    &self.daemon_shutdown,
1438                );
1439            }
1440            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1441        }
1442    }
1443
1444    /// Reconcile only live, attested route bindings after a capability deny edge
1445    /// or target claim was added. This is deliberately a control-plane census:
1446    /// the opaque forwarding hot path must not grow a per-frame capability check.
1447    fn enforce_capability_denies(&self) {
1448        let registrations = match self.registry.list_registered_modules() {
1449            Ok(snapshot) => snapshot,
1450            Err(err) => {
1451                warn!(error = %err, "failed to read registrations for capability deny census");
1452                return;
1453            }
1454        };
1455        let manifests = registrations
1456            .into_iter()
1457            .map(|registration| {
1458                (
1459                    registration.manifest.module_id.clone(),
1460                    registration.manifest,
1461                )
1462            })
1463            .collect::<BTreeMap<_, _>>();
1464        let census = match self.forwarding.route_census(None) {
1465            Ok(census) => census,
1466            Err(err) => {
1467                warn!(error = %err, "failed to read route census for capability deny enforcement");
1468                return;
1469            }
1470        };
1471
1472        for (target_module_id, routes) in census {
1473            let Some(target_manifest) = manifests.get(&target_module_id) else {
1474                continue;
1475            };
1476            let mut closed_routes = Vec::new();
1477            let mut goodbyes = Vec::new();
1478            for route in routes {
1479                let Principal::Reserved {
1480                    module_id: opening_module_id,
1481                } = &route.principal
1482                else {
1483                    continue;
1484                };
1485                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1486                    continue;
1487                };
1488                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1489                    continue;
1490                };
1491
1492                match self.forwarding.release_client_route(
1493                    route.goodbye_target.connection_id,
1494                    route.goodbye_target.channel,
1495                    route.goodbye_target.epoch,
1496                ) {
1497                    Ok(RouteRelease::Removed(module_goodbye)) => {
1498                        warn!(
1499                            opening_module_id,
1500                            target_module_id,
1501                            capability,
1502                            "force-closing route because an attested capability deny edge now matches"
1503                        );
1504                        goodbyes.push(module_goodbye);
1505                        goodbyes.push(route.goodbye_target.clone());
1506                        closed_routes.push(route);
1507                    }
1508                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1509                    Err(err) => warn!(
1510                        opening_module_id,
1511                        target_module_id,
1512                        capability,
1513                        error = %err,
1514                        "failed to force-close capability-denied route"
1515                    ),
1516                }
1517            }
1518
1519            if closed_routes.is_empty() {
1520                continue;
1521            }
1522            send_route_control_pushes(
1523                &self.forwarding,
1524                closed_routes,
1525                ClientControlPush::RouteClosed {
1526                    module_id: target_module_id,
1527                    channels: Vec::new(),
1528                    reason: RouteCloseReason::CapabilityDenied,
1529                    drained: false,
1530                    abandoned: 0,
1531                    excluded_subscriptions: 0,
1532                    terminal: Some(false),
1533                },
1534            );
1535            // route.closed records the reason; the route GOODBYE ends the
1536            // client's handle and settles any calls still pending on it.
1537            self.emit_route_goodbyes(goodbyes);
1538        }
1539    }
1540
1541    /// Why a registered module is not accepting new route binds, or `None` when
1542    /// it is. This is the module's effective readiness: its declared readiness
1543    /// first, then every `need: required` capability it declares evaluating to
1544    /// `provided`. `route.open` and `catalog.list` both read it here so the
1545    /// catalog never reports a module routable that `route.open` would refuse.
1546    fn not_ready_reason(
1547        &self,
1548        registration: &crate::registry::ModuleRegistration,
1549    ) -> Option<NotReadyReason> {
1550        if !registration.ready {
1551            return Some(NotReadyReason {
1552                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1553                capability: None,
1554            });
1555        }
1556        self.first_unprovided_required_capability(registration)
1557            .map(|capability| NotReadyReason {
1558                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1559                capability: Some(capability),
1560            })
1561    }
1562
1563    /// The lexicographically first capability this registration declares
1564    /// `need: required` whose evaluator verdict is not `provided`.
1565    ///
1566    /// The verdicts are the capability evaluator's own; nothing here decides
1567    /// what "provided" means. The evaluator counts a capability provided as
1568    /// soon as a module claiming it has REGISTERED, not once that module is
1569    /// ready. That distinction is what keeps two modules that require each
1570    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1571    /// is ready", each would wait for the other to become ready first and
1572    /// neither ever would. Do not tighten it to readiness.
1573    ///
1574    /// A required capability with no verdict at all means this registration's
1575    /// HELLO or catalog.update landed after the last recompute; recompute once
1576    /// rather than let a missing verdict read as either answer. If it is still
1577    /// missing (the recompute itself failed) the capability counts as
1578    /// unprovided: the refusal is retryable, and routing a module whose
1579    /// required provider is unknown is the outcome this check exists to stop.
1580    fn first_unprovided_required_capability(
1581        &self,
1582        registration: &crate::registry::ModuleRegistration,
1583    ) -> Option<String> {
1584        let required = registration
1585            .manifest
1586            .capabilities
1587            .iter()
1588            .flat_map(|declarations| declarations.requires.iter())
1589            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1590            .map(|requirement| requirement.capability.as_str())
1591            .collect::<BTreeSet<_>>();
1592        if required.is_empty() {
1593            return None;
1594        }
1595        let module_id = registration.manifest.module_id.as_str();
1596        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1597        if required
1598            .iter()
1599            .any(|capability| verdict(capability).is_none())
1600        {
1601            self.refresh_capability_requirements();
1602        }
1603        required
1604            .into_iter()
1605            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1606            .map(str::to_string)
1607    }
1608
1609    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1610        self.capability_evaluator
1611            .statuses()
1612            .into_iter()
1613            .map(capability_requirement_status)
1614            .collect()
1615    }
1616
1617    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1618    /// registration-release watch. The signal is what the supervisor waits on
1619    /// before spawning a replacement, so it must only fire once forwarding
1620    /// teardown is also done (see [`Self::cleanup_connection`] /
1621    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1622    /// admitted but its forwarding endpoint could not be installed.
1623    fn deregister_connection(
1624        &self,
1625        connection_id: ConnectionId,
1626        reason: RegistrationEndReason,
1627    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1628        self.registry
1629            .deregister_connection_with_reason(connection_id, reason)
1630    }
1631
1632    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1633        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1634            return None;
1635        }
1636        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1637            parse_client_control_request(&frame.body)
1638        else {
1639            return None;
1640        };
1641        Some(target_module_id(&target).to_string())
1642    }
1643
1644    pub(crate) fn route_open_capacity_refusal(
1645        &self,
1646        ctx: &RouteCtx,
1647        frame: &Frame,
1648        target_module_id: &str,
1649        in_flight: usize,
1650        limit: usize,
1651    ) -> Result<Frame, RouterError> {
1652        // Sent before the request reaches `handle_route_open`, so the caller
1653        // fields come from the raw frame here.
1654        let caller = self.route_open_caller_from_frame(frame);
1655        self.route_open_admission_refusal_frame(
1656            ctx,
1657            &caller,
1658            frame,
1659            target_module_id,
1660            "open_admission_full",
1661            (in_flight, limit),
1662            format!(
1663                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1664            ),
1665        )
1666    }
1667
1668    fn route_open_target_capacity_refusal(
1669        &self,
1670        ctx: &RouteCtx,
1671        caller: &RouteOpenCaller,
1672        frame: &Frame,
1673        target_module_id: &str,
1674        in_flight: usize,
1675    ) -> Result<Frame, RouterError> {
1676        self.route_open_admission_refusal_frame(
1677            ctx,
1678            caller,
1679            frame,
1680            target_module_id,
1681            "target_binds_full",
1682            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1683            format!(
1684                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1685            ),
1686        )
1687    }
1688
1689    /// Admission pressure clears as existing binds settle, so its refusal must
1690    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1691    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1692    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1693    /// cannot currently reach its target; `module_timeout` would falsely claim
1694    /// that a wait expired. A new, cleaner code would be terminal to deployed
1695    /// clients, so it requires a client-tolerance rollout before daemon emission.
1696    #[allow(clippy::too_many_arguments)]
1697    fn route_open_admission_refusal_frame(
1698        &self,
1699        ctx: &RouteCtx,
1700        caller: &RouteOpenCaller,
1701        frame: &Frame,
1702        target_module_id: &str,
1703        reason: &'static str,
1704        (in_flight, limit): (usize, usize),
1705        message: impl Into<String>,
1706    ) -> Result<Frame, RouterError> {
1707        let code = error_codes::TARGET_UNAVAILABLE;
1708        self.counters.increment_route_open_refused(code);
1709        info!(
1710            target: "control",
1711            code,
1712            reason,
1713            module_id = ?target_module_id,
1714            connection_id = ctx.connection_id.get(),
1715            principal = %caller.principal,
1716            scope = %caller.scope,
1717            harness = %caller.harness,
1718            in_flight,
1719            limit,
1720            "route.open refused"
1721        );
1722        control_error_frame(frame, code, message.into())
1723    }
1724
1725    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1726    ///
1727    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1728    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1729    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1730    #[cfg(test)]
1731    pub fn handle_control(
1732        &self,
1733        connection_id: ConnectionId,
1734        frame: Frame,
1735    ) -> Result<Vec<Frame>, RouterError> {
1736        match frame.header.ty {
1737            FrameType::Ping => Ok(vec![pong(&frame)?]),
1738            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1739            FrameType::Goodbye => self.handle_goodbye(connection_id),
1740            ty => Ok(vec![control_error_frame(
1741                &frame,
1742                "unsupported_control_frame",
1743                format!("unsupported channel-0 frame {ty:?}"),
1744            )?]),
1745        }
1746    }
1747
1748    pub async fn handle_control_frame(
1749        &self,
1750        ctx: &RouteCtx,
1751        frame: Frame,
1752    ) -> Result<Vec<Frame>, RouterError> {
1753        self.handle_control_frame_timed(ctx, frame, None).await
1754    }
1755
1756    pub(crate) fn record_caller_endpoints(
1757        &self,
1758        connection: ConnectionId,
1759        endpoints: Option<(SocketAddr, SocketAddr)>,
1760    ) {
1761        let mut callers = self
1762            .caller_endpoints
1763            .lock()
1764            .unwrap_or_else(|p| p.into_inner());
1765        callers.remove(&connection);
1766        if let Some(endpoints) = endpoints {
1767            callers.insert(connection, endpoints);
1768        }
1769    }
1770
1771    #[cfg(test)]
1772    pub(crate) fn with_caller_lookup_for_test(
1773        mut self,
1774        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1775    ) -> Self {
1776        self.caller_lookup = Some(Arc::new(lookup));
1777        self
1778    }
1779
1780    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1781        let endpoints = self
1782            .caller_endpoints
1783            .lock()
1784            .unwrap_or_else(|p| p.into_inner())
1785            .get(&connection)
1786            .copied();
1787        #[cfg(test)]
1788        let lookup = self.caller_lookup.clone();
1789        tokio::task::spawn_blocking(move || {
1790            #[cfg(test)]
1791            if let Some(lookup) = lookup {
1792                return lookup(endpoints);
1793            }
1794            endpoints
1795                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1796                .unwrap_or_default()
1797        })
1798        .await
1799        .unwrap_or_default()
1800    }
1801
1802    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1803        frame.header.channel == 0
1804            && frame.header.ty == FrameType::Request
1805            && self
1806                .forwarding
1807                .module_endpoint_for_connection(ctx.connection_id)
1808                .is_ok_and(|endpoint| endpoint.is_none())
1809            && parse_client_control_request(&frame.body)
1810                .ok()
1811                .is_some_and(|request| privileged_operation(&request).is_some())
1812    }
1813
1814    pub(crate) async fn handle_control_frame_timed(
1815        &self,
1816        ctx: &RouteCtx,
1817        frame: Frame,
1818        dispatch_started_at: Option<StdInstant>,
1819    ) -> Result<Vec<Frame>, RouterError> {
1820        match frame.header.ty {
1821            FrameType::Ping => Ok(vec![pong(&frame)?]),
1822            FrameType::Hello => {
1823                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1824            }
1825            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1826            FrameType::Cancel => {
1827                // A Cancel on channel 0 names a waiting operator.confirm or a
1828                // subscription. Successful cancellation has no ordinary reply.
1829                if self
1830                    .forwarding
1831                    .operator_confirms()
1832                    .cancel(ctx.connection_id, frame.header.corr)
1833                    || self
1834                        .supervisor
1835                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1836                    || self.cancel_activity_subscription(ctx, &frame)?
1837                {
1838                    Ok(Vec::new())
1839                } else {
1840                    Ok(vec![control_error_frame(
1841                        &frame,
1842                        "unknown_subscription",
1843                        "no subscription has this correlation id",
1844                    )?])
1845                }
1846            }
1847            FrameType::Request => {
1848                // This additive operation is not part of the existing exhaustive
1849                // module-control enum. Probe the op before decoding that enum.
1850                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1851                if op
1852                    .as_ref()
1853                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1854                {
1855                    return self.handle_activity_record(ctx, frame).await;
1856                }
1857                if op.as_ref().is_some_and(|probe| {
1858                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1859                }) {
1860                    return self.handle_activity_read(ctx, frame).await;
1861                }
1862                if op
1863                    .as_ref()
1864                    .is_some_and(|probe| probe.op == "operator.confirm")
1865                {
1866                    return self.handle_operator_confirm(ctx, frame);
1867                }
1868                if op
1869                    .as_ref()
1870                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1871                {
1872                    return self.handle_scope_served_by(ctx.connection_id, frame);
1873                }
1874                if self
1875                    .forwarding
1876                    .module_endpoint_for_connection(ctx.connection_id)
1877                    .map_err(RouterError::Forwarding)?
1878                    .is_some()
1879                {
1880                    if !is_known_module_request_op(&frame.body) {
1881                        return Ok(vec![control_error_frame(
1882                            &frame,
1883                            "unsupported_control_frame",
1884                            "module-originated channel-0 REQUEST is not supported",
1885                        )?]);
1886                    }
1887                    let request = match parse_module_control_request_from_module(&frame.body) {
1888                        Ok(request) => request,
1889                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1890                            return Ok(vec![control_error_frame(
1891                                &frame,
1892                                "unsupported_control_frame",
1893                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1894                            )?])
1895                        }
1896                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1897                            return Ok(vec![control_error_frame(
1898                                &frame,
1899                                "invalid_control_body",
1900                                format!("malformed module control body: {err}"),
1901                            )?])
1902                        }
1903                    };
1904                    let op = module_control_request_op(&request);
1905                    let corr = frame.header.corr;
1906                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1907                    let result =
1908                        self.handle_module_control_request(ctx.connection_id, frame, request);
1909                    log_slow_control_dispatch(
1910                        dispatch_started_at,
1911                        op,
1912                        ctx.connection_id,
1913                        corr,
1914                        None,
1915                    );
1916                    return result;
1917                }
1918
1919                if is_known_module_request_op(&frame.body) {
1920                    return Ok(vec![control_error_frame(
1921                        &frame,
1922                        "not_registered",
1923                        "catalog.update requires an active module registration owned by this connection",
1924                    )?]);
1925                }
1926
1927                let request = match parse_client_control_request(&frame.body) {
1928                    Ok(request) => request,
1929                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1930                        return Ok(vec![control_error_frame(
1931                            &frame,
1932                            "unknown_control_op",
1933                            format!("unknown client control op: {err}"),
1934                        )?])
1935                    }
1936                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1937                        return Ok(vec![control_error_frame(
1938                            &frame,
1939                            "invalid_control_body",
1940                            format!("malformed client control body: {err}"),
1941                        )?])
1942                    }
1943                };
1944                let op = client_control_request_op(&request);
1945                let corr = frame.header.corr;
1946                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1947                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1948                #[cfg(test)]
1949                if let Some(delay) = self.control_dispatch_delay {
1950                    tokio::time::sleep(delay).await;
1951                }
1952                let result = self
1953                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1954                    .await;
1955                if let Some(timings) = route_open_timings.as_mut() {
1956                    timings.finish();
1957                }
1958                log_slow_control_dispatch(
1959                    dispatch_started_at,
1960                    op,
1961                    ctx.connection_id,
1962                    corr,
1963                    route_open_timings.as_ref(),
1964                );
1965                result
1966            }
1967            FrameType::Push => {
1968                let Some(endpoint) = self
1969                    .forwarding
1970                    .module_endpoint_for_connection(ctx.connection_id)
1971                    .map_err(RouterError::Forwarding)?
1972                else {
1973                    return Ok(vec![control_error_frame(
1974                        &frame,
1975                        "unsupported_control_frame",
1976                        "client-originated channel-0 PUSH is not supported",
1977                    )?]);
1978                };
1979                self.handle_status_update(endpoint, frame)
1980            }
1981            FrameType::Response | FrameType::Error
1982                if self
1983                    .forwarding
1984                    .module_endpoint_for_connection(ctx.connection_id)
1985                    .map_err(RouterError::Forwarding)?
1986                    .is_some() =>
1987            {
1988                self.handle_module_relay_response(ctx.connection_id, frame)
1989            }
1990            ty => Ok(vec![control_error_frame(
1991                &frame,
1992                "unsupported_control_frame",
1993                format!("unsupported channel-0 frame {ty:?}"),
1994            )?]),
1995        }
1996    }
1997
1998    pub fn cleanup_connection(
1999        &self,
2000        connection_id: ConnectionId,
2001    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2002        self.caller_endpoints
2003            .lock()
2004            .unwrap_or_else(|p| p.into_inner())
2005            .remove(&connection_id);
2006        self.forwarding
2007            .operator_confirms()
2008            .connection_closed(connection_id);
2009        self.cleanup_connection_with_end_reason(
2010            connection_id,
2011            RegistrationEndReason::ConnectionClosed,
2012        )
2013    }
2014
2015    fn cleanup_connection_with_end_reason(
2016        &self,
2017        connection_id: ConnectionId,
2018        requested_reason: RegistrationEndReason,
2019    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2020        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2021            self.registry
2022                .get_module_by_connection(connection_id)?
2023                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2024                .and_then(|module| module.registration_end_reason().ok().flatten())
2025                .unwrap_or(requested_reason)
2026        } else {
2027            requested_reason
2028        };
2029        let crash_closed = self
2030            .registry
2031            .get_module_by_connection(connection_id)?
2032            .and_then(|registration| {
2033                self.forwarding
2034                    .module_endpoint_for_connection(connection_id)
2035                    .ok()
2036                    .flatten()
2037                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2038                    .map(|routes| (registration.manifest.module_id, routes))
2039            });
2040        let crash_closed = crash_closed.map(|(module_id, routes)| {
2041            let terminal = match self.supervisor.get(&module_id) {
2042                None => false,
2043                Some(module) => match module.will_recover_after_connection_loss() {
2044                    Ok(will_recover) => !will_recover,
2045                    Err(err) => {
2046                        warn!(
2047                            %module_id,
2048                            error = %err,
2049                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2050                        );
2051                        false
2052                    }
2053                },
2054            };
2055            // The forwarding table gates all providers at the start of daemon
2056            // shutdown, before their connections are closed. An ordinary
2057            // module disconnect still reports crash if that gate is not set.
2058            let (reason, drained) = match self
2059                .forwarding
2060                .daemon_drain_result_for_connection(connection_id)
2061            {
2062                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2063                Ok(None) => (RouteCloseReason::Crash, false),
2064                Err(err) => {
2065                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2066                    (RouteCloseReason::Crash, false)
2067                }
2068            };
2069            (module_id, routes, reason, drained, terminal)
2070        });
2071        let registrations = self.deregister_connection(connection_id, end_reason);
2072        let cleanup = if crash_closed.is_some() {
2073            self.forwarding.cleanup_connection_counted(connection_id)
2074        } else {
2075            // A module connection's teardown needs the count of abandoned
2076            // route.bind relays for its route.closed notice below. Any other
2077            // connection, such as a client's, sends no such notice and needs
2078            // only its routes released, so it uses the route-only wrapper and
2079            // reports zero.
2080            self.forwarding
2081                .cleanup_connection(connection_id)
2082                .map(|released| crate::forwarding::ConnectionCleanup {
2083                    released,
2084                    abandoned_relays: 0,
2085                })
2086        };
2087        // The route.closed push waits for forwarding teardown because only
2088        // teardown knows how many pending route.bind relays it aborted. It still
2089        // goes out before the GOODBYEs for the released routes, and its targets
2090        // were captured above, before teardown removed those routes.
2091        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2092            let abandoned = cleanup
2093                .as_ref()
2094                .map_or(0, |cleanup| cleanup.abandoned_relays);
2095            send_route_control_pushes(
2096                &self.forwarding,
2097                routes,
2098                ClientControlPush::RouteClosed {
2099                    module_id,
2100                    channels: Vec::new(),
2101                    reason,
2102                    drained,
2103                    abandoned,
2104                    excluded_subscriptions: 0,
2105                    terminal: Some(terminal),
2106                },
2107            );
2108        }
2109        if let Ok(cleanup) = cleanup {
2110            self.emit_route_goodbyes(cleanup.released);
2111        }
2112        // Signal the registration-release watch only now that BOTH registry and
2113        // forwarding teardown are done, so a supervisor waiting to spawn a
2114        // replacement never observes release while old routes still exist.
2115        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2116            crate::supervise::notify_registration_release();
2117            self.capability_evaluator.wake_deadline_loop();
2118            self.refresh_capability_requirements();
2119        }
2120        self.supervisor.remove_spawn_subscribers(connection_id);
2121        self.activity_streams
2122            .lock()
2123            .unwrap_or_else(|p| p.into_inner())
2124            .retain(|(id, _), task| {
2125                if *id == connection_id {
2126                    task.abort();
2127                    false
2128                } else {
2129                    true
2130                }
2131            });
2132        // Sync authority dies with its connection, so the owner's next
2133        // connection can take it; the owner's scopes stay as they are.
2134        self.hello_launch_nonces
2135            .lock()
2136            .unwrap_or_else(|poisoned| poisoned.into_inner())
2137            .forget(connection_id);
2138        self.scopes
2139            .write()
2140            .unwrap_or_else(|poisoned| poisoned.into_inner())
2141            .release_connection(connection_id);
2142        registrations
2143    }
2144
2145    pub(crate) fn handle_route_goodbye(
2146        &self,
2147        connection_id: ConnectionId,
2148        route_channel: u16,
2149        route_epoch: u32,
2150    ) -> Result<bool, RouterError> {
2151        debug!(
2152            connection_id = connection_id.get(),
2153            route_channel, route_epoch, "handling route GOODBYE"
2154        );
2155        let RouteRelease::Removed(released_route) = self
2156            .forwarding
2157            .release_client_route(connection_id, route_channel, route_epoch)
2158            .map_err(RouterError::Forwarding)?
2159        else {
2160            return Ok(false);
2161        };
2162        self.emit_route_goodbyes(vec![released_route]);
2163        Ok(true)
2164    }
2165
2166    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2167        for released in released_routes {
2168            let frame = match Frame::build_with_version(
2169                released.negotiated_ver,
2170                FrameType::Goodbye,
2171                control_flags(),
2172                released.channel,
2173                released.epoch,
2174                0,
2175                Vec::new(),
2176            ) {
2177                Ok(frame) => frame,
2178                Err(err) => {
2179                    warn!(
2180                        route_channel = released.channel,
2181                        error = %err,
2182                        "failed to build route GOODBYE frame"
2183                    );
2184                    continue;
2185                }
2186            };
2187            if !released.close_on_delivery_failure() {
2188                crate::forwarding::send_module_route_goodbye(
2189                    &self.counters,
2190                    &released.sink,
2191                    frame,
2192                    released.module_id.as_deref(),
2193                    "client route released",
2194                );
2195                continue;
2196            }
2197            if let Err(err) = released.sink.try_send(frame) {
2198                warn!(
2199                    target_connection_id = released.connection_id.get(),
2200                    route_channel = released.channel,
2201                    error = %err,
2202                    "route GOODBYE was not delivered to client; closing target connection"
2203                );
2204                if self
2205                    .forwarding
2206                    .escalate_client_delivery_failure(
2207                        released.connection_id,
2208                        released.channel,
2209                        released.epoch,
2210                        CloseReason::new(
2211                            "route_goodbye_delivery_failed",
2212                            format!(
2213                                "failed to enqueue route GOODBYE for channel {}: {err}",
2214                                released.channel
2215                            ),
2216                        ),
2217                        crate::forwarding::UndeliveredFrame {
2218                            module_id: released.module_id.as_deref(),
2219                            sink: &released.sink,
2220                        },
2221                    )
2222                    .unwrap_or(false)
2223                {
2224                    self.counters.increment_goodbye_relay_client_failed();
2225                }
2226            }
2227        }
2228    }
2229
2230    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2231    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2232    /// own commit failed after the module had already accepted). Without this, a
2233    /// module that accepts late keeps a binding subc has torn down, so a later
2234    /// frame on that module channel could misdeliver if the channel is reused.
2235    ///
2236    /// Never closes the shared module connection on failure: a dropped notification
2237    /// only wastes a bounded amount of warm module-side state, which the module's
2238    /// own idle reaper reclaims. Only call this once the route.bind relay was
2239    /// actually enqueued to the module — if the relay send itself failed, the
2240    /// module never created a binding and there is nothing to tear down.
2241    fn send_abandoned_route_bind_goodbye(
2242        &self,
2243        module_sink: &crate::FrameSink,
2244        negotiated_ver: u8,
2245        module_channel: u16,
2246        module_epoch: u32,
2247    ) {
2248        let frame = match Frame::build_with_version(
2249            negotiated_ver,
2250            FrameType::Goodbye,
2251            control_flags(),
2252            module_channel,
2253            module_epoch,
2254            0,
2255            Vec::new(),
2256        ) {
2257            Ok(frame) => frame,
2258            Err(err) => {
2259                warn!(
2260                    route_channel = module_channel,
2261                    error = %err,
2262                    "failed to build GOODBYE for abandoned route.bind"
2263                );
2264                return;
2265            }
2266        };
2267        crate::forwarding::send_module_route_goodbye(
2268            &self.counters,
2269            module_sink,
2270            frame,
2271            None,
2272            "abandoned route.bind",
2273        );
2274    }
2275
2276    fn handle_hello(
2277        &self,
2278        connection_id: ConnectionId,
2279        sink: Option<crate::FrameSink>,
2280        frame: Frame,
2281    ) -> Result<Vec<Frame>, RouterError> {
2282        debug!(
2283            connection_id = connection_id.get(),
2284            corr = frame.header.corr,
2285            "handling HELLO"
2286        );
2287        // A module connection has one identity for its entire lifetime. A second
2288        // registration would leave the old registry owner behind while replacing
2289        // its forwarding endpoint and launch nonce.
2290        if self
2291            .registry
2292            .get_module_by_connection(connection_id)
2293            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2294            .is_some()
2295        {
2296            return Ok(vec![control_error_frame(
2297                &frame,
2298                "invalid_hello",
2299                "connection is already registered as a module",
2300            )?]);
2301        }
2302        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2303            Ok(value) => value,
2304            Err(err) => {
2305                return Ok(vec![control_error_frame(
2306                    &frame,
2307                    "invalid_hello",
2308                    format!("malformed HELLO body: {err}"),
2309                )?])
2310            }
2311        };
2312        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2313            return Ok(vec![control_error_frame(
2314                &frame,
2315                "invalid_capability_grammar",
2316                err.to_string(),
2317            )?]);
2318        }
2319        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2320            return Ok(vec![control_error_frame(
2321                &frame,
2322                "invalid_manifest",
2323                err.to_string(),
2324            )?]);
2325        }
2326        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2327            return Ok(vec![control_error_frame(
2328                &frame,
2329                "invalid_event_declaration",
2330                err.to_string(),
2331            )?]);
2332        }
2333        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2334            return Ok(vec![control_error_frame(
2335                &frame,
2336                "invalid_manifest",
2337                err.to_string(),
2338            )?]);
2339        }
2340        if let Some(provenance) = hello_value
2341            .get("manifest")
2342            .and_then(|manifest| manifest.get("provenance"))
2343        {
2344            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2345                return Ok(vec![control_error_frame(
2346                    &frame,
2347                    "invalid_manifest",
2348                    format!("malformed manifest provenance: {err}"),
2349                )?]);
2350            }
2351        }
2352        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2353            let module_id = hello_value["manifest"]["module_id"]
2354                .as_str()
2355                .unwrap_or("<unknown>");
2356            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2357        }
2358        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2359            let module_id = hello_value["manifest"]["module_id"]
2360                .as_str()
2361                .unwrap_or("<unknown>");
2362            warn!(module_id, %reason, "dropping flow_operations from HELLO");
2363        }
2364        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2365            Ok(hello) => hello,
2366            Err(err) => {
2367                return Ok(vec![control_error_frame(
2368                    &frame,
2369                    "invalid_hello",
2370                    format!("malformed HELLO body: {err}"),
2371                )?])
2372            }
2373        };
2374
2375        if hello.protocol_ver != hello.manifest.protocol_ver {
2376            return Ok(vec![control_error_frame(
2377                &frame,
2378                "invalid_manifest",
2379                format!(
2380                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2381                    hello.protocol_ver, hello.manifest.protocol_ver
2382                ),
2383            )?]);
2384        }
2385
2386        if hello.manifest.module_id.trim().is_empty() {
2387            return Ok(vec![control_error_frame(
2388                &frame,
2389                "invalid_manifest",
2390                "manifest module_id must not be empty",
2391            )?]);
2392        }
2393
2394        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2395            Ok(negotiated_ver) => negotiated_ver,
2396            Err(message) => {
2397                return Ok(vec![control_error_frame(
2398                    &frame,
2399                    "version_unsupported",
2400                    message,
2401                )?])
2402            }
2403        };
2404
2405        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2406        // swap is open for this id, the only HELLO admitted as a second process
2407        // is the one carrying the candidate's launch nonce (the swap token), and
2408        // it registers into the candidate slot rather than being refused as a
2409        // duplicate. Run after the reserved gate, a reserved module's candidate
2410        // would be refused `reserved_module` for presenting a nonce that gate
2411        // does not know. See `SupervisorHandle::swap_hello_admission`.
2412        let swap_admission = self
2413            .supervisor
2414            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2415        if swap_admission == SwapHelloAdmission::Refused {
2416            warn!(
2417                module_id = %hello.manifest.module_id,
2418                connection_id = connection_id.get(),
2419                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2420            );
2421            return Ok(vec![control_error_frame(
2422                &frame,
2423                "swap_token_invalid",
2424                format!(
2425                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2426                    hello.manifest.module_id
2427                ),
2428            )?]);
2429        }
2430        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2431
2432        // A swap candidate proves its identity with the candidate nonce above.
2433        // Other registrations receive this catalog fact only when reserved-module
2434        // policy required and accepted a nonce; unreserved registrations require
2435        // no nonce check.
2436        let nonce_verified = if swap_candidate {
2437            true
2438        } else {
2439            match self.supervisor.reserved_hello_verification(
2440                &hello.manifest.module_id,
2441                hello.launch_nonce.as_deref(),
2442            ) {
2443                Ok(verified) => verified,
2444                Err(rejection) => {
2445                    let message = match rejection {
2446                        ReservedHelloRejection::Exact { module_id } => format!(
2447                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2448                        ),
2449                        ReservedHelloRejection::Prefix {
2450                            prefix,
2451                            owner_module_id,
2452                        } => format!(
2453                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2454                            hello.manifest.module_id
2455                        ),
2456                    };
2457                    return Ok(vec![control_error_frame(
2458                        &frame,
2459                        "reserved_module",
2460                        message,
2461                    )?]);
2462                }
2463            }
2464        };
2465
2466        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2467            &hello.manifest.module_id,
2468            hello.manifest.capabilities.as_ref(),
2469        );
2470        if let Some(refusal) = reserved_capability_refusals.first() {
2471            let capability = refusal.capability.clone();
2472            let bound_module = refusal.claimants[0].clone();
2473            log_duplicate_claim_events(reserved_capability_refusals);
2474            return Ok(vec![control_error_frame(
2475                &frame,
2476                "reserved_capability",
2477                format!(
2478                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2479                    capability, bound_module, hello.manifest.module_id
2480                ),
2481            )?]);
2482        }
2483
2484        // A connection that already opened client routes must not also register as
2485        // a module: cleanup would then release only one side and leak the other.
2486        if self
2487            .forwarding
2488            .connection_has_client_routes(connection_id)
2489            .map_err(RouterError::Forwarding)?
2490        {
2491            return Ok(vec![control_error_frame(
2492                &frame,
2493                "invalid_hello",
2494                "connection has open client routes and cannot also register as a module",
2495            )?]);
2496        }
2497
2498        // Kept for scope sync authority, which goes only to the connection that
2499        // presented the module's current launch nonce. Recorded before the
2500        // registration is attempted: a connection whose registration then fails
2501        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2502        self.hello_launch_nonces
2503            .lock()
2504            .unwrap_or_else(|poisoned| poisoned.into_inner())
2505            .record(connection_id, hello.launch_nonce.as_deref());
2506        let control_ops = effective_module_control_ops(hello.control_ops);
2507        // Built before anything is registered so an encoding failure leaves no
2508        // registry or forwarding state behind.
2509        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2510        if swap_candidate {
2511            return self.register_swap_candidate(
2512                connection_id,
2513                sink,
2514                &frame,
2515                hello.manifest,
2516                negotiated_ver,
2517                control_ops,
2518                hello_ack,
2519            );
2520        }
2521        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2522        // is refused, but `catalog.list` does not show the module and the
2523        // catalog generation does not move until it is published below, once
2524        // its forwarding endpoint exists. Listing it now would advertise a
2525        // module that `route.open` still refuses for having no forwarding
2526        // connection.
2527        let registration = match self
2528            .registry
2529            .register_unpublished_verified_with_control_ops(
2530                hello.manifest,
2531                negotiated_ver,
2532                connection_id,
2533                control_ops,
2534                nonce_verified,
2535            ) {
2536            Ok(registration) => registration,
2537            Err(RegistryError::DuplicateModuleId { module_id }) => {
2538                return Ok(vec![control_error_frame(
2539                    &frame,
2540                    "duplicate_module_id",
2541                    format!(
2542                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2543                    ),
2544                )?])
2545            }
2546            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2547                return Ok(vec![control_error_frame(
2548                    &frame,
2549                    "invalid_module_id",
2550                    err.to_string(),
2551                )?])
2552            }
2553            Err(err) => {
2554                return Ok(vec![control_error_frame(
2555                    &frame,
2556                    "registry_error",
2557                    err.to_string(),
2558                )?])
2559            }
2560        };
2561        #[cfg(test)]
2562        self.hold_between_registry_and_forwarding();
2563
2564        let reply = if let Some(sink) = sink {
2565            // The forwarding table's module store is also the daemon-to-module
2566            // control-RPC lane, so every HELLO gets a live endpoint even when the
2567            // manifest has no routable provider role. Non-routable modules still
2568            // cannot receive route.bind in production: `handle_route_open` checks
2569            // the registry manifest with `target_has_required_role` before the
2570            // only production call to `begin_route_bind_relay_for` below that
2571            // route.open path. The remaining direct relay callers are unit tests
2572            // and benchmark harnesses that construct forwarding state explicitly.
2573            //
2574            // The HELLO_ACK is queued by the forwarding table itself, before the
2575            // endpoint becomes visible, and is NOT returned as a reply. A module
2576            // reads HELLO_ACK first and exits on anything else; a reply is only
2577            // written after this handler returns, by which time a route.open on
2578            // another connection could already have queued a route.bind request
2579            // for this module ahead of it.
2580            let concurrency = manifest_concurrency(&registration.manifest);
2581            if let Err(err) = self.forwarding.register_module_connection_acked(
2582                connection_id,
2583                registration.manifest.module_id.clone(),
2584                negotiated_ver,
2585                concurrency,
2586                sink,
2587                hello_ack,
2588            ) {
2589                // Forwarding registration failed, so there is no forwarding
2590                // state to tear down. Remove the registry entry, which was never
2591                // published and so never listed or counted in the catalog
2592                // generation, and signal the release watch directly.
2593                if matches!(
2594                    self.deregister_connection(
2595                        connection_id,
2596                        RegistrationEndReason::RegistrationFailed,
2597                    ),
2598                    Ok(r) if !r.is_empty()
2599                ) {
2600                    crate::supervise::notify_registration_release();
2601                }
2602                return Ok(vec![control_error_frame(
2603                    &frame,
2604                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2605                        "invalid_hello"
2606                    } else {
2607                        forwarding_error_code(&err)
2608                    },
2609                    err.to_string(),
2610                )?]);
2611            }
2612            Vec::new()
2613        } else {
2614            // No sink means no forwarding endpoint, so nothing can be routed
2615            // ahead of the ack; it goes out as the reply.
2616            vec![hello_ack]
2617        };
2618
2619        // The module is routable now (or, without a sink, as routable as it
2620        // will ever be), so the catalog may list it. Publishing bumps the
2621        // catalog generation in the same registry critical section that makes
2622        // the module listed.
2623        if !self
2624            .registry
2625            .publish_registration(connection_id)
2626            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2627        {
2628            debug!(
2629                module_id = %registration.manifest.module_id,
2630                connection_id = connection_id.get(),
2631                "registration was removed before it could be published"
2632            );
2633        }
2634
2635        // Exposure over assumption: Concurrency's serde default is pinned to the
2636        // pre-field behavior (ModuleManaged), so a management surface that is
2637        // genuinely Serial and just never declared it inherits concurrent
2638        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2639        // turns "no module has been bitten yet" into the checkable claim "no
2640        // module is exposed" -- one read of the boot log instead of a fleet
2641        // audit. Detected from the raw HELLO bytes because the serde default
2642        // deliberately erases the absent/declared distinction from the type.
2643        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2644            info!(
2645                module_id = %registration.manifest.module_id,
2646                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2647            );
2648        }
2649
2650        self.apply_registration_capabilities(&registration);
2651
2652        self.supervisor
2653            .record_activity_registration(&registration.manifest.module_id);
2654
2655        info!(
2656            module_id = %registration.manifest.module_id,
2657            module_version = %registration.manifest.module_version,
2658            negotiated_ver,
2659            routable_provider = manifest_provides_routable_role(&registration.manifest),
2660            connection_id = connection_id.get(),
2661            "module registered"
2662        );
2663
2664        Ok(reply)
2665    }
2666
2667    /// Register a HELLO the swap gate admitted into the candidate slot of the
2668    /// registry and of forwarding, where it is reachable over its own
2669    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2670    /// nothing routes to it until the supervisor cuts over.
2671    ///
2672    /// Registry first (unpublished), then forwarding, then publication, the
2673    /// same order as an ordinary HELLO; a forwarding failure removes the
2674    /// registry entry again. The swap supervisor treats a published, ready
2675    /// candidate as registered and promotes it, and promotion needs the
2676    /// candidate's forwarding endpoint, so the candidate is published only
2677    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2678    /// recompute capability verdicts or run the capability deny census
2679    /// (`enforce_capability_denies`): both are about the module in the active
2680    /// slot, which `route.open` can target. A candidate sits in the candidate
2681    /// slot, so it neither provides capabilities to other modules nor can be
2682    /// routed to until promotion moves it into the active slot.
2683    #[allow(clippy::too_many_arguments)]
2684    fn register_swap_candidate(
2685        &self,
2686        connection_id: ConnectionId,
2687        sink: Option<crate::FrameSink>,
2688        frame: &Frame,
2689        manifest: ModuleManifest,
2690        negotiated_ver: u8,
2691        control_ops: Vec<String>,
2692        hello_ack: Frame,
2693    ) -> Result<Vec<Frame>, RouterError> {
2694        let module_id = manifest.module_id.clone();
2695        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2696            manifest,
2697            negotiated_ver,
2698            connection_id,
2699            control_ops,
2700        ) {
2701            Ok(registration) => registration,
2702            Err(RegistryError::DuplicateModuleId { module_id }) => {
2703                return Ok(vec![control_error_frame(
2704                    frame,
2705                    "duplicate_module_id",
2706                    format!(
2707                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2708                    ),
2709                )?])
2710            }
2711            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2712                return Ok(vec![control_error_frame(
2713                    frame,
2714                    "invalid_module_id",
2715                    err.to_string(),
2716                )?])
2717            }
2718            Err(err) => {
2719                return Ok(vec![control_error_frame(
2720                    frame,
2721                    "registry_error",
2722                    err.to_string(),
2723                )?])
2724            }
2725        };
2726        #[cfg(test)]
2727        self.hold_between_registry_and_forwarding();
2728        let reply = if let Some(sink) = sink {
2729            // Same ordering as an ordinary HELLO: the forwarding table queues
2730            // the HELLO_ACK before the candidate endpoint is inserted, because
2731            // a module exits if its first frame after HELLO is anything else.
2732            let concurrency = manifest_concurrency(&registration.manifest);
2733            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2734                connection_id,
2735                module_id.clone(),
2736                negotiated_ver,
2737                concurrency,
2738                sink,
2739                hello_ack,
2740            ) {
2741                if matches!(
2742                    self.deregister_connection(
2743                        connection_id,
2744                        RegistrationEndReason::RegistrationFailed,
2745                    ),
2746                    Ok(r) if !r.is_empty()
2747                ) {
2748                    crate::supervise::notify_registration_release();
2749                }
2750                return Ok(vec![control_error_frame(
2751                    frame,
2752                    forwarding_error_code(&err),
2753                    err.to_string(),
2754                )?]);
2755            }
2756            Vec::new()
2757        } else {
2758            vec![hello_ack]
2759        };
2760        if !self
2761            .registry
2762            .publish_registration(connection_id)
2763            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2764        {
2765            debug!(
2766                module_id = %module_id,
2767                connection_id = connection_id.get(),
2768                "swap candidate registration was removed before it could be published"
2769            );
2770        }
2771        self.supervisor.mark_swap_candidate_admitted(&module_id);
2772        info!(
2773            module_id = %module_id,
2774            module_version = %registration.manifest.module_version,
2775            negotiated_ver,
2776            ready = registration.ready,
2777            connection_id = connection_id.get(),
2778            "swap candidate registered; not routable until cutover"
2779        );
2780        Ok(reply)
2781    }
2782
2783    fn build_hello_ack(
2784        &self,
2785        frame: &Frame,
2786        negotiated_ver: u8,
2787        module_id: &str,
2788    ) -> Result<Frame, RouterError> {
2789        let ack = ModuleHelloAckBody {
2790            negotiated_ver,
2791            subc_ops: module_subc_ops(),
2792            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2793            storage: self
2794                .storage_config
2795                .as_ref()
2796                .map(|cfg| cfg.descriptor_for(module_id)),
2797            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2798        };
2799        let body = serde_json::to_vec(&ack).map_err(|err| {
2800            RouterError::backend(
2801                0,
2802                frame.header.corr,
2803                format!("failed to encode HELLO_ACK: {err}"),
2804            )
2805        })?;
2806
2807        Frame::build_with_version(
2808            negotiated_ver,
2809            FrameType::HelloAck,
2810            control_flags(),
2811            0,
2812            0,
2813            frame.header.corr,
2814            body,
2815        )
2816        .map_err(RouterError::FrameBuild)
2817    }
2818
2819    async fn handle_client_control_request(
2820        &self,
2821        ctx: &RouteCtx,
2822        frame: Frame,
2823        request: ClientControlRequest,
2824        route_open_timings: &mut Option<RouteOpenTimings>,
2825    ) -> Result<Vec<Frame>, RouterError> {
2826        if let Some(operation) = privileged_operation(&request) {
2827            let caller = self.resolve_caller(ctx.connection_id).await;
2828            let op = client_control_request_op(&request);
2829            let text = prompt_text(operation, &caller);
2830            match self.privileged_operator_prompt {
2831                PrivilegedOperatorPrompt::Log => {
2832                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2833                        target = ?operation, caller = %caller.description(), summary = %text,
2834                        "privileged request would prompt the operator");
2835                }
2836                PrivilegedOperatorPrompt::Enforce => {
2837                    let key =
2838                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2839                    let outcome = self
2840                        .forwarding
2841                        .operator_confirms()
2842                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2843                        .await;
2844                    if outcome.code != "confirmed" {
2845                        return Ok(vec![outcome.frame(&frame)]);
2846                    }
2847                }
2848            }
2849        }
2850        match request {
2851            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2852            ClientControlRequest::CatalogList { module_id } => {
2853                self.handle_catalog_list(frame, module_id)
2854            }
2855            ClientControlRequest::RouteOpen {
2856                target,
2857                identity,
2858                consumer_identity,
2859                consumer_capabilities,
2860                role_versions,
2861                admission_facts,
2862                scope,
2863            } => {
2864                self.handle_route_open(
2865                    ctx,
2866                    frame,
2867                    RouteOpenRequest {
2868                        target,
2869                        identity,
2870                        consumer_identity,
2871                        consumer_capabilities,
2872                        role_versions,
2873                        admission_facts,
2874                        scope,
2875                    },
2876                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2877                )
2878                .await
2879            }
2880            ClientControlRequest::RoutePoll {
2881                route_channel,
2882                route_epoch,
2883                kind,
2884            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2885            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2886            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2887                self.handle_supervisor_spawn_snapshot(frame)
2888            }
2889            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2890                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2891            }
2892            ClientControlRequest::SupervisorRestart {
2893                module_id,
2894                drain_timeout_ms,
2895            } => {
2896                self.log_supervisor_request_received(
2897                    ctx,
2898                    frame.header.corr,
2899                    ops::SUPERVISOR_RESTART,
2900                    Some(&module_id),
2901                    None,
2902                )?;
2903                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2904                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2905                    .await
2906            }
2907            ClientControlRequest::SupervisorSwap {
2908                module_id,
2909                ready_timeout_ms,
2910            } => {
2911                self.log_supervisor_request_received(
2912                    ctx,
2913                    frame.header.corr,
2914                    ops::SUPERVISOR_SWAP,
2915                    Some(&module_id),
2916                    None,
2917                )?;
2918                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2919                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2920                    .await
2921            }
2922            ClientControlRequest::SupervisorReload { module_id } => {
2923                self.log_supervisor_request_received(
2924                    ctx,
2925                    frame.header.corr,
2926                    ops::SUPERVISOR_RELOAD,
2927                    Some(&module_id),
2928                    None,
2929                )?;
2930                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2931                self.handle_supervisor_reload(frame, module_id, requester)
2932                    .await
2933            }
2934            ClientControlRequest::SupervisorRescan { preview } => {
2935                if !preview {
2936                    self.log_supervisor_request_received(
2937                        ctx,
2938                        frame.header.corr,
2939                        ops::SUPERVISOR_RESCAN,
2940                        None,
2941                        None,
2942                    )?;
2943                }
2944                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2945                self.handle_supervisor_rescan(
2946                    frame,
2947                    if preview {
2948                        RescanMode::Preview
2949                    } else {
2950                        RescanMode::Execute
2951                    },
2952                    requester,
2953                )
2954                .await
2955            }
2956            ClientControlRequest::SupervisorRescanChecked {
2957                preview,
2958                candidate,
2959                expect,
2960            } => {
2961                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2962                    .await
2963            }
2964            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2965                self.log_supervisor_request_received(
2966                    ctx,
2967                    frame.header.corr,
2968                    ops::SUPERVISOR_RELEASE_RESERVED,
2969                    Some(&module_id),
2970                    None,
2971                )?;
2972                self.handle_supervisor_release_reserved(frame, module_id)
2973                    .await
2974            }
2975            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2976                self.log_supervisor_request_received(
2977                    ctx,
2978                    frame.header.corr,
2979                    ops::SUPERVISOR_SET_ENABLED,
2980                    Some(&module_id),
2981                    Some(enabled),
2982                )?;
2983                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2984                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2985                    .await
2986            }
2987            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2988                self.handle_supervisor_health_probe(frame, module_id).await
2989            }
2990            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2991            ClientControlRequest::SupervisorRoutes { module_id } => {
2992                self.handle_supervisor_routes(frame, module_id)
2993            }
2994            ClientControlRequest::SupervisorProvenance { module_id } => {
2995                self.handle_supervisor_provenance(frame, module_id).await
2996            }
2997            ClientControlRequest::SupervisorStderrTail {
2998                module_id,
2999                max_lines,
3000                max_bytes,
3001            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3002            ClientControlRequest::SupervisorTerminals { module_id } => {
3003                self.handle_supervisor_terminals(frame, module_id).await
3004            }
3005        }
3006    }
3007
3008    fn log_supervisor_request_received(
3009        &self,
3010        ctx: &RouteCtx,
3011        corr: u64,
3012        op: &'static str,
3013        module_id: Option<&str>,
3014        enabled: Option<bool>,
3015    ) -> Result<(), RouterError> {
3016        let caller = self.supervisor_requester(ctx, corr)?;
3017        let caller = principal_label(&caller);
3018        // `Option` fields are recorded only when present, so a request with no
3019        // module id (rescan) or no enabled flag simply omits that field.
3020        info!(
3021            target: "control",
3022            op,
3023            module_id,
3024            enabled,
3025            connection_id = ctx.connection_id.get(),
3026            caller = %caller,
3027            "supervisor request received"
3028        );
3029        Ok(())
3030    }
3031
3032    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3033        Ok(self
3034            .registry
3035            .get_module_by_connection(ctx.connection_id)
3036            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3037            .map(|registration| Principal::Reserved {
3038                module_id: registration.manifest.module_id,
3039            })
3040            .unwrap_or(Principal::Direct))
3041    }
3042
3043    fn handle_module_control_request(
3044        &self,
3045        connection_id: ConnectionId,
3046        frame: Frame,
3047        request: ModuleControlRequestFromModule,
3048    ) -> Result<Vec<Frame>, RouterError> {
3049        match request {
3050            ModuleControlRequestFromModule::CatalogUpdate {
3051                provides,
3052                capabilities,
3053                ready,
3054            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3055            ModuleControlRequestFromModule::LiveRoots {} => {
3056                let registered = self
3057                    .registry
3058                    .get_module_by_connection(connection_id)
3059                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3060                let Some(registration) = registered else {
3061                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3062                };
3063                let response = self
3064                    .forwarding
3065                    .live_roots(&registration.manifest.module_id)
3066                    .map_err(RouterError::Forwarding)?;
3067                Ok(vec![control_response_body_frame(
3068                    &frame,
3069                    &response,
3070                    "ModuleControlResponseToModule::LiveRoots",
3071                )?])
3072            }
3073            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3074                self.handle_scope_sync(connection_id, frame, generation, scopes)
3075            }
3076            ModuleControlRequestFromModule::ScopeApply {
3077                generation,
3078                upsert,
3079                end,
3080            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3081            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3082                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3083            }
3084        }
3085    }
3086
3087    fn handle_scope_served_by(
3088        &self,
3089        connection_id: ConnectionId,
3090        frame: Frame,
3091    ) -> Result<Vec<Frame>, RouterError> {
3092        let registered = self
3093            .registry
3094            .get_module_by_connection(connection_id)
3095            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3096        let Some(registered) = registered else {
3097            return Ok(vec![control_error_frame(
3098                &frame,
3099                "not_registered",
3100                "scope.served_by requires an active module registration owned by this connection",
3101            )?]);
3102        };
3103        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3104        {
3105            Ok(request) => request,
3106            Err(err) => {
3107                return Ok(vec![control_error_frame(
3108                    &frame,
3109                    "invalid_control_body",
3110                    format!("malformed scope.served_by body: {err}"),
3111                )?])
3112            }
3113        };
3114        let answer = self
3115            .scopes
3116            .read()
3117            .unwrap_or_else(|poisoned| poisoned.into_inner())
3118            .served_by(
3119                &registered.manifest.module_id,
3120                &request.scope_ref,
3121                request.scope_epoch,
3122                &request.module_id,
3123            );
3124        let reply = subc_control::ScopeServedByReply::new(
3125            answer,
3126            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3127        );
3128        Ok(vec![control_response_body_frame(
3129            &frame,
3130            &reply,
3131            "ScopeServedByReply",
3132        )?])
3133    }
3134
3135    fn handle_operator_confirm(
3136        &self,
3137        ctx: &RouteCtx,
3138        frame: Frame,
3139    ) -> Result<Vec<Frame>, RouterError> {
3140        use crate::operator_confirm::{audit, Outcome};
3141        let registration = self
3142            .registry
3143            .get_module_by_connection(ctx.connection_id)
3144            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3145        let Some(registration) = registration else {
3146            let outcome = Outcome::refusal("not_registered");
3147            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3148            return Ok(vec![outcome.frame(&frame)]);
3149        };
3150        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3151            Ok(request) => request,
3152            Err(_) => {
3153                let outcome = Outcome::refusal("invalid_control_body");
3154                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3155                return Ok(vec![outcome.frame(&frame)]);
3156            }
3157        };
3158        let module_id = registration.manifest.module_id;
3159        // Read the launch nonce (under its own lock) before taking the forwarding
3160        // table's lock below: holding forwarding while waiting on another daemon
3161        // lock risks a lock-order deadlock with paths that take them the other way.
3162        let nonce = self
3163            .hello_launch_nonces
3164            .lock()
3165            .unwrap_or_else(|p| p.into_inner())
3166            .nonce(ctx.connection_id)
3167            .map(str::to_owned);
3168        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3169            self.supervisor
3170                .spawned_consumer_authorized(&module_id, nonce)
3171        });
3172        let confirms = self.forwarding.operator_confirms();
3173        self.forwarding
3174            .with_operator_route(
3175                ctx.connection_id,
3176                request.route_channel,
3177                request.route_epoch,
3178                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3179            )
3180            .map_err(RouterError::Forwarding)
3181    }
3182
3183    /// Resolve folders before taking the scope-table lock: a slow filesystem
3184    /// must not hold up admissions for unrelated scoped routes.
3185    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3186        let mut scopes = ScopeRecords::from(scopes);
3187        for (index, record) in scopes.records.iter_mut().enumerate() {
3188            if let Some(refusal) = Self::canonicalize_workdir(record) {
3189                scopes.refusals.insert(index, refusal);
3190            }
3191        }
3192        scopes
3193    }
3194
3195    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3196        let workdir = record.workdir.as_mut()?;
3197        let path = std::path::Path::new(workdir);
3198        if !path.is_absolute() {
3199            return Some((
3200                error_codes::INVALID_WORKDIR,
3201                "workdir must be absolute".to_string(),
3202            ));
3203        }
3204        match ProjectRootId::from_path_allowing_missing(path) {
3205            Ok(root) => match root.as_path().to_str() {
3206                Some(canonical) => {
3207                    *workdir = canonical.to_string();
3208                    None
3209                }
3210                None => Some((
3211                    error_codes::INVALID_WORKDIR,
3212                    "canonical workdir is not UTF-8".to_string(),
3213                )),
3214            },
3215            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3216        }
3217    }
3218
3219    /// `scope.sync`: the owner is the module registered on this connection.
3220    /// A connection with no registration (every client connection, `direct`
3221    /// included) is refused `not_registered` before the table is consulted.
3222    fn handle_scope_sync(
3223        &self,
3224        connection_id: ConnectionId,
3225        frame: Frame,
3226        generation: u64,
3227        scopes: Vec<ScopeRecord>,
3228    ) -> Result<Vec<Frame>, RouterError> {
3229        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3230    }
3231
3232    fn handle_scope_change(
3233        &self,
3234        connection_id: ConnectionId,
3235        frame: Frame,
3236        generation: u64,
3237        scopes: Vec<ScopeRecord>,
3238        end: Option<Vec<ScopeEnd>>,
3239    ) -> Result<Vec<Frame>, RouterError> {
3240        let is_apply = end.is_some();
3241        let Some(registration) = self
3242            .registry
3243            .get_module_by_connection(connection_id)
3244            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3245        else {
3246            return Ok(vec![control_error_frame(
3247                &frame,
3248                "not_registered",
3249                "scope.sync and scope.apply require an active module registration owned by this connection",
3250            )?]);
3251        };
3252        let owner = registration.manifest.module_id;
3253        let scopes = Self::prepare_scope_records(scopes);
3254        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3255        let is_current_launch = |connection: ConnectionId| {
3256            self.hello_launch_nonces
3257                .lock()
3258                .unwrap_or_else(|poisoned| poisoned.into_inner())
3259                .presented(connection, current_nonce.as_deref())
3260        };
3261        // Lock order is the scope table, then the forwarding table: the new
3262        // tags are published, and the routes the change closes are selected,
3263        // while the scope table is still write-locked, so no admission can read
3264        // a record whose tag is not yet published.
3265        let mut table = self
3266            .scopes
3267            .write()
3268            .unwrap_or_else(|poisoned| poisoned.into_inner());
3269        let outcome = match end {
3270            Some(end) => table.apply(
3271                &owner,
3272                connection_id,
3273                is_current_launch,
3274                generation,
3275                scopes,
3276                end,
3277            ),
3278            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3279        };
3280        let drained = match &outcome {
3281            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3282            Err(_) => Vec::new(),
3283        };
3284        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3285        drop(table);
3286        if has_deadlines {
3287            self.scope_deadline_added.notify_one();
3288        }
3289        match outcome {
3290            Ok(applied) => {
3291                let counts = ScopeOutcomeCounts::of(&applied.results);
3292                info!(
3293                    owner = %owner,
3294                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3295                    generation,
3296                    records = applied.results.len(),
3297                    created = counts.created,
3298                    replaced = counts.replaced,
3299                    updated = counts.updated,
3300                    unchanged = counts.unchanged,
3301                    refused = counts.refused,
3302                    ended = applied.ended.len(),
3303                    tag_changes = applied.tag_changes.len(),
3304                    routes_closed = drained.len(),
3305                    "scope change accepted"
3306                );
3307                // An accepted scope change can still refuse individual records, and the
3308                // owner is the only party that sees the reply. Name them here so
3309                // an operator can tell a refused session from a missing one
3310                // without the owner's logs. Capped so a call that refuses
3311                // thousands cannot flood the log; the count above is complete.
3312                for refused in applied
3313                    .results
3314                    .iter()
3315                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3316                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3317                {
3318                    warn!(
3319                        owner = %owner,
3320                        generation,
3321                        scope_ref = %refused.scope_ref,
3322                        scope_epoch = refused.scope_epoch,
3323                        code = refused.code.as_deref().unwrap_or(""),
3324                        "scope record refused"
3325                    );
3326                }
3327                self.close_scope_drained_routes(drained);
3328                let response = if is_apply {
3329                    ModuleControlResponseToModule::ScopeApply {
3330                        generation,
3331                        results: applied.results,
3332                        end_results: applied.end_results,
3333                        ended: applied.ended,
3334                    }
3335                } else {
3336                    ModuleControlResponseToModule::ScopeSync {
3337                        generation,
3338                        results: applied.results,
3339                        ended: applied.ended,
3340                    }
3341                };
3342                Ok(vec![control_response_body_frame(
3343                    &frame,
3344                    &response,
3345                    "scope change response",
3346                )?])
3347            }
3348            Err(refusal) => {
3349                info!(
3350                    owner = %owner,
3351                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3352                    generation,
3353                    code = refusal.code,
3354                    "scope change refused"
3355                );
3356                Ok(vec![control_error_frame(
3357                    &frame,
3358                    refusal.code,
3359                    refusal.message,
3360                )?])
3361            }
3362        }
3363    }
3364
3365    /// Tell both ends of each route a scope change closed. The module gets a
3366    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3367    /// the client's route handle. The client also gets `route.closed` with the
3368    /// scope reason, one push per module and reason, so it can tell a revoked
3369    /// route from an ordinary close and not reopen it.
3370    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3371        if drained.is_empty() {
3372            return;
3373        }
3374        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3375            BTreeMap::new();
3376        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3377        for route in drained {
3378            warn!(
3379                module_id = %route.module_id,
3380                reason = ?route.reason,
3381                client_connection_id = route.client.connection_id.get(),
3382                route_channel = route.client.channel,
3383                "closing route because its scope changed"
3384            );
3385            pushes
3386                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3387                .or_insert_with(|| (route.reason, Vec::new()))
3388                .1
3389                .push(EndpointRoute {
3390                    goodbye_target: route.client.clone(),
3391                    principal: Principal::Unverified,
3392                    bound_at: Instant::now(),
3393                    draining: false,
3394                    drain_reason: None,
3395                });
3396            goodbyes.push(route.module);
3397            goodbyes.push(route.client);
3398        }
3399        for ((module_id, _), (reason, routes)) in pushes {
3400            send_route_control_pushes(
3401                &self.forwarding,
3402                routes,
3403                ClientControlPush::RouteClosed {
3404                    module_id,
3405                    channels: Vec::new(),
3406                    reason,
3407                    drained: false,
3408                    abandoned: 0,
3409                    excluded_subscriptions: 0,
3410                    terminal: Some(false),
3411                },
3412            );
3413        }
3414        self.emit_route_goodbyes(goodbyes);
3415    }
3416
3417    /// `scope.describe`: any registered module may read any scope, because a
3418    /// provider must read the scope a route it serves is stamped with.
3419    fn handle_scope_describe(
3420        &self,
3421        connection_id: ConnectionId,
3422        frame: Frame,
3423        owner: Principal,
3424        scope_ref: String,
3425    ) -> Result<Vec<Frame>, RouterError> {
3426        let registered = self
3427            .registry
3428            .get_module_by_connection(connection_id)
3429            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3430        let Some(registered) = registered else {
3431            return Ok(vec![control_error_frame(
3432                &frame,
3433                "not_registered",
3434                "scope.describe requires an active module registration owned by this connection",
3435            )?]);
3436        };
3437        let mut description = self
3438            .scopes
3439            .read()
3440            .unwrap_or_else(|poisoned| poisoned.into_inner())
3441            .describe(&owner, &scope_ref);
3442        // Describe readers also decode strict ScopeAttributes. Attribution is
3443        // safe to omit unless the requesting module declares it understands it.
3444        if !registered
3445            .manifest
3446            .capabilities
3447            .as_ref()
3448            .is_some_and(|caps| {
3449                caps.provides
3450                    .iter()
3451                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3452            })
3453        {
3454            if let Some(stamp) = description.stamp.as_mut() {
3455                stamp.attributes.on_behalf_of_agent = None;
3456            }
3457        }
3458        if !registered
3459            .manifest
3460            .capabilities
3461            .as_ref()
3462            .is_some_and(|caps| {
3463                caps.provides
3464                    .iter()
3465                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3466            })
3467        {
3468            if let Some(stamp) = description.stamp.as_mut() {
3469                stamp.attributes.flow_version = None;
3470                stamp.attributes.flow_code_hash = None;
3471                stamp.attributes.flow_install_id = None;
3472            }
3473        }
3474        let owner_configured = match &owner {
3475            // Ask whether the owner is configured (`is_configured`), not
3476            // whether it is on the roster (`get(..).is_some()`): a supervised
3477            // module's process can register and describe a scope before the
3478            // supervisor has put it on the roster.
3479            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3480            _ => false,
3481        };
3482        let response = ModuleControlResponseToModule::ScopeDescribe {
3483            status: description.status,
3484            scope_epoch: description.scope_epoch,
3485            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3486            owner_synced: description.owner_synced,
3487            owner_configured,
3488            scope: description.stamp,
3489        };
3490        Ok(vec![control_response_body_frame(
3491            &frame,
3492            &response,
3493            "ModuleControlResponseToModule::ScopeDescribe",
3494        )?])
3495    }
3496
3497    fn handle_catalog_update(
3498        &self,
3499        connection_id: ConnectionId,
3500        frame: Frame,
3501        provides: Vec<ProviderRole>,
3502        capabilities: Option<CapabilityDeclarations>,
3503        ready: Option<bool>,
3504    ) -> Result<Vec<Frame>, RouterError> {
3505        self.refresh_capability_requirements();
3506        let Some(registration) = self
3507            .registry
3508            .get_module_by_connection(connection_id)
3509            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3510        else {
3511            return Ok(vec![control_error_frame(
3512                &frame,
3513                "not_registered",
3514                "catalog.update requires an active module registration owned by this connection",
3515            )?]);
3516        };
3517
3518        if let Some(message) =
3519            catalog_update_frozen_field_message(&registration.manifest, &provides)
3520        {
3521            return Ok(vec![control_error_frame(
3522                &frame,
3523                "catalog_update_frozen_field",
3524                message,
3525            )?]);
3526        }
3527
3528        let mut candidate = registration.manifest.clone();
3529        candidate.provides = provides.clone();
3530        candidate.capabilities = capabilities
3531            .clone()
3532            .or_else(|| registration.manifest.capabilities.clone());
3533        if let Some(rich_ui) = &candidate.rich_ui {
3534            if let Err(err) = rich_ui.validate() {
3535                return Ok(vec![control_error_frame(
3536                    &frame,
3537                    "invalid_manifest",
3538                    err.to_string(),
3539                )?]);
3540            }
3541        }
3542        if let Err(err) = candidate.validate_capability_grammar() {
3543            return Ok(vec![control_error_frame(
3544                &frame,
3545                "invalid_capability_grammar",
3546                err.to_string(),
3547            )?]);
3548        }
3549
3550        // Updates must honor the same reserved owner as initial registration;
3551        // otherwise an empty HELLO could acquire the claim after admission.
3552        let mut conflicts = self
3553            .capability_evaluator
3554            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3555        if let Some(conflict) = conflicts.first() {
3556            let message = format!(
3557                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3558                conflict.capability, conflict.claimants[0], candidate.module_id
3559            );
3560            for conflict in &mut conflicts {
3561                conflict.source = DuplicateClaimSource::CatalogUpdate;
3562            }
3563            log_duplicate_claim_events(conflicts);
3564            return Ok(vec![control_error_frame(
3565                &frame,
3566                "reserved_capability",
3567                message,
3568            )?]);
3569        }
3570
3571        let updated = self
3572            .registry
3573            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3574            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3575        if updated.is_none() {
3576            return Ok(vec![control_error_frame(
3577                &frame,
3578                "not_registered",
3579                "catalog.update requires an active module registration owned by this connection",
3580            )?]);
3581        }
3582        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3583            log_duplicate_claim_events(
3584                self.capability_evaluator
3585                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3586            );
3587        }
3588        if capability_census_trigger(
3589            registration.manifest.capabilities.as_ref(),
3590            updated
3591                .as_ref()
3592                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3593        ) {
3594            self.enforce_capability_denies();
3595        }
3596        self.refresh_capability_requirements();
3597
3598        let response = ModuleControlResponseToModule::CatalogUpdate {};
3599        control_response_body_frame(
3600            &frame,
3601            &response,
3602            "ModuleControlResponseToModule::CatalogUpdate",
3603        )
3604        .map(|frame| vec![frame])
3605    }
3606
3607    pub(crate) fn with_activity_ledger_config(
3608        mut self,
3609        run_dir: &Path,
3610        incarnation: String,
3611        config: crate::activity::ActivityConfig,
3612    ) -> Self {
3613        self.activity = Some(crate::activity::ActivityLedger::open_config(
3614            run_dir.join("activity.jsonl"),
3615            incarnation,
3616            config,
3617        ));
3618        self.supervisor.set_activity(self.activity.clone());
3619        self
3620    }
3621
3622    pub(crate) async fn configure_activity(
3623        &self,
3624        config: crate::activity::ActivityConfig,
3625    ) -> Result<(), &'static str> {
3626        match &self.activity {
3627            Some(ledger) => ledger.configure(config).await,
3628            None => Ok(()),
3629        }
3630    }
3631
3632    fn cancel_activity_subscription(
3633        &self,
3634        ctx: &RouteCtx,
3635        frame: &Frame,
3636    ) -> Result<bool, RouterError> {
3637        let task = self
3638            .activity_streams
3639            .lock()
3640            .unwrap_or_else(|p| p.into_inner())
3641            .remove(&(ctx.connection_id, frame.header.corr));
3642        let Some(task) = task else {
3643            return Ok(false);
3644        };
3645        task.abort();
3646        let end = Frame::build_with_version(
3647            response_version(frame),
3648            FrameType::StreamEnd,
3649            control_flags(),
3650            0,
3651            0,
3652            frame.header.corr,
3653            Vec::new(),
3654        )
3655        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3656        let sink = ctx.egress.clone();
3657        tokio::spawn(async move {
3658            let _ = sink.send(end).await;
3659        });
3660        Ok(true)
3661    }
3662
3663    async fn handle_activity_read(
3664        &self,
3665        ctx: &RouteCtx,
3666        frame: Frame,
3667    ) -> Result<Vec<Frame>, RouterError> {
3668        use subc_control::{
3669            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3670            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3671        };
3672        #[derive(Deserialize)]
3673        struct Identity {
3674            #[serde(default)]
3675            consumer_identity: Option<ConsumerIdentity>,
3676        }
3677        // A registered module connection already identifies its caller. Ignore
3678        // consumer_identity there so a module cannot borrow a mirror's identity.
3679        // Only separate consumer connections use the Direct/nonce trust boundary.
3680        let principal = match self
3681            .forwarding
3682            .module_id_for_connection(ctx.connection_id)
3683            .map_err(RouterError::Forwarding)?
3684        {
3685            Some(module_id) => Ok(Principal::Reserved { module_id }),
3686            None => {
3687                let identity: Identity = match serde_json::from_slice(&frame.body) {
3688                    Ok(identity) => identity,
3689                    Err(_) => {
3690                        return Ok(vec![control_error_frame(
3691                            &frame,
3692                            ACTIVITY_MALFORMED,
3693                            "malformed activity read",
3694                        )?])
3695                    }
3696                };
3697                self.route_open_principal(&frame, identity.consumer_identity)?
3698            }
3699        };
3700        let admitted = matches!(principal, Ok(Principal::Direct))
3701            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3702        if !admitted {
3703            return Ok(vec![control_error_frame(
3704                &frame,
3705                ACTIVITY_READ_FORBIDDEN,
3706                "activity reads require an operator or trusted mirror principal",
3707            )?]);
3708        }
3709        let Some(ledger) = &self.activity else {
3710            return Ok(vec![control_error_frame(
3711                &frame,
3712                ACTIVITY_UNAVAILABLE,
3713                "activity ledger unavailable",
3714            )?]);
3715        };
3716        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3717            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3718        {
3719            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3720                Ok(request) => ledger.query(request).await,
3721                Err(_) => Err(ACTIVITY_MALFORMED),
3722            };
3723            return match result {
3724                Ok(reply) => Ok(vec![control_response_body_frame(
3725                    &frame,
3726                    &reply,
3727                    ops::ACTIVITY_QUERY,
3728                )?]),
3729                Err(code) => Ok(vec![control_error_frame(
3730                    &frame,
3731                    code,
3732                    "activity query refused",
3733                )?]),
3734            };
3735        }
3736        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3737            Ok(request) => request,
3738            Err(_) => {
3739                return Ok(vec![control_error_frame(
3740                    &frame,
3741                    ACTIVITY_MALFORMED,
3742                    "malformed activity subscription",
3743                )?])
3744            }
3745        };
3746        let mut subscription = match ledger.subscribe(request.since).await {
3747            Ok(subscription) => subscription,
3748            Err(code) => {
3749                return Ok(vec![control_error_frame(
3750                    &frame,
3751                    code,
3752                    "activity subscription refused",
3753                )?])
3754            }
3755        };
3756        let key = (ctx.connection_id, frame.header.corr);
3757        let sink = ctx.egress.clone();
3758        let mut streams = self
3759            .activity_streams
3760            .lock()
3761            .unwrap_or_else(|p| p.into_inner());
3762        streams.retain(|_, task| !task.is_finished());
3763        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3764            return Ok(vec![control_error_frame(
3765                &frame,
3766                "duplicate_subscription",
3767                "activity subscription correlation id already in use",
3768            )?]);
3769        }
3770        let task = tokio::spawn(async move {
3771            loop {
3772                let entry = if let Some(entry) = subscription.replay.pop_front() {
3773                    Some(entry)
3774                } else {
3775                    subscription.receiver.recv().await
3776                };
3777                let Some(entry) = entry else {
3778                    break;
3779                };
3780                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3781                    return;
3782                };
3783                let Ok(next) = Frame::build_with_version(
3784                    response_version(&frame),
3785                    FrameType::StreamData,
3786                    control_flags(),
3787                    0,
3788                    0,
3789                    frame.header.corr,
3790                    body,
3791                ) else {
3792                    return;
3793                };
3794                if sink.send(next).await.is_err() {
3795                    return;
3796                }
3797            }
3798            // Dropping a full subscriber's sender drains its accepted entries
3799            // before the terminal error, preserving their order on the wire.
3800            if let Some(code) = subscription.terminal_code() {
3801                if let Ok(end) =
3802                    control_error_frame(&frame, code, "activity subscriber fell behind")
3803                {
3804                    let _ = sink.send(end).await;
3805                }
3806            }
3807        });
3808        streams.insert(key, task.abort_handle());
3809        Ok(Vec::new())
3810    }
3811
3812    async fn handle_activity_record(
3813        &self,
3814        ctx: &RouteCtx,
3815        frame: Frame,
3816    ) -> Result<Vec<Frame>, RouterError> {
3817        use subc_control::{
3818            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3819            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3820        };
3821        let result = async {
3822            let module = self
3823                .forwarding
3824                .module_id_for_connection(ctx.connection_id)
3825                .ok()
3826                .flatten()
3827                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3828            let request: ActivityRecordRequest =
3829                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3830            crate::activity::validate(&module, &request)?;
3831            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3832            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3833            let at_ms = std::time::SystemTime::now()
3834                .duration_since(std::time::UNIX_EPOCH)
3835                .unwrap_or_default()
3836                .as_millis() as u64;
3837            ledger.record(module, request, actor, at_ms).await
3838        }
3839        .await;
3840        match result {
3841            Ok(entry) => Ok(vec![control_response_body_frame(
3842                &frame,
3843                &ActivityRecordResponse { seq: entry.seq },
3844                "activity.record",
3845            )?]),
3846            Err(code) => Ok(vec![control_error_frame(
3847                &frame,
3848                code,
3849                "activity recording refused",
3850            )?]),
3851        }
3852    }
3853
3854    fn activity_actor(
3855        &self,
3856        connection: ConnectionId,
3857        module: &str,
3858        on: Option<&subc_control::ActivityOn>,
3859    ) -> Result<subc_control::ActivityActor, &'static str> {
3860        use crate::forwarding::{DataRoute, DataRouteState};
3861        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3862        let principal = |id: &str| Principal::Reserved {
3863            module_id: id.to_owned(),
3864        };
3865        let mut actor = ActivityActor {
3866            principal: principal(module),
3867            scope_owner: None,
3868            agent_id: None,
3869            run_id: None,
3870            flow_id: None,
3871            project_root: None,
3872            workdir: None,
3873            project_id: None,
3874        };
3875        let scope = match on {
3876            None => return Ok(actor),
3877            Some(ActivityOn::Scope(scope)) => {
3878                let selector = subc_protocol::scope::ScopeSelector::new(
3879                    principal(module),
3880                    scope.scope_ref.clone(),
3881                    Some(scope.epoch),
3882                );
3883                self.scopes
3884                    .read()
3885                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3886                    .attribute(&principal(module), module, &selector, true)
3887                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3888            }
3889            Some(ActivityOn::Route(route)) => {
3890                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3891                    .forwarding
3892                    .lookup_data_route(connection, route.channel, route.epoch)
3893                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3894                else {
3895                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3896                };
3897                if binding.module_id != module {
3898                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3899                }
3900                actor.principal = binding.principal.clone();
3901                actor.project_root = binding
3902                    .project_root
3903                    .as_ref()
3904                    .map(|root| root.as_path().to_path_buf());
3905                let Some(bound) = &binding.scope else {
3906                    return Ok(actor);
3907                };
3908                let owner = principal(&bound.owner);
3909                let selector = subc_protocol::scope::ScopeSelector::new(
3910                    owner.clone(),
3911                    bound.scope_ref.clone(),
3912                    Some(bound.tag.scope_epoch),
3913                );
3914                let admitted = self
3915                    .scopes
3916                    .read()
3917                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3918                    .attribute(&owner, module, &selector, true)
3919                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3920                if admitted.tag != bound.tag {
3921                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3922                }
3923                admitted
3924            }
3925        };
3926        actor.scope_owner = Some(scope.stamp.owner);
3927        actor.agent_id = scope.stamp.attributes.agent_id;
3928        actor.run_id = scope.stamp.attributes.run_id;
3929        actor.flow_id = scope.stamp.attributes.flow_id;
3930        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3931        actor.project_id = scope.stamp.project_id;
3932        Ok(actor)
3933    }
3934
3935    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3936        self.refresh_capability_requirements();
3937        // A bare connection count is ambiguous between many clients holding a
3938        // route each and one client accumulating hundreds, so publish the
3939        // concentration alongside it. Route state is best-effort here: a
3940        // diagnostic endpoint must still answer if the forwarding lock is
3941        // contended.
3942        let mut counters = self.counters.snapshot();
3943        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3944            obj.insert(
3945                "activity_unparseable_lines".into(),
3946                ledger.read_errors().into(),
3947            );
3948            obj.insert(
3949                "activity_lifecycle_append_failures".into(),
3950                ledger.lifecycle_failures().into(),
3951            );
3952            obj.insert(
3953                "activity_started_digest_skips".into(),
3954                ledger.lifecycle_digest_skips().into(),
3955            );
3956        }
3957        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3958            self.forwarding.client_route_concentration(),
3959            counters.as_object_mut(),
3960        ) {
3961            obj.insert(
3962                "client_connections_with_routes".into(),
3963                connections_with_routes.into(),
3964            );
3965            obj.insert("max_routes_on_one_connection".into(), max.into());
3966        }
3967        // A module that is being fast-refused and a module that is fine look
3968        // identical from a client that retries and succeeds, so name the open
3969        // breakers here. This rides the existing free-form counters object
3970        // rather than a new wire field, so no sibling that deserializes
3971        // `ServerDescribe` has to be rebuilt to keep reading it.
3972        if let (Some(open_breakers), Some(obj)) = (
3973            self.route_bind_breakers.open_snapshot(),
3974            counters.as_object_mut(),
3975        ) {
3976            obj.insert("route_bind_breakers_open".into(), open_breakers);
3977        }
3978        let response = ClientControlResponse::ServerDescribe {
3979            protocol_ver: PROTOCOL_VERSION,
3980            subc_ops: subc_ops(),
3981            capabilities: self.describe_capabilities(),
3982            connected_clients: self.connected_clients.count(),
3983            counters: Some(counters),
3984            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3985            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3986            capability_requirements: self.capability_requirement_statuses(),
3987            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3988            config_path: self
3989                .rescan
3990                .as_ref()
3991                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3992        };
3993        Ok(vec![control_response_body_frame(
3994            &frame,
3995            &response,
3996            "ClientControlResponse::ServerDescribe",
3997        )?])
3998    }
3999
4000    fn handle_catalog_list(
4001        &self,
4002        frame: Frame,
4003        module_id: Option<String>,
4004    ) -> Result<Vec<Frame>, RouterError> {
4005        let (generation, modules) = self
4006            .registry
4007            .list_modules_with_nonce_verification()
4008            .map_err(|err| {
4009                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4010            })?;
4011        let entries = modules
4012            .into_iter()
4013            .filter(|(registration, _)| {
4014                module_id
4015                    .as_deref()
4016                    .map(|wanted| registration.manifest.module_id == wanted)
4017                    .unwrap_or(true)
4018            })
4019            .map(|(registration, nonce_verified)| {
4020                let not_ready = self.not_ready_reason(&registration);
4021                let roles = registration.manifest.provides;
4022                CatalogEntry::new(
4023                    registration.manifest.module_id,
4024                    roles,
4025                    registration.control_ops,
4026                )
4027                .with_ready(not_ready.is_none())
4028                .with_not_ready(not_ready)
4029                .with_module_version(Some(registration.manifest.module_version))
4030                .with_capabilities(registration.manifest.capabilities)
4031                .with_self_signals(registration.manifest.self_signals)
4032                .with_events(registration.manifest.events)
4033                .with_operation_delivery(registration.manifest.operation_delivery)
4034                .with_operation_payload(registration.manifest.operation_payload)
4035                .with_flow_operations(registration.manifest.flow_operations)
4036                .with_rich_ui(registration.manifest.rich_ui)
4037                .with_role_declarations(registration.manifest.role_declarations)
4038                .with_nonce_verified(nonce_verified)
4039            })
4040            .collect();
4041        let response = ClientControlResponse::CatalogList {
4042            generation,
4043            modules: entries,
4044            subc_ops: subc_ops(),
4045        };
4046        Ok(vec![control_response_body_frame(
4047            &frame,
4048            &response,
4049            "ClientControlResponse::CatalogList",
4050        )?])
4051    }
4052
4053    /// The principal a route.open's `consumer_identity` attests: `Direct`
4054    /// when it presents none, `Reserved` when its launch nonce verifies, and
4055    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4056    /// can run before any refusal check without changing which check refuses.
4057    fn attest_route_open_consumer(
4058        &self,
4059        consumer_identity: Option<&ConsumerIdentity>,
4060    ) -> Option<Principal> {
4061        let Some(consumer_identity) = consumer_identity else {
4062            return Some(Principal::Direct);
4063        };
4064        self.supervisor
4065            .spawned_consumer_authorized(
4066                &consumer_identity.module_id,
4067                &consumer_identity.launch_nonce,
4068            )
4069            .then(|| Principal::Reserved {
4070                module_id: consumer_identity.module_id.clone(),
4071            })
4072    }
4073
4074    /// The caller fields for a refusal that only has the raw request frame,
4075    /// such as the connection-level admission refusal sent before the request
4076    /// reaches [`Self::handle_route_open`].
4077    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4078        let Ok(ClientControlRequest::RouteOpen {
4079            identity,
4080            consumer_identity,
4081            scope,
4082            ..
4083        }) = parse_client_control_request(&frame.body)
4084        else {
4085            return RouteOpenCaller::unknown();
4086        };
4087        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4088        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4089    }
4090
4091    fn route_open_principal(
4092        &self,
4093        frame: &Frame,
4094        consumer_identity: Option<ConsumerIdentity>,
4095    ) -> Result<Result<Principal, Frame>, RouterError> {
4096        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4097        self.route_open_principal_attested(frame, consumer_identity, attested)
4098    }
4099
4100    /// [`Self::route_open_principal`] with `attested` already taken by
4101    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4102    /// open checks the launch nonce once.
4103    fn route_open_principal_attested(
4104        &self,
4105        frame: &Frame,
4106        consumer_identity: Option<ConsumerIdentity>,
4107        attested: Option<Principal>,
4108    ) -> Result<Result<Principal, Frame>, RouterError> {
4109        if let Some(principal) = attested {
4110            return Ok(Ok(principal));
4111        }
4112        let Some(consumer_identity) = consumer_identity else {
4113            return Ok(Ok(Principal::Direct));
4114        };
4115
4116        Ok(Err(control_error_frame(
4117            frame,
4118            "bad_consumer_identity",
4119            format!(
4120                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4121                consumer_identity.module_id
4122            ),
4123        )?))
4124    }
4125
4126    /// Ordinary `route.open` refusals go through here; admission and breaker
4127    /// refusals log separately with their capacity or breaker state. The daemon can
4128    /// attest which code it sent: without the event, a client's "the daemon
4129    /// refused me" and the daemon's own view could only be reconciled by
4130    /// argument. Malformed input (`invalid_project_root`) does not come here;
4131    /// rejecting a request that was never a valid open is not a refusal of one.
4132    #[allow(clippy::too_many_arguments)]
4133    fn route_open_refusal_frame(
4134        &self,
4135        ctx: &RouteCtx,
4136        caller: &RouteOpenCaller,
4137        frame: &Frame,
4138        module_id: &str,
4139        reason: &'static str,
4140        code: &'static str,
4141        message: impl Into<String>,
4142    ) -> Result<Frame, RouterError> {
4143        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4144        control_error_frame(frame, code, message.into())
4145    }
4146
4147    /// Refuse a `route.open` because the target module's bind-relay breaker is
4148    /// open, without attempting the relay.
4149    ///
4150    /// The wire code is `module_timeout`, which is the truth (the module has
4151    /// not been answering binds) and which both SDKs already classify as
4152    /// retryable with capped backoff. Reusing it is what keeps this change out
4153    /// of both SDKs; the daemon-side distinction lives in the counter key
4154    /// instead.
4155    ///
4156    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4157    /// While a breaker is open this fires on every open to that module, and the
4158    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4159    /// produced 261 lines about a single module inside 3000 lines of daemon
4160    /// log. The rare transitions are logged at warn/info instead and the volume
4161    /// is carried by the counter, so the evidence survives without the flood.
4162    /// The debug line keeps a per-refusal record reachable for whoever turns
4163    /// the level up.
4164    #[allow(clippy::too_many_arguments)]
4165    fn route_open_breaker_refusal_frame(
4166        &self,
4167        ctx: &RouteCtx,
4168        caller: &RouteOpenCaller,
4169        frame: &Frame,
4170        module_id: &str,
4171        consecutive_timeouts: u32,
4172        retry_in: Duration,
4173        probe_in_flight: bool,
4174    ) -> Result<Frame, RouterError> {
4175        self.counters
4176            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4177        debug!(
4178            target: "control",
4179            code = "module_timeout",
4180            module_id = ?module_id,
4181            connection_id = ctx.connection_id.get(),
4182            principal = %caller.principal,
4183            scope = %caller.scope,
4184            harness = %caller.harness,
4185            consecutive_timeouts,
4186            retry_in_ms = retry_in.as_millis() as u64,
4187            probe_in_flight,
4188            "route.open refused by open bind-relay breaker"
4189        );
4190        // Say what a caller can act on. An open bind-relay breaker means the
4191        // module timed out accepting several new routes in a row. The module
4192        // is still running and its established routes keep working; only new
4193        // route.open requests are refused until the cooldown ends and one
4194        // test route (the probe) gets through. A message that only counts
4195        // failed relays reads as "the module is down" to a worker that sees it.
4196        let detail = if probe_in_flight {
4197            "one test route is already being tried; retry once it settles".to_string()
4198        } else {
4199            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4200        };
4201        control_error_frame(
4202            frame,
4203            "module_timeout",
4204            format!(
4205                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4206                 timed out in a row); {detail}; its established routes are unaffected"
4207            ),
4208        )
4209    }
4210
4211    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4212    /// requester's bytes (an unknown target is whatever the client sent) and
4213    /// is Debug-formatted so control characters land in the log escaped
4214    /// rather than as terminal sequences for whoever tails it.
4215    ///
4216    /// `reason` names the check that refused, because one wire code has
4217    /// several senders: after a module registers, `target_unavailable` can
4218    /// come from a missing role, an inactive registration, a supervisor that
4219    /// has not marked the process live, a missing forwarding connection, or a
4220    /// failed relay, and a log that records only the code cannot say which of
4221    /// them fired. It is a static, daemon-chosen label per branch, so it is
4222    /// safe to print plainly and stays a closed set.
4223    fn observe_route_open_refusal(
4224        &self,
4225        ctx: &RouteCtx,
4226        caller: &RouteOpenCaller,
4227        module_id: &str,
4228        reason: &'static str,
4229        code: &'static str,
4230    ) {
4231        self.counters.increment_route_open_refused(code);
4232        info!(
4233            target: "control",
4234            code,
4235            reason,
4236            module_id = ?module_id,
4237            connection_id = ctx.connection_id.get(),
4238            principal = %caller.principal,
4239            scope = %caller.scope,
4240            harness = %caller.harness,
4241            "route.open refused"
4242        );
4243        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4244            self.route_outages.record_not_serving(module_id, reason);
4245        }
4246    }
4247
4248    /// Record an ACCEPTED route.open.
4249    ///
4250    /// Refusals have been logged and counted since the attestation work; accepts
4251    /// were invisible, so the daemon knew every principal it stamped and wrote
4252    /// none of them down. The party that attests the identity was the only party
4253    /// not recording it, which left a credential vault unable to name the sender
4254    /// of a call that reached it (claustrum #43) and left the launch-nonce
4255    /// concurrency question unanswerable from the outside.
4256    ///
4257    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4258    /// `code`/`module_id`/`connection_id` returns both directions of the same
4259    /// decision rather than two shapes a reader has to join by hand.
4260    ///
4261    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4262    /// and the difference carries information rather than being an
4263    /// inconsistency. This line is only reachable after a successful bind to a
4264    /// REGISTERED module, so the value has already passed HELLO validation
4265    /// including the path-hazard refusal and cannot contain control bytes. A
4266    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4267    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4268    ///
4269    /// Bare is also what every other daemon line already emits (`module
4270    /// registered`, `configured module supervised`). Shipping `?module_id` here
4271    /// made this instrument the only one in the file whose ids did not answer
4272    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4273    /// line whose whole purpose is being grepped beside its sibling.
4274    ///
4275    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4276    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4277    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4278    /// one are recorded identically. A test written against that harness passes
4279    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4280    /// green assertion that cannot fail. The same limit applies to the escaping
4281    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4282    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4283    /// Fencing either needs the real formatter, not the capture layer.
4284    ///
4285    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4286    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4287    /// to record. The ephemeral port would decay within minutes and answer only a
4288    /// live question. The identity question is instead answered by counting
4289    /// distinct live connections presenting one module's `consumer_identity` --
4290    /// "is anyone else holding this secret" rather than "is this the right
4291    /// process".
4292    fn observe_route_open_accept(
4293        &self,
4294        ctx: &RouteCtx,
4295        module_id: &str,
4296        principal: &str,
4297        scope: &str,
4298    ) {
4299        self.route_outages.record_accepted(module_id);
4300        self.counters.increment_route_open_accepted(principal);
4301        info!(
4302            target: "control",
4303            principal,
4304            module_id,
4305            connection_id = ctx.connection_id.get(),
4306            scope = %scope,
4307            "route.open accepted"
4308        );
4309    }
4310
4311    fn supervised_absent_route_open_refusal_frame(
4312        &self,
4313        ctx: &RouteCtx,
4314        caller: &RouteOpenCaller,
4315        frame: &Frame,
4316        module_id: &str,
4317        code: &'static str,
4318        status: &crate::supervise::ModuleStatus,
4319    ) -> Result<Frame, RouterError> {
4320        self.counters.increment_route_open_refused(code);
4321        info!(
4322            target: "control",
4323            code,
4324            reason = "supervised_not_registered",
4325            module_id = ?module_id,
4326            connection_id = ctx.connection_id.get(),
4327            principal = %caller.principal,
4328            scope = %caller.scope,
4329            harness = %caller.harness,
4330            state = %status.state,
4331            enabled = status.enabled,
4332            live = status.live,
4333            "route.open refused"
4334        );
4335        // A supervised module whose process has not registered is not
4336        // serving, whatever the reason; the supervisor knows this id, so it is
4337        // safe to track.
4338        self.route_outages
4339            .record_not_serving(module_id, "supervised_not_registered");
4340        let message = if let Some(pid) = status.waiting_for_previous_pid {
4341            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4342                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4343            })
4344        } else {
4345            format!(
4346                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4347                status.state, status.enabled, status.live
4348            )
4349        };
4350        control_error_frame(frame, code, message)
4351    }
4352
4353    async fn handle_route_open(
4354        &self,
4355        ctx: &RouteCtx,
4356        frame: Frame,
4357        request: RouteOpenRequest,
4358        timings: &mut RouteOpenTimings,
4359    ) -> Result<Vec<Frame>, RouterError> {
4360        let RouteOpenRequest {
4361            target,
4362            mut identity,
4363            consumer_identity,
4364            consumer_capabilities,
4365            role_versions,
4366            admission_facts,
4367            scope,
4368        } = request;
4369        let target_module_id = target_module_id(&target).to_string();
4370        timings.module_id.clone_from(&target_module_id);
4371        if self
4372            .registry
4373            .get_module_by_connection(ctx.connection_id)
4374            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4375            .is_some()
4376        {
4377            return Ok(vec![control_error_frame(
4378                &frame,
4379                "invalid_request",
4380                "module connections cannot open client routes",
4381            )?]);
4382        }
4383        debug!(
4384            connection_id = ctx.connection_id.get(),
4385            corr = frame.header.corr,
4386            module_id = %target_module_id,
4387            "handling route.open"
4388        );
4389
4390        // The caller's attested principal, requested scope and harness, for
4391        // every `route.open refused` line below. The launch nonce is checked
4392        // here, once, and the answer is reused where the principal is
4393        // enforced further down, so the order of refusals is unchanged.
4394        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4395        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4396
4397        // A malformed declaration is refused first, before anything about the
4398        // target is looked up: the same body would be refused against any
4399        // module, so the caller learns nothing by retrying or waiting. An empty
4400        // map declares nothing and travels as no field at all, so a provider
4401        // only ever sees a missing field or a non-empty one.
4402        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4403        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4404            self.observe_route_open_refusal(
4405                ctx,
4406                &caller,
4407                &target_module_id,
4408                "invalid_role_versions",
4409                error_codes::INVALID_REQUEST,
4410            );
4411            return Ok(vec![control_error_body_frame(
4412                &frame,
4413                ErrorBody {
4414                    code: error_codes::INVALID_REQUEST.to_string(),
4415                    message: error.to_string(),
4416                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4417                },
4418            )?]);
4419        }
4420
4421        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4422            if status.waiting_for_previous_pid.is_some() {
4423                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4424                    ctx,
4425                    &caller,
4426                    &frame,
4427                    &target_module_id,
4428                    "module_warming",
4429                    &status,
4430                )?]);
4431            }
4432        }
4433
4434        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4435        // opposite. Below, a caller learns whether a module is unregistered,
4436        // supervised-but-down (with state/enabled/live), or registered without the
4437        // requested role. Elsewhere that is an enumeration leak: a probe learning
4438        // the shape of a fleet it cannot otherwise see.
4439        //
4440        // It is not one here, and the reason is the ACCESS MODEL rather than
4441        // anything about these errors. Reaching route.open requires the
4442        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4443        // connection file, so any caller who completes it already runs as this
4444        // user -- and can read subc.jsonc for the module list and `ck module
4445        // status` for live state. The reply discloses nothing the caller cannot
4446        // read more easily from disk, while the precision is load-bearing:
4447        // `unknown_module` is retryable and a missing role is not.
4448        //
4449        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4450        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4451        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4452        let Some(registration) = self
4453            .registry
4454            .get_module(&target_module_id)
4455            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4456        else {
4457            if let Some((status, warming)) =
4458                self.supervisor_status(&target_module_id, frame.header.corr)?
4459            {
4460                // BEFORE the two availability codes below, because for a module
4461                // that speaks no subc wire both of them are false comfort: they
4462                // say "not right now" and are retried, and this module will
4463                // never register no matter how long the caller waits. The
4464                // absence here is the declaration being honoured, not a module
4465                // that is late.
4466                if status.protocol == ModuleProtocol::None {
4467                    return Ok(vec![self.route_open_refusal_frame(
4468                        ctx,
4469                        &caller,
4470                        &frame,
4471                        &target_module_id,
4472                        "protocol_none",
4473                        error_codes::MODULE_NO_PROTOCOL,
4474                        format!(
4475                            "module_id '{target_module_id}' is declared protocol: none; \
4476                             it speaks no subc wire and serves no routes"
4477                        ),
4478                    )?]);
4479                }
4480                let code = if warming {
4481                    "module_warming"
4482                } else {
4483                    "target_unavailable"
4484                };
4485                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4486                    ctx,
4487                    &caller,
4488                    &frame,
4489                    &target_module_id,
4490                    code,
4491                    &status,
4492                )?]);
4493            }
4494            if let Some(removed_ago_ms) =
4495                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4496            {
4497                return Ok(vec![self.route_open_refusal_frame(
4498                    ctx,
4499                    &caller,
4500                    &frame,
4501                    &target_module_id,
4502                    "removed",
4503                    error_codes::MODULE_REMOVED,
4504                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4505                )?]);
4506            }
4507            return Ok(vec![self.route_open_refusal_frame(
4508                ctx,
4509                &caller,
4510                &frame,
4511                &target_module_id,
4512                "not_registered",
4513                error_codes::UNKNOWN_MODULE,
4514                format!("module_id '{target_module_id}' is not registered"),
4515            )?]);
4516        };
4517
4518        // Best-effort only: registry readiness and forwarding reservation use
4519        // different locks, so a module can flip readiness between this read and
4520        // the relay. Modules must still tolerate an `on_bind` while not ready.
4521        if !registration.ready {
4522            self.counters
4523                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4524            info!(
4525                target: "control",
4526                code = error_codes::MODULE_WARMING,
4527                module_id = ?target_module_id,
4528                connection_id = ctx.connection_id.get(),
4529                principal = %caller.principal,
4530                scope = %caller.scope,
4531                harness = %caller.harness,
4532                reason = "declared_not_ready",
4533                "route.open refused"
4534            );
4535            // The module is registered but says it cannot take work, which is
4536            // an outage from the caller's side even though its process is up.
4537            self.route_outages
4538                .record_not_serving(&target_module_id, "declared_not_ready");
4539            return Ok(vec![control_error_body_frame(
4540                &frame,
4541                ErrorBody {
4542                    code: error_codes::MODULE_WARMING.to_string(),
4543                    message: format!(
4544                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4545                    ),
4546                    detail: Some(serde_json::json!({
4547                        "reason": "declared_not_ready"
4548                    })),
4549                },
4550            )?]);
4551        }
4552
4553        // Effective readiness, second half: a module that declares a capability
4554        // `need: required` is not routable while that capability has no
4555        // registered provider. It is enforced HERE, as a retryable routing
4556        // refusal, and deliberately not as spawn ordering or a boot block. The
4557        // module is still started and registered and can make its own calls;
4558        // spawn ordering is a promise that cannot be kept once a provider
4559        // crashes at runtime, and refusing to boot would stop the whole
4560        // machine, including the tools needed to fix its configuration.
4561        //
4562        // "Provided" is the evaluator's verdict, which counts a provider as
4563        // soon as it has REGISTERED, not once it is ready. Two modules that
4564        // require each other's capabilities are therefore both routable once
4565        // both register; counting readiness instead would deadlock them.
4566        //
4567        // Only new opens are refused. Routes already bound when a provider
4568        // goes away stay bound: nothing here tears them down, and the module
4569        // answers them as it can. Like the readiness read above this is
4570        // best-effort against a provider registering or leaving concurrently.
4571        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4572            self.counters
4573                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4574            info!(
4575                target: "control",
4576                code = error_codes::MODULE_WARMING,
4577                module_id = ?target_module_id,
4578                connection_id = ctx.connection_id.get(),
4579                principal = %caller.principal,
4580                scope = %caller.scope,
4581                harness = %caller.harness,
4582                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4583                capability = %capability,
4584                "route.open refused"
4585            );
4586            return Ok(vec![control_error_body_frame(
4587                &frame,
4588                ErrorBody {
4589                    code: error_codes::MODULE_WARMING.to_string(),
4590                    message: format!(
4591                        "module_id '{target_module_id}' requires capability '{capability}', \
4592                         which no registered module provides; retry"
4593                    ),
4594                    detail: Some(serde_json::json!({
4595                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4596                        "capability": capability,
4597                    })),
4598                },
4599            )?]);
4600        }
4601
4602        if !target_has_required_role(&target, &registration.manifest.provides) {
4603            return Ok(vec![self.route_open_refusal_frame(
4604                ctx,
4605                &caller,
4606                &frame,
4607                &target_module_id,
4608                "role_not_provided",
4609                "target_unavailable",
4610                format!("module_id '{target_module_id}' does not provide the requested target"),
4611            )?]);
4612        }
4613
4614        if registration.state != ChannelState::Active {
4615            return Ok(vec![self.route_open_refusal_frame(
4616                ctx,
4617                &caller,
4618                &frame,
4619                &target_module_id,
4620                "registration_not_active",
4621                "target_unavailable",
4622                format!("module_id '{target_module_id}' is not active"),
4623            )?]);
4624        }
4625
4626        if self
4627            .forwarding
4628            .module_is_draining(&target_module_id)
4629            .map_err(RouterError::Forwarding)?
4630        {
4631            return Ok(vec![self.route_open_refusal_frame(
4632                ctx,
4633                &caller,
4634                &frame,
4635                &target_module_id,
4636                "reloading",
4637                "module_reloading",
4638                format!("module_id '{target_module_id}' is reloading"),
4639            )?]);
4640        }
4641
4642        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4643            process_liveness.process_live(&target_module_id) == Some(false)
4644        }) {
4645            // A module the supervisor is restarting or reloading can still hold
4646            // a registration: the old process before its connection closes, or
4647            // a new one that registered while the supervisor was draining. The
4648            // forwarding table does not see that as draining, but the consumer
4649            // should still be told to retry soon, exactly as for the drain
4650            // above, rather than that the target is unavailable.
4651            if process_liveness.process_replacing(&target_module_id) {
4652                return Ok(vec![self.route_open_refusal_frame(
4653                    ctx,
4654                    &caller,
4655                    &frame,
4656                    &target_module_id,
4657                    "reloading",
4658                    "module_reloading",
4659                    format!("module_id '{target_module_id}' is reloading"),
4660                )?]);
4661            }
4662            return Ok(vec![self.route_open_refusal_frame(
4663                ctx,
4664                &caller,
4665                &frame,
4666                &target_module_id,
4667                "supervisor_not_live",
4668                "target_unavailable",
4669                format!("module_id '{target_module_id}' is not live"),
4670            )?]);
4671        }
4672
4673        if !self
4674            .forwarding
4675            .has_live_module_connection(&target_module_id)
4676            .map_err(RouterError::Forwarding)?
4677        {
4678            return Ok(vec![self.route_open_refusal_frame(
4679                ctx,
4680                &caller,
4681                &frame,
4682                &target_module_id,
4683                "no_forwarding_connection",
4684                "target_unavailable",
4685                format!("module_id '{target_module_id}' has no live forwarding connection"),
4686            )?]);
4687        }
4688
4689        if let Some(error) =
4690            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4691        {
4692            self.observe_route_open_refusal(
4693                ctx,
4694                &caller,
4695                &target_module_id,
4696                "op_not_allowed",
4697                "op_not_allowed",
4698            );
4699            return Ok(vec![error]);
4700        }
4701
4702        let principal =
4703            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4704                Ok(principal) => principal,
4705                Err(error) => {
4706                    self.observe_route_open_refusal(
4707                        ctx,
4708                        &caller,
4709                        &target_module_id,
4710                        "bad_consumer_identity",
4711                        "bad_consumer_identity",
4712                    );
4713                    return Ok(vec![error]);
4714                }
4715            };
4716        timings.principal = principal_label(&principal);
4717
4718        // This is attested, control-plane policy for supervised module origins.
4719        // Keep it before route reservation and out of the opaque forwarding hot
4720        // path: data frames must never acquire a per-frame capability check.
4721        if let Principal::Reserved {
4722            module_id: opening_module_id,
4723        } = &principal
4724        {
4725            if let Some(opening_registration) = self
4726                .registry
4727                .get_module(opening_module_id)
4728                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4729            {
4730                if let Some(capability) =
4731                    denied_capability(&opening_registration.manifest, &registration.manifest)
4732                {
4733                    warn!(
4734                        opening_module_id,
4735                        target_module_id,
4736                        capability,
4737                        "refusing route.open because an attested capability deny edge matches"
4738                    );
4739                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4740                        &frame,
4741                        &target_module_id,
4742                        "capability_deny_edge",
4743                        "capability_forbidden",
4744                        format!(
4745                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4746                        ),
4747                    )?]);
4748                }
4749            }
4750        }
4751
4752        if admission_facts.is_some() {
4753            let carrier_matches = matches!(
4754                &principal,
4755                Principal::Reserved { module_id }
4756                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4757            );
4758            if !carrier_matches {
4759                return Ok(vec![self.route_open_refusal_frame(
4760                    ctx,
4761                    &caller,
4762                    &frame,
4763                    &target_module_id,
4764                    "admission_facts_carrier_not_permitted",
4765                    "admission_facts_not_permitted",
4766                    "admission facts may only be carried by the configured reserved module",
4767                )?]);
4768            }
4769
4770            let target_allowed = self
4771                .admission_facts_targets
4772                .as_ref()
4773                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4774            if !target_allowed {
4775                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4776                    &frame,
4777                    &target_module_id,
4778                    "admission_facts_target_not_listed",
4779                    "admission_facts_target_not_allowed",
4780                    format!(
4781                        "admission facts are not permitted for target module_id '{target_module_id}'"
4782                    ),
4783                )?]);
4784            }
4785
4786            // Keep the value opaque to subc. The downstream admission validator owns
4787            // schema and semantic checks; this daemon only enforces carrier authority
4788            // and the configured destination allowlist.
4789        }
4790
4791        // Scope admission, on the attested principal above and never on the
4792        // request body. The tag read here travels with the pending bind and is
4793        // compared with the published one at commit, so a sync between here
4794        // and the module's ack refuses the open instead of binding a stamp
4795        // that is no longer true.
4796        let (bound_scope, mut scope_stamp) = match &scope {
4797            None => (None, None),
4798            Some(selector) => {
4799                let owner_configured = match &selector.owner {
4800                    // A reserved owner counts as configured from before its
4801                    // process is spawned (see `SupervisorHandle::is_configured`).
4802                    // So an owner that has not synced its scopes yet is refused
4803                    // as retryable (`scope_not_synced`), not as one that will
4804                    // never sync.
4805                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4806                    _ => false,
4807                };
4808                let mut scopes = self
4809                    .scopes
4810                    .write()
4811                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4812                let admitted = scopes
4813                    .admit(&principal, &target_module_id, selector, owner_configured)
4814                    .and_then(|admission| {
4815                        crate::scopes::check_target_flow_support(
4816                            &admission.stamp,
4817                            &target_module_id,
4818                            registration.manifest.capabilities.as_ref(),
4819                        )?;
4820                        crate::scopes::check_target_agent_run_support(
4821                            &admission.stamp,
4822                            &target_module_id,
4823                            registration.manifest.capabilities.as_ref(),
4824                        )?;
4825                        let bound = scopes.reserve_bind(
4826                            &admission,
4827                            &principal,
4828                            ctx.connection_id,
4829                            selector.hold_presence,
4830                        );
4831                        Ok((admission, bound))
4832                    });
4833                match admitted {
4834                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4835                    Err(refusal) => {
4836                        return Ok(vec![self.route_open_refusal_frame(
4837                            ctx,
4838                            &caller,
4839                            &frame,
4840                            &target_module_id,
4841                            refusal.code,
4842                            refusal.code,
4843                            refusal.message,
4844                        )?]);
4845                    }
4846                }
4847            }
4848        };
4849        // From here on a refusal names the scope by the stamp admission
4850        // issued, exactly as the `route.open accepted` line does.
4851        if let Some(stamp) = &scope_stamp {
4852            caller.scope = route_open_scope_label(stamp);
4853        }
4854
4855        // Bind admits a root that no longer exists on disk, because refusing here
4856        // closes the only exit from a paused run: cancel needs a bound route, and a
4857        // renamed or reclaimed directory makes that route unopenable forever. The
4858        // run itself is intact and still addressable by its recorded identity.
4859        //
4860        // This does NOT relax the rule the strict constructor protects. That rule is
4861        // that no root is ever aliased into NEW durable state -- a missing component
4862        // can reappear as a symlink elsewhere, which would move the identity and
4863        // split a session's history across two of them. The engine now refuses the
4864        // two operations that create such state (send and import) at admission,
4865        // which is a narrower way to hold the same invariant: reads and terminations
4866        // are admitted, writes are not. That refusal had to ship before this line
4867        // changed, or there is an interval where a send commits under a provisional
4868        // identity -- the exact failure the original policy existed to prevent.
4869        //
4870        // Resolution follows realpath rather than lexical cleanup: the longest
4871        // existing ancestor is canonicalized and the missing tail re-appended, so a
4872        // live root is unchanged and a vanished leaf keeps the identity it was
4873        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4874        // same caller the moment the directory vanished, which strands the run more
4875        // quietly than refusing it.
4876        timings.enter(RouteOpenPhase::ProjectRoot);
4877        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4878            Ok(project_root) => project_root,
4879            Err(err) => {
4880                return Ok(vec![control_error_frame(
4881                    &frame,
4882                    "invalid_project_root",
4883                    err.to_string(),
4884                )?])
4885            }
4886        };
4887        timings.enter(RouteOpenPhase::Admission);
4888        identity.project_root = project_root.as_path().to_path_buf();
4889
4890        // Last gate before any relay work, and deliberately after the cheap
4891        // registry and availability checks above: those name a more precise
4892        // condition (unknown, removed, reloading) and a caller is better served
4893        // by the precise code than by this one.
4894        //
4895        // Everything below this point costs an egress permit, a reserved handle
4896        // pair and, if the module does not answer, the whole relay budget. The
4897        // reader no longer waits for that budget, so cap each target explicitly;
4898        // serial dispatch used to provide the accidental cap of one relay per
4899        // connection. Admission is a mutex-protected count and never waits.
4900        let _concurrency_guard = match self
4901            .route_bind_concurrency
4902            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4903        {
4904            Ok(guard) => guard,
4905            Err(in_flight) => {
4906                return Ok(vec![self.route_open_target_capacity_refusal(
4907                    ctx,
4908                    &caller,
4909                    &frame,
4910                    &target_module_id,
4911                    in_flight,
4912                )?]);
4913            }
4914        };
4915
4916        // A module that has already burned the whole budget `threshold` times
4917        // in a row does not get to charge it again until a probe says it recovered.
4918        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4919            RouteBindAdmission::Admitted { guard, probe } => {
4920                if probe {
4921                    info!(
4922                        module_id = %target_module_id,
4923                        connection_id = ctx.connection_id.get(),
4924                        "route.bind breaker half-open: admitting one probe"
4925                    );
4926                }
4927                guard
4928            }
4929            RouteBindAdmission::Refused {
4930                consecutive_timeouts,
4931                retry_in,
4932                probe_in_flight,
4933            } => {
4934                return Ok(vec![self.route_open_breaker_refusal_frame(
4935                    ctx,
4936                    &caller,
4937                    &frame,
4938                    &target_module_id,
4939                    consecutive_timeouts,
4940                    retry_in,
4941                    probe_in_flight,
4942                )?]);
4943            }
4944        };
4945
4946        // Resolve the per-module budget here so the wait matches the operator's
4947        // intent for this specific target. A per-module override in
4948        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4949        // daemons) wins over the daemon-wide default.
4950        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4951        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4952        timings.enter(RouteOpenPhase::EgressReservation);
4953        let pending = match self
4954            .forwarding
4955            .begin_route_bind_relay_for(
4956                ctx.connection_id,
4957                ctx.egress.clone(),
4958                response_version(&frame),
4959                frame.header.corr,
4960                &target_module_id,
4961                principal.clone(),
4962                bound_scope,
4963                Some(project_root),
4964                relay_deadline,
4965            )
4966            .await
4967        {
4968            Ok(pending) => pending,
4969            Err(err) => {
4970                return Ok(vec![self.route_open_refusal_frame(
4971                    ctx,
4972                    &caller,
4973                    &frame,
4974                    &target_module_id,
4975                    "relay_reservation_failed",
4976                    forwarding_error_code(&err),
4977                    err.to_string(),
4978                )?])
4979            }
4980        };
4981        timings.enter(RouteOpenPhase::Admission);
4982        let crate::forwarding::PendingRouteBindRelay {
4983            endpoint,
4984            module_sink,
4985            negotiated_ver,
4986            client_channel,
4987            client_epoch,
4988            module_channel,
4989            module_epoch,
4990            corr: relay_corr,
4991            receiver,
4992        } = pending;
4993        let mut reservation =
4994            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4995
4996        // Reserving egress can wait while a module reconnects or a swap cuts
4997        // over. Check the relay's captured connection, not the earlier by-id
4998        // lookup: the original module's flow/run capabilities cannot authorize
4999        // its replacement. The captured sink stays bound to that connection.
5000        let relay_registration = self
5001            .registry
5002            .get_module_by_connection(endpoint.connection_id)
5003            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5004        let relay_capabilities = relay_registration
5005            .as_ref()
5006            .and_then(|registration| registration.manifest.capabilities.as_ref());
5007        if let Some(selector) = &scope {
5008            // Egress reservation may wait across a scope update. Recheck the
5009            // proof against the current record before sending the captured bind.
5010            if let Err(refusal) = self
5011                .scopes
5012                .read()
5013                .unwrap_or_else(|poisoned| poisoned.into_inner())
5014                .check_bind_proof(&principal, selector)
5015            {
5016                reservation.release_and_disarm();
5017                return Ok(vec![self.route_open_refusal_frame(
5018                    ctx,
5019                    &caller,
5020                    &frame,
5021                    &target_module_id,
5022                    refusal.code,
5023                    refusal.code,
5024                    refusal.message,
5025                )?]);
5026            }
5027        }
5028        if let Some(stamp) = scope_stamp
5029            .as_ref()
5030            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5031        {
5032            if let Err(refusal) = crate::scopes::check_target_flow_support(
5033                stamp,
5034                &target_module_id,
5035                relay_capabilities,
5036            )
5037            .and_then(|()| {
5038                crate::scopes::check_target_agent_run_support(
5039                    stamp,
5040                    &target_module_id,
5041                    relay_capabilities,
5042                )
5043            }) {
5044                reservation.release_and_disarm();
5045                return Ok(vec![self.route_open_refusal_frame(
5046                    ctx,
5047                    &caller,
5048                    &frame,
5049                    &target_module_id,
5050                    refusal.code,
5051                    refusal.code,
5052                    refusal.message,
5053                )?]);
5054            }
5055        }
5056
5057        // Older readers may reject unknown fields. Select optional fields from
5058        // the captured endpoint's declaration, including on unscoped binds.
5059        if !relay_capabilities.is_some_and(|caps| {
5060            caps.provides
5061                .iter()
5062                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5063        }) {
5064            if let Some(stamp) = scope_stamp.as_mut() {
5065                stamp.workdir = None;
5066                stamp.project_id = None;
5067            }
5068        }
5069        // A worker run has no agent_id; on_behalf_of_agent is its only agent
5070        // attribution. The run gate above has already required the target to
5071        // declare `worker-runs/v1`, which promises to read both
5072        // `on_behalf_of_agent` and `call_mode`.
5073        let worker_run = scope_stamp
5074            .as_ref()
5075            .is_some_and(crate::scopes::is_worker_run)
5076            && relay_capabilities.is_some_and(|caps| {
5077                caps.provides
5078                    .iter()
5079                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5080            });
5081        if !worker_run
5082            && !relay_capabilities.is_some_and(|caps| {
5083                caps.provides
5084                    .iter()
5085                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5086            })
5087        {
5088            if let Some(stamp) = scope_stamp.as_mut() {
5089                stamp.attributes.on_behalf_of_agent = None;
5090            }
5091        }
5092        if !relay_capabilities.is_some_and(|caps| {
5093            caps.provides
5094                .iter()
5095                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5096        }) {
5097            if let Some(stamp) = scope_stamp.as_mut() {
5098                stamp.attributes.flow_version = None;
5099                stamp.attributes.flow_code_hash = None;
5100                stamp.attributes.flow_install_id = None;
5101            }
5102        }
5103        let call_mode = if worker_run
5104            || relay_capabilities
5105                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5106        {
5107            Some(derive_call_mode(scope_stamp.as_ref()))
5108        } else {
5109            if let Some(stamp) = scope_stamp.as_mut() {
5110                stamp.expires_at_ms = None;
5111            }
5112            None
5113        };
5114
5115        debug!(
5116            connection_id = ctx.connection_id.get(),
5117            client_channel,
5118            client_epoch,
5119            module_channel,
5120            module_epoch,
5121            "reserved route handle pair"
5122        );
5123        // Rendered BEFORE the move into the relay, because the accept arm below
5124        // is where it is logged and the principal is gone by then.
5125        let route_principal_label = principal_label(&principal);
5126        let relay = ModuleControlRequest::RouteBind {
5127            route_channel: module_channel,
5128            epoch: module_epoch,
5129            target,
5130            identity,
5131            principal: Some(principal),
5132            consumer_capabilities,
5133            role_versions,
5134            admission_facts,
5135            scope: scope_stamp,
5136            call_mode,
5137        };
5138        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5139            RouterError::backend(
5140                0,
5141                frame.header.corr,
5142                format!("failed to encode route.bind request: {err}"),
5143            )
5144        })?;
5145        let relay_frame = Frame::build_with_version(
5146            negotiated_ver,
5147            FrameType::Request,
5148            control_flags(),
5149            0,
5150            0,
5151            relay_corr,
5152            relay_body,
5153        )
5154        .map_err(RouterError::FrameBuild)?;
5155
5156        timings.enter(RouteOpenPhase::BindSend);
5157        let sent = module_sink.send(relay_frame).await;
5158        timings.enter(RouteOpenPhase::Admission);
5159        if let Err(err) = sent {
5160            reservation.release_and_disarm();
5161            return Ok(vec![self.route_open_refusal_frame(
5162                ctx,
5163                &caller,
5164                &frame,
5165                &target_module_id,
5166                "relay_send_failed",
5167                "target_unavailable",
5168                err.to_string(),
5169            )?]);
5170        }
5171
5172        #[cfg(test)]
5173        if let Some(hold) = &self.route_bind_relay_sent_hold {
5174            // First wait: tell the test the bind is queued. Second wait: stay
5175            // here until the test has driven whatever it wants to happen
5176            // between queueing the bind and marking it enqueued.
5177            hold.wait().await;
5178            hold.wait().await;
5179        }
5180
5181        // A missing relay is not by itself an abandoned bind: the module may
5182        // already have acked it and the route may be committed. Only a relay
5183        // dropped without any module reply needs this GOODBYE; one a reply
5184        // settled has already had its GOODBYE decided by the reply path.
5185        if self
5186            .forwarding
5187            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5188            .map_err(RouterError::Forwarding)?
5189            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5190        {
5191            self.send_abandoned_route_bind_goodbye(
5192                &module_sink,
5193                negotiated_ver,
5194                module_channel,
5195                module_epoch,
5196            );
5197        }
5198
5199        timings.enter(RouteOpenPhase::BindAckWait);
5200        let outcome = timeout_at(relay_deadline, receiver).await;
5201        timings.enter(RouteOpenPhase::Admission);
5202        match outcome {
5203            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5204                reservation.disarm();
5205                if breaker.record_accepted() {
5206                    info!(
5207                        module_id = %target_module_id,
5208                        "route.bind breaker closed: the probe was accepted"
5209                    );
5210                }
5211                self.observe_route_open_accept(
5212                    ctx,
5213                    &target_module_id,
5214                    &route_principal_label,
5215                    &caller.scope,
5216                );
5217                Ok(Vec::new())
5218            }
5219            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5220                reservation.release_and_disarm();
5221                // A module that says no in microseconds is healthy. Rejection
5222                // is a different condition with its own refusal and must not
5223                // move the breaker.
5224                breaker.record_inconclusive();
5225                // The daemon's own commit re-check refused the bind because the
5226                // scope ended or changed after admission. The module accepted;
5227                // counting it as a module rejection would blame the module.
5228                let scope_code = match body.code.as_str() {
5229                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5230                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5231                    _ => None,
5232                };
5233                if let Some(code) = scope_code {
5234                    self.observe_route_open_refusal(
5235                        ctx,
5236                        &caller,
5237                        &target_module_id,
5238                        "scope_changed_before_commit",
5239                        code,
5240                    );
5241                    return Ok(vec![control_error_body_frame(&frame, body)?]);
5242                }
5243                self.counters
5244                    .increment_route_open_refused("module_rejected");
5245                info!(
5246                    target: "control",
5247                    code = "module_rejected",
5248                    module_code = ?body.code,
5249                    module_id = ?target_module_id,
5250                    connection_id = ctx.connection_id.get(),
5251                    principal = %caller.principal,
5252                    scope = %caller.scope,
5253                    harness = %caller.harness,
5254                    "route.open refused"
5255                );
5256                Ok(vec![control_error_body_frame(&frame, body)?])
5257            }
5258            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5259                reservation.release_and_disarm();
5260                breaker.record_inconclusive();
5261                // Fires when the module's connection closes while a relayed
5262                // bind is pending -- typically a caller racing a module restart
5263                // whose bind was relayed BEFORE the drain mark went up. Logged
5264                // because the caller sees only its own error and the fleet has
5265                // already spent one diagnosis round unable to tell this arm
5266                // from a relay timeout without daemon-side evidence.
5267                tracing::warn!(
5268                    module_id = %target_module_id,
5269                    "route.bind relay abandoned: {message}"
5270                );
5271                Ok(vec![self.route_open_refusal_frame(
5272                    ctx,
5273                    &caller,
5274                    &frame,
5275                    &target_module_id,
5276                    "relay_abandoned",
5277                    "target_unavailable",
5278                    message,
5279                )?])
5280            }
5281            Ok(Err(_)) => {
5282                reservation.release_and_disarm();
5283                breaker.record_inconclusive();
5284                Ok(vec![self.route_open_refusal_frame(
5285                    ctx,
5286                    &caller,
5287                    &frame,
5288                    &target_module_id,
5289                    "relay_waiter_canceled",
5290                    "target_unavailable",
5291                    "route.bind relay waiter was canceled before the module responded",
5292                )?])
5293            }
5294            Err(_) => {
5295                reservation.release_and_disarm();
5296                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5297                // answer at all is the one condition a fast refusal can
5298                // usefully stand in for; every other arm already answered.
5299                if let Some(opened) = breaker.record_timeout(
5300                    self.route_bind_breaker_threshold,
5301                    self.route_bind_breaker_cooldown,
5302                ) {
5303                    warn!(
5304                        module_id = %target_module_id,
5305                        consecutive_timeouts = opened.consecutive_timeouts,
5306                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5307                        reopened_after_probe = opened.reopened_after_probe,
5308                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5309                    );
5310                }
5311                // The generous budget just burned to no answer: the module is
5312                // registered and its connection is up, but its bind handler sat
5313                // on the ack for the full budget (warm-on-bind, cold configure,
5314                // or a wedged handler). Every earlier unavailability shape
5315                // fast-refuses BEFORE the relay, so this arm firing means the
5316                // slowness is module-side -- log it so the per-module timeline
5317                // is reconstructable without client audit rows.
5318                tracing::warn!(
5319                    module_id = %target_module_id,
5320                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5321                    "route.bind relay timed out: module did not ack within budget"
5322                );
5323                Ok(vec![self.route_open_refusal_frame(
5324                    ctx,
5325                    &caller,
5326                    &frame,
5327                    &target_module_id,
5328                    "relay_timed_out",
5329                    "module_timeout",
5330                    format!(
5331                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5332                        route_bind_relay_timeout
5333                    ),
5334                )?])
5335            }
5336        }
5337    }
5338
5339    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5340        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5341            snapshot: self.supervisor.spawn_snapshot(),
5342        };
5343        Ok(vec![control_response_body_frame(
5344            &frame,
5345            &response,
5346            "ClientControlResponse::SupervisorSpawnSnapshot",
5347        )?])
5348    }
5349
5350    fn handle_supervisor_spawn_subscribe(
5351        &self,
5352        ctx: &RouteCtx,
5353        frame: Frame,
5354        since: Option<SpawnCursor>,
5355    ) -> Result<Vec<Frame>, RouterError> {
5356        match self.supervisor.subscribe_spawns(
5357            ctx.connection_id,
5358            frame.header.corr,
5359            response_version(&frame),
5360            since,
5361            ctx.egress.clone(),
5362        ) {
5363            Ok(()) => Ok(Vec::new()),
5364            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5365                Ok(vec![control_error_body_frame(
5366                    &frame,
5367                    ErrorBody {
5368                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5369                        message: "spawn cursor belongs to a different daemon incarnation"
5370                            .to_string(),
5371                        detail: Some(serde_json::json!({
5372                            "current_daemon_incarnation": current
5373                        })),
5374                    },
5375                )?])
5376            }
5377            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5378                &frame,
5379                ErrorBody {
5380                    code: "spawn_cursor_too_old".to_string(),
5381                    message: "spawn cursor predates the retained event ring".to_string(),
5382                    detail: Some(serde_json::json!({
5383                        "oldest_retained_cursor": oldest
5384                    })),
5385                },
5386            )?]),
5387            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5388                0,
5389                frame.header.corr,
5390                format!("failed to open supervisor spawn subscription: {error}"),
5391            )),
5392        }
5393    }
5394
5395    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5396        let generation = self
5397            .registry
5398            .generation()
5399            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5400        let mut modules = Vec::new();
5401        for module in self.supervisor.list() {
5402            let status = module.status_for_control("list").map_err(|err| {
5403                RouterError::backend(
5404                    0,
5405                    frame.header.corr,
5406                    format!("failed to read supervisor status: {err}"),
5407                )
5408            })?;
5409            let (configured, _) = module.configuration().map_err(|err| {
5410                RouterError::backend(
5411                    0,
5412                    frame.header.corr,
5413                    format!("failed to read module configuration: {err}"),
5414                )
5415            })?;
5416            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5417                RouterError::backend(
5418                    0,
5419                    frame.header.corr,
5420                    format!("failed to read retry time: {err}"),
5421                )
5422            })?;
5423            // Status and configuration snapshots release their locks before the image probe awaits.
5424            let image = module.running_image_agreement().await;
5425            // Read per request so the figure is current when the operator asks;
5426            // the daemon samples nothing in between.
5427            let resources = Some(module.child_resource_usage());
5428            let mut verdict =
5429                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5430            verdict.spec = Some(launch_spec_agreement(
5431                &configured,
5432                status.process_alive,
5433                status.launch_spec_digest.as_ref(),
5434            ));
5435            let pending_reload = Some(verdict);
5436            let configured_enabled = self
5437                .capability_evaluator
5438                .configured_enabled(&status.module_id);
5439            // Keep the retired policy field on the wire for one release so
5440            // existing status consumers still receive the platform policy.
5441            modules.push(
5442                SupervisorEntry::new(
5443                    status.module_id,
5444                    status.state.to_string(),
5445                    status.enabled,
5446                    status.live,
5447                    status.health.status,
5448                )
5449                .with_configured_enabled(configured_enabled)
5450                .with_next_retry_at_ms(next_retry_at_ms)
5451                .with_launch_nonce_env(Some(!cfg!(unix)))
5452                .with_launch_nonce_source(status.launch_nonce_source)
5453                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5454                .with_protocol(status.protocol)
5455                .with_pending_reload(pending_reload)
5456                .with_last_probe_ms(status.health.last_probe_ms)
5457                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5458                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5459                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5460                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5461                .with_restart_count(Some(status.restart_count))
5462                .with_max_restarts(Some(status.max_restarts))
5463                .with_lifetime_restarts(Some(status.lifetime_restarts))
5464                .with_spawn_generation(Some(status.spawn_generation))
5465                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5466                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5467                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5468                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5469                .with_resources(resources),
5470            );
5471        }
5472        let response = ClientControlResponse::SupervisorList {
5473            generation,
5474            modules,
5475        };
5476        Ok(vec![control_response_body_frame(
5477            &frame,
5478            &response,
5479            "ClientControlResponse::SupervisorList",
5480        )?])
5481    }
5482
5483    fn handle_supervisor_stderr_tail(
5484        &self,
5485        frame: Frame,
5486        module_id: String,
5487        max_lines: Option<u32>,
5488        max_bytes: Option<u32>,
5489    ) -> Result<Vec<Frame>, RouterError> {
5490        let Some(module) = self.supervisor.get(&module_id) else {
5491            return Ok(vec![control_error_frame(
5492                &frame,
5493                "unknown_module",
5494                format!("module_id '{module_id}' is not supervised"),
5495            )?]);
5496        };
5497
5498        let snapshot = module.stderr_tail(
5499            max_lines.map(|value| value as usize),
5500            max_bytes.map(|value| value as usize),
5501        );
5502
5503        let response = ClientControlResponse::SupervisorStderrTail {
5504            module_id,
5505            tail: StderrTail {
5506                capture: match snapshot.capture {
5507                    CaptureState::Captured => StderrCaptureState::Captured,
5508                    CaptureState::Incomplete { reason } => {
5509                        StderrCaptureState::Incomplete { reason }
5510                    }
5511                    CaptureState::NotCaptured { reason } => {
5512                        StderrCaptureState::NotCaptured { reason }
5513                    }
5514                },
5515                entries: snapshot
5516                    .entries
5517                    .into_iter()
5518                    .map(|entry| match entry {
5519                        TailEntry::Line {
5520                            text,
5521                            truncated,
5522                            at_ms,
5523                        } => StderrTailEntry::Line {
5524                            text,
5525                            truncated,
5526                            at_ms,
5527                        },
5528                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5529                    })
5530                    .collect(),
5531                dropped_lines: snapshot.dropped_lines,
5532            },
5533        };
5534        Ok(vec![control_response_body_frame(
5535            &frame,
5536            &response,
5537            "ClientControlResponse::SupervisorStderrTail",
5538        )?])
5539    }
5540
5541    async fn handle_supervisor_terminals(
5542        &self,
5543        frame: Frame,
5544        module_id: String,
5545    ) -> Result<Vec<Frame>, RouterError> {
5546        let Some(module) = self.supervisor.get(&module_id) else {
5547            return Ok(vec![control_error_frame(
5548                &frame,
5549                "unknown_module",
5550                format!("module_id '{module_id}' is not supervised"),
5551            )?]);
5552        };
5553
5554        // The journal read runs on a blocking thread: it can be megabytes of
5555        // file I/O and must not occupy a runtime worker.
5556        let terminals = module
5557            .read_durable_terminal_history()
5558            .await
5559            .map_err(|error| {
5560                RouterError::backend(
5561                    0,
5562                    frame.header.corr,
5563                    format!("failed to read terminal history: {error}"),
5564                )
5565            })?;
5566        let response = ClientControlResponse::SupervisorTerminals {
5567            module_id,
5568            terminals,
5569        };
5570        Ok(vec![control_response_body_frame(
5571            &frame,
5572            &response,
5573            "ClientControlResponse::SupervisorTerminals",
5574        )?])
5575    }
5576
5577    fn handle_supervisor_routes(
5578        &self,
5579        frame: Frame,
5580        module_id: Option<String>,
5581    ) -> Result<Vec<Frame>, RouterError> {
5582        let modules = self
5583            .forwarding
5584            .route_census(module_id.as_deref())
5585            .map_err(RouterError::Forwarding)?
5586            .into_iter()
5587            .map(|(module_id, routes)| SupervisorRouteModule {
5588                module_id,
5589                routes: routes
5590                    .into_iter()
5591                    .map(|route| SupervisorRoute {
5592                        consumer: match route.principal {
5593                            Principal::Reserved { module_id } => {
5594                                SupervisorRouteConsumer::Reserved { module_id }
5595                            }
5596                            Principal::Direct | Principal::Unverified => {
5597                                SupervisorRouteConsumer::Direct {
5598                                    connection_id: route.goodbye_target.connection_id.get(),
5599                                }
5600                            }
5601                        },
5602                        age_ms: Instant::now()
5603                            .saturating_duration_since(route.bound_at)
5604                            .as_millis()
5605                            .try_into()
5606                            .unwrap_or(u64::MAX),
5607                        draining: route.draining,
5608                        drain_reason: route.drain_reason,
5609                    })
5610                    .collect(),
5611            })
5612            .collect();
5613        let response = ClientControlResponse::SupervisorRoutes { modules };
5614        Ok(vec![control_response_body_frame(
5615            &frame,
5616            &response,
5617            "ClientControlResponse::SupervisorRoutes",
5618        )?])
5619    }
5620
5621    async fn handle_supervisor_provenance(
5622        &self,
5623        frame: Frame,
5624        module_id: Option<String>,
5625    ) -> Result<Vec<Frame>, RouterError> {
5626        let mut selected = if let Some(module_id) = module_id {
5627            let Some(module) = self.supervisor.get(&module_id) else {
5628                return Ok(vec![control_error_frame(
5629                    &frame,
5630                    "unknown_module",
5631                    format!("module_id '{module_id}' is not supervised"),
5632                )?]);
5633            };
5634            vec![module]
5635        } else {
5636            self.supervisor.list()
5637        };
5638
5639        let mut modules = Vec::with_capacity(selected.len());
5640        for module in selected.drain(..) {
5641            let (status, observed_image) = module
5642                .status_and_running_image_agreement()
5643                .await
5644                .map_err(|err| {
5645                    RouterError::backend(
5646                        0,
5647                        frame.header.corr,
5648                        format!("failed to read supervisor status: {err}"),
5649                    )
5650                })?;
5651            let module_declared = self
5652                .registry
5653                .get_module(&status.module_id)
5654                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5655                .and_then(|registration| registration.manifest.provenance)
5656                .map(|build| ModuleDeclaredProvenance::Reported { build })
5657                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5658            #[cfg(test)]
5659            let running_image = match &self.provenance_probe_override {
5660                Some(result) => result.clone(),
5661                None => observed_image,
5662            };
5663            #[cfg(not(test))]
5664            let running_image = observed_image;
5665            modules.push(SupervisorModuleProvenance {
5666                module_id: status.module_id,
5667                module_declared,
5668                daemon_observed: SupervisorObservedProcess {
5669                    launch_nonce_source: status.launch_nonce_source,
5670                    pid: status.pid,
5671                    spawned_at_ms: status.spawned_at_ms,
5672                    spawned_from: status.spawned_from,
5673                    running_image,
5674                },
5675            });
5676        }
5677        let daemon = SupervisorDaemonProvenance {
5678            daemon_build: self.daemon_provenance.build.clone(),
5679            daemon_observed: DaemonObservedProcess {
5680                pid: self.daemon_provenance.pid,
5681                started_at_ms: self
5682                    .daemon_provenance
5683                    .start_clock
5684                    .map(|clock| clock.started_at_ms())
5685                    .or(self.daemon_provenance.started_at_ms),
5686                running_image: self
5687                    .daemon_provenance
5688                    .probe
5689                    .observe(
5690                        self.daemon_provenance.pid,
5691                        self.daemon_provenance.executable_path.as_deref(),
5692                        self.daemon_provenance.executable_identity.clone(),
5693                        self.daemon_provenance.process_start_time,
5694                    )
5695                    .await,
5696            },
5697        };
5698        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5699        Ok(vec![control_response_body_frame(
5700            &frame,
5701            &response,
5702            "ClientControlResponse::SupervisorProvenance",
5703        )?])
5704    }
5705
5706    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5707        self.refresh_capability_requirements();
5708        let generation = self
5709            .registry
5710            .generation()
5711            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5712        let modules = self
5713            .supervisor
5714            .list()
5715            .into_iter()
5716            .map(|module| {
5717                let status = module.status_for_control("health").map_err(|err| {
5718                    RouterError::backend(
5719                        0,
5720                        frame.header.corr,
5721                        format!("failed to read supervisor health: {err}"),
5722                    )
5723                })?;
5724                let module_id = status.module_id;
5725                let capability_detail = self
5726                    .capability_evaluator
5727                    .required_problem_detail(&module_id);
5728                Ok(SupervisorHealthEntry {
5729                    module_id,
5730                    status: status.health.status,
5731                    detail: append_capability_problem_detail(
5732                        status.health.detail,
5733                        capability_detail,
5734                    ),
5735                    metrics: status.health.metrics,
5736                    consecutive_failures: status.health.consecutive_failures,
5737                    late_answer_count: status.health.late_answer_count,
5738                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5739                    last_action: status.health.last_action,
5740                    last_action_ms: status.health.last_action_ms,
5741                    last_probe_ms: status.health.last_probe_ms,
5742                })
5743            })
5744            .collect::<Result<Vec<_>, RouterError>>()?;
5745        let response = ClientControlResponse::SupervisorHealth {
5746            generation,
5747            modules,
5748        };
5749        Ok(vec![control_response_body_frame(
5750            &frame,
5751            &response,
5752            "ClientControlResponse::SupervisorHealth",
5753        )?])
5754    }
5755
5756    async fn handle_supervisor_restart(
5757        &self,
5758        frame: Frame,
5759        module_id: String,
5760        drain_timeout_ms: Option<u64>,
5761        requester: Principal,
5762    ) -> Result<Vec<Frame>, RouterError> {
5763        let operation_lock = self.supervisor.operation_lock();
5764        let _operation_guard = operation_lock.lock().await;
5765        let Some(module) = self.supervisor.get(&module_id) else {
5766            return Ok(vec![control_error_frame(
5767                &frame,
5768                "unknown_module",
5769                format!("module_id '{module_id}' is not supervised"),
5770            )?]);
5771        };
5772
5773        self.route_outages.mark_operator_action(&module_id);
5774        if let Err(err) = module
5775            .restart_requested(drain_timeout_ms, Some(requester))
5776            .await
5777        {
5778            self.route_outages
5779                .operator_action_ended_unrefused(&module_id);
5780            let (code, message) = match err {
5781                crate::supervise::SuperviseError::Disabled { .. } => {
5782                    ("module_disabled", err.to_string())
5783                }
5784                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5785                    ("swap_in_progress", err.to_string())
5786                }
5787                _ => (
5788                    "target_unavailable",
5789                    format!("failed to restart module_id '{module_id}': {err}"),
5790                ),
5791            };
5792            return Ok(vec![control_error_frame(&frame, code, message)?]);
5793        }
5794
5795        let response = ClientControlResponse::SupervisorAck {
5796            module_id,
5797            applied: true,
5798        };
5799        Ok(vec![control_response_body_frame(
5800            &frame,
5801            &response,
5802            "ClientControlResponse::SupervisorAck",
5803        )?])
5804    }
5805
5806    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5807    /// when the old process has finished draining: a caller whose own lane
5808    /// rides the old process must get its reply before that drain waits on it.
5809    async fn handle_supervisor_swap(
5810        &self,
5811        frame: Frame,
5812        module_id: String,
5813        ready_timeout_ms: Option<u64>,
5814        requester: Principal,
5815    ) -> Result<Vec<Frame>, RouterError> {
5816        // The daemon-wide operation lock is held only to resolve the handle,
5817        // not across the swap. The swap can take its whole readiness budget,
5818        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5819        // holding it here would park an operator's stop behind the swap it is
5820        // meant to abort. A rescan or stop that reaches the module during the
5821        // swap is served by the swap itself (see `supervise_swap`).
5822        let module = {
5823            let operation_lock = self.supervisor.operation_lock();
5824            let _operation_guard = operation_lock.lock().await;
5825            self.supervisor.get(&module_id)
5826        };
5827        let Some(module) = module else {
5828            return Ok(vec![control_error_frame(
5829                &frame,
5830                "unknown_module",
5831                format!("module_id '{module_id}' is not supervised"),
5832            )?]);
5833        };
5834
5835        self.route_outages.mark_operator_action(&module_id);
5836        if let Err(err) = module
5837            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5838            .await
5839        {
5840            self.route_outages
5841                .operator_action_ended_unrefused(&module_id);
5842            use crate::supervise::SuperviseError;
5843            let message = err.to_string();
5844            let error = match err {
5845                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5846                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5847                    code: "swap_refused".to_string(),
5848                    message,
5849                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5850                },
5851                SuperviseError::SwapFailed {
5852                    arm,
5853                    candidate_exit,
5854                    ..
5855                } => ErrorBody {
5856                    code: "swap_failed".to_string(),
5857                    message,
5858                    detail: Some(serde_json::json!({
5859                        "arm": arm.as_str(),
5860                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5861                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5862                    })),
5863                },
5864                _ => ErrorBody::new(
5865                    "target_unavailable",
5866                    format!("failed to swap module_id '{module_id}': {message}"),
5867                ),
5868            };
5869            return Ok(vec![control_error_body_frame(&frame, error)?]);
5870        }
5871        // A completed swap kept the incumbent serving until cutover, so it
5872        // usually opened no outage; a mark left behind would make the next,
5873        // unrelated outage read as requested.
5874        self.route_outages
5875            .operator_action_ended_unrefused(&module_id);
5876
5877        let response = ClientControlResponse::SupervisorAck {
5878            module_id,
5879            applied: true,
5880        };
5881        Ok(vec![control_response_body_frame(
5882            &frame,
5883            &response,
5884            "ClientControlResponse::SupervisorAck",
5885        )?])
5886    }
5887
5888    async fn handle_supervisor_reload(
5889        &self,
5890        frame: Frame,
5891        module_id: String,
5892        requester: Principal,
5893    ) -> Result<Vec<Frame>, RouterError> {
5894        let operation_lock = self.supervisor.operation_lock();
5895        let _operation_guard = operation_lock.lock().await;
5896        let Some(module) = self.supervisor.get(&module_id) else {
5897            return Ok(vec![control_error_frame(
5898                &frame,
5899                "unknown_module",
5900                format!("module_id '{module_id}' is not supervised"),
5901            )?]);
5902        };
5903
5904        self.route_outages.mark_operator_action(&module_id);
5905        if let Err(err) = module.reload_requested(Some(requester)).await {
5906            self.route_outages
5907                .operator_action_ended_unrefused(&module_id);
5908            let (code, message) = match err {
5909                crate::supervise::SuperviseError::Disabled { .. } => {
5910                    ("module_disabled", err.to_string())
5911                }
5912                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5913                    ("swap_in_progress", err.to_string())
5914                }
5915                _ => (
5916                    "reload_failed",
5917                    format!("failed to reload module_id '{module_id}': {err}"),
5918                ),
5919            };
5920            return Ok(vec![control_error_frame(&frame, code, message)?]);
5921        }
5922
5923        let response = ClientControlResponse::SupervisorAck {
5924            module_id,
5925            applied: true,
5926        };
5927        Ok(vec![control_response_body_frame(
5928            &frame,
5929            &response,
5930            "ClientControlResponse::SupervisorAck",
5931        )?])
5932    }
5933
5934    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5935    /// the configuration comes from and whether to apply it; every mode then
5936    /// runs the same comparison, so a preview describes the apply it previews.
5937    async fn handle_supervisor_rescan(
5938        &self,
5939        frame: Frame,
5940        mode: RescanMode,
5941        requester: Principal,
5942    ) -> Result<Vec<Frame>, RouterError> {
5943        let preview = mode.previews();
5944        let Some(context) = self.rescan.clone() else {
5945            return Ok(vec![control_error_frame(
5946                &frame,
5947                "rescan_unavailable",
5948                "the daemon was not started with a reloadable config path".to_string(),
5949            )?]);
5950        };
5951
5952        let operation_lock = self.supervisor.operation_lock();
5953        let _operation_guard = operation_lock.lock().await;
5954        // Loaded under the operation lock, so no other rescan can change the
5955        // staged configuration between a checked apply's comparison and the
5956        // apply itself.
5957        let loaded = match self.load_rescan_config(&mode, &context) {
5958            Ok(config) => config,
5959            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5960        };
5961        // `load` reports a missing file as Ok(None), which is correct at boot
5962        // (no config, nothing to supervise) and catastrophic here: rescan treats
5963        // "not in the config" as "remove it", so an absent file would read as an
5964        // empty module list and retire the entire running fleet. An editor
5965        // writing via write-new-then-rename, or a half-finished edit, is enough
5966        // to open that window. Refuse instead: a config that cannot be read
5967        // carries no instruction to remove anything.
5968        let Some((config, activity_config)) = loaded else {
5969            return Ok(vec![control_error_frame(
5970                &frame,
5971                "invalid_daemon_config",
5972                format!(
5973                    "daemon config not found at {}; refusing to rescan (an absent config would \
5974                     retire every supervised module)",
5975                    context.config_path.display()
5976                ),
5977            )?]);
5978        };
5979        let (
5980            configured_port,
5981            storage_config,
5982            admission_facts_carrier_module_id,
5983            admission_facts_targets,
5984            scope_authority_owners,
5985            privileged_operator_prompt,
5986            modules,
5987            reserved_capabilities,
5988        ) = (
5989            config.port,
5990            config.storage,
5991            config.admission_facts_carrier_module_id,
5992            config.admission_facts_targets,
5993            config.scope_authority_owners,
5994            config.privileged_operator_prompt,
5995            config.modules,
5996            config.reserved_capabilities,
5997        );
5998
5999        // Collect the sections rescan cannot apply, so the REPLY carries them.
6000        //
6001        // The warning below has always been correct and has always gone only to
6002        // the journal -- addressed to whoever reads logs, while the person who
6003        // just edited the config is looking at the CLI. Naming each section
6004        // individually rather than setting a flag: "something outside modules
6005        // changed" sends the operator back to diffing their own file, which is
6006        // the work this is meant to save.
6007        let mut restart_required = Vec::new();
6008        for section in RestartRequiredSection::ALL {
6009            let changed = match section {
6010                RestartRequiredSection::Port => configured_port != context.configured_port,
6011                RestartRequiredSection::Storage => storage_config != context.storage_config,
6012                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6013                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6014                }
6015                RestartRequiredSection::AdmissionFactsTargets => {
6016                    admission_facts_targets != context.admission_facts_targets
6017                }
6018                RestartRequiredSection::ScopeAuthorityOwners => {
6019                    scope_authority_owners != context.scope_authority_owners
6020                }
6021                RestartRequiredSection::PrivilegedOperatorPrompt => {
6022                    privileged_operator_prompt != context.privileged_operator_prompt
6023                }
6024            };
6025            if changed {
6026                restart_required.push(section.label().to_string());
6027            }
6028        }
6029        if !restart_required.is_empty() {
6030            warn!(
6031                config_path = %context.config_path.display(),
6032                sections = %restart_required.join(", "),
6033                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6034            );
6035        }
6036
6037        for configured in &modules {
6038            if let Err(err) = validate_spec(&configured.module_spec()) {
6039                return Ok(vec![control_error_frame(
6040                    &frame,
6041                    "invalid_daemon_config",
6042                    format!("supervisor rescan rejected daemon config: {err}"),
6043                )?]);
6044            }
6045        }
6046
6047        let configured_capabilities = modules
6048            .iter()
6049            .map(|module| (module.module_id.clone(), module.enabled))
6050            .collect::<Vec<_>>();
6051        let preview_capability_warnings = if preview {
6052            let (_, registrations) = self.runtime_capability_snapshot()?;
6053            let current_modules = self
6054                .supervisor
6055                .list()
6056                .into_iter()
6057                .map(|module| module.module_id().to_string())
6058                .collect::<BTreeSet<_>>();
6059            let resulting_modules = configured_capabilities.clone();
6060            let removed = current_modules
6061                .into_iter()
6062                .filter(|module_id| {
6063                    !resulting_modules
6064                        .iter()
6065                        .any(|(configured_id, _)| configured_id == module_id)
6066                })
6067                .collect::<Vec<_>>();
6068            // A candidate preview also names providers the candidate disables.
6069            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6070            let disabled = match &mode {
6071                RescanMode::Candidate(_) => resulting_modules
6072                    .iter()
6073                    .filter(|(_, enabled)| !enabled)
6074                    .map(|(module_id, _)| module_id.clone())
6075                    .collect::<Vec<_>>(),
6076                _ => Vec::new(),
6077            };
6078            self.capability_evaluator.preview_warnings(
6079                resulting_modules,
6080                &removed,
6081                &disabled,
6082                &registrations,
6083            )
6084        } else {
6085            Vec::new()
6086        };
6087        let reconciled = match &mode {
6088            RescanMode::Candidate(candidate) => self
6089                .preview_candidate(&context, modules, candidate)
6090                .map(|result| (result, Vec::new())),
6091            _ => {
6092                self.reconcile_supervised_modules(
6093                    &context.supervisor,
6094                    modules,
6095                    preview,
6096                    requester.clone(),
6097                )
6098                .await
6099            }
6100        };
6101        let (result, changed) = match reconciled {
6102            Ok(result) => result,
6103            Err(message) => {
6104                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6105            }
6106        };
6107        if !preview {
6108            if let Some(ledger) = &self.activity {
6109                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6110                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6111            }
6112            if let Err(code) = self.configure_activity(activity_config).await {
6113                warn!(
6114                    code,
6115                    "activity retention could not be applied during rescan"
6116                );
6117            }
6118            self.capability_evaluator
6119                .configure(configured_capabilities, reserved_capabilities);
6120            self.capability_evaluator.wake_deadline_loop();
6121            self.refresh_capability_requirements();
6122        }
6123        let mut result = result;
6124        result.restart_required = restart_required;
6125        result.capability_warnings = preview_capability_warnings;
6126        let response = ClientControlResponse::SupervisorRescan { result };
6127        Ok(vec![control_response_body_frame(
6128            &frame,
6129            &response,
6130            "ClientControlResponse::SupervisorRescan",
6131        )?])
6132    }
6133
6134    async fn handle_supervisor_release_reserved(
6135        &self,
6136        frame: Frame,
6137        module_id: String,
6138    ) -> Result<Vec<Frame>, RouterError> {
6139        let Some(context) = self.rescan.clone() else {
6140            return Ok(vec![control_error_frame(
6141                &frame,
6142                "release_unavailable",
6143                "reserved-id release requires a daemon started with a reloadable config path",
6144            )?]);
6145        };
6146        let operation_lock = self.supervisor.operation_lock();
6147        let _operation_guard = operation_lock.lock().await;
6148        let loaded = match crate::daemon_config::load(&context.config_path) {
6149            Ok(Some(config)) => config,
6150            Ok(None) => {
6151                return Ok(vec![control_error_frame(
6152                    &frame,
6153                    "invalid_daemon_config",
6154                    format!(
6155                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6156                        context.config_path.display()
6157                    ),
6158                )?])
6159            }
6160            Err(err) => {
6161                return Ok(vec![control_error_frame(
6162                    &frame,
6163                    "invalid_daemon_config",
6164                    format!("unable to verify reserved-id release against daemon config: {err}"),
6165                )?])
6166            }
6167        };
6168        if loaded
6169            .modules
6170            .iter()
6171            .any(|configured| configured.module_id == module_id)
6172        {
6173            return Ok(vec![control_error_frame(
6174                &frame,
6175                "reserved_module_configured",
6176                format!(
6177                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6178                ),
6179            )?]);
6180        }
6181        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6182            return Ok(vec![control_error_frame(
6183                &frame,
6184                "reserved_gate_not_retained",
6185                format!(
6186                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6187                ),
6188            )?]);
6189        }
6190
6191        let response = ClientControlResponse::SupervisorAck {
6192            module_id,
6193            applied: true,
6194        };
6195        Ok(vec![control_response_body_frame(
6196            &frame,
6197            &response,
6198            "ClientControlResponse::SupervisorAck",
6199        )?])
6200    }
6201
6202    /// Compare the configured module set with the supervised one, changing
6203    /// nothing. The staged spec each supervised module holds is what the
6204    /// configured entry is compared with.
6205    fn plan_rescan(
6206        &self,
6207        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6208    ) -> Result<RescanPlan, String> {
6209        let mut current = BTreeMap::new();
6210        for module in self.supervisor.list() {
6211            let (spec, health) = module.configuration().map_err(|err| {
6212                format!(
6213                    "failed to read configuration for module_id '{}': {err}",
6214                    module.module_id()
6215                )
6216            })?;
6217            let enabled = module
6218                .status()
6219                .map_err(|err| {
6220                    format!(
6221                        "failed to read status for module_id '{}': {err}",
6222                        module.module_id()
6223                    )
6224                })?
6225                .enabled;
6226            current.insert(
6227                module.module_id().to_string(),
6228                (module, spec, health, enabled),
6229            );
6230        }
6231        let configured = configured_modules
6232            .into_iter()
6233            .map(|module| (module.module_id.clone(), module))
6234            .collect::<BTreeMap<_, _>>();
6235
6236        let added = configured
6237            .keys()
6238            .filter(|module_id| !current.contains_key(*module_id))
6239            .cloned()
6240            .collect::<Vec<_>>();
6241        let removed = current
6242            .keys()
6243            .filter(|module_id| !configured.contains_key(*module_id))
6244            .cloned()
6245            .collect::<Vec<_>>();
6246        let mut changed_pending_reload = Vec::new();
6247        let mut configuration_changes = BTreeSet::new();
6248        let mut enabled_changes = BTreeSet::new();
6249        let mut unchanged = 0_u32;
6250
6251        for (module_id, configured_module) in &configured {
6252            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6253            else {
6254                continue;
6255            };
6256            // Compare the whole launch spec so a future launch field cannot
6257            // accidentally become a live-only policy change. Health is stored
6258            // separately and applies live without replacing the process.
6259            let launch_changed = *current_spec != configured_module.module_spec();
6260            let configuration_changed =
6261                launch_changed || *current_health != configured_module.health;
6262            let enabled_changed = *current_enabled != configured_module.enabled;
6263            if configuration_changed {
6264                configuration_changes.insert(module_id.clone());
6265            }
6266            if launch_changed {
6267                changed_pending_reload.push(module_id.clone());
6268            }
6269            if enabled_changed {
6270                enabled_changes.insert(module_id.clone());
6271            }
6272            if !configuration_changed && !enabled_changed {
6273                unchanged = unchanged.saturating_add(1);
6274            }
6275        }
6276        Ok(RescanPlan {
6277            current,
6278            configured,
6279            added,
6280            removed,
6281            changed_pending_reload,
6282            configuration_changes,
6283            enabled_changes,
6284            unchanged,
6285        })
6286    }
6287
6288    /// Reconcile the running module set against the configured one.
6289    ///
6290    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6291    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6292    /// deliberately shares this function with the executing path rather than
6293    /// computing the same diff somewhere else -- two implementations of one
6294    /// decision agree until they do not, and the whole value of a preview is that
6295    /// it describes the operation that will actually run.
6296    async fn reconcile_supervised_modules(
6297        &self,
6298        supervisor: &Supervisor,
6299        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6300        preview: bool,
6301        requester: Principal,
6302    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6303        let plan = self.plan_rescan(configured_modules)?;
6304
6305        // Everything above this point is pure computation over two snapshots.
6306        // Everything below MUTATES. The preview returns here so the boundary is a
6307        // single early return rather than a condition repeated at each mutation
6308        // site, where one missed guard would apply part of a change the caller was
6309        // told would not happen.
6310        if preview {
6311            return Ok((plan.preview_result(), Vec::new()));
6312        }
6313        let changed = plan
6314            .configuration_changes
6315            .union(&plan.enabled_changes)
6316            .cloned()
6317            .collect::<Vec<_>>();
6318        let RescanPlan {
6319            current,
6320            configured,
6321            added,
6322            removed,
6323            changed_pending_reload,
6324            configuration_changes,
6325            enabled_changes,
6326            unchanged,
6327        } = plan;
6328        // The staged generation moves before the first change rather than
6329        // after the last, so a rescan that fails partway still invalidates
6330        // every earlier `supervisor.rescan_checked` preview: part of its diff
6331        // may have run.
6332        if !(added.is_empty()
6333            && removed.is_empty()
6334            && configuration_changes.is_empty()
6335            && enabled_changes.is_empty())
6336        {
6337            if let Some(context) = &self.rescan {
6338                context
6339                    .generation
6340                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6341            }
6342        }
6343
6344        for module_id in &removed {
6345            let module = &current
6346                .get(module_id)
6347                .expect("removed module came from current supervisor state")
6348                .0;
6349            module
6350                .retire_requested(Some(requester.clone()))
6351                .await
6352                .map_err(|err| {
6353                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6354                })?;
6355            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6356            //
6357            // `handle_route_open` resolves an absent module in three steps:
6358            // registry, then supervisor status, then tombstone. Retiring first
6359            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6360            // went with the teardown above, the supervisor entry went with
6361            // `retire`, and the tombstone does not exist yet -- so a route.open
6362            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6363            // it") for a module that was deliberately removed and whose caller
6364            // should get `module_removed` (TERMINAL, carrying a removal age).
6365            //
6366            // Writing the tombstone first closes it: during the window the
6367            // supervisor entry still answers, so the caller gets
6368            // `target_unavailable` -- retryable, and TRUE, because the module
6369            // is mid-teardown. After both statements it is `module_removed`.
6370            // No instant remains where a removed module reads as one that
6371            // never existed.
6372            //
6373            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6374            // the absence of a test beside a fix invites deletion: these are
6375            // two sync statements with no await between them, so reaching the
6376            // window needs a second worker thread to land exactly between them
6377            // and there is no hook to force it. MEASURED: the 25 daemon_config
6378            // tests pass identically with the old order and the new one, so
6379            // the existing suite cannot see this and a green run is not
6380            // evidence either way. What the suite does hold is the
6381            // post-condition -- a removed module answers `module_removed` --
6382            // which this preserves.
6383            //
6384            // Found by an Athena panel reading the shipped tree against a
6385            // design note (2026-09-19), as the one concrete instance of that
6386            // note's class that survived contact with source. Direction is
6387            // benign: retryable where terminal was intended, never the reverse.
6388            self.supervisor.record_rescan_removal(module_id);
6389            self.supervisor.retire(module_id);
6390            self.route_outages.forget(module_id);
6391        }
6392
6393        for module_id in configured.keys() {
6394            let Some((module, _, _, _)) = current.get(module_id) else {
6395                continue;
6396            };
6397            let configured_module = configured
6398                .get(module_id)
6399                .expect("configured module id came from configured map");
6400            if configuration_changes.contains(module_id) {
6401                module
6402                    .update_configuration(
6403                        configured_module.module_spec(),
6404                        configured_module.health.clone(),
6405                        configured_module.drain_timeout_ms,
6406                    )
6407                    .await
6408                    .map_err(|err| {
6409                        format!(
6410                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6411                        )
6412                    })?;
6413            }
6414            if enabled_changes.contains(module_id) {
6415                // A rescan that starts or stops a module applies an operator's
6416                // edit to the config, so the resulting outage was asked for.
6417                self.route_outages.mark_operator_action(module_id);
6418                module
6419                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6420                    .await
6421                    .map_err(|err| {
6422                        self.route_outages.operator_action_ended_unrefused(module_id);
6423                        format!(
6424                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6425                            configured_module.enabled
6426                        )
6427                    })?;
6428            }
6429        }
6430
6431        for module_id in &added {
6432            let configured_module = configured
6433                .get(module_id)
6434                .expect("added module id came from configured map");
6435            supervisor
6436                .supervise_configured_requested(
6437                    configured_module.module_spec(),
6438                    configured_module.enabled,
6439                    configured_module.health.clone(),
6440                    configured_module.drain_timeout_ms,
6441                    configured_module.restart,
6442                    configured_module.register_timeout,
6443                    Some(requester.clone()),
6444                )
6445                .map_err(|err| {
6446                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6447                })?;
6448        }
6449
6450        Ok((
6451            SupervisorRescanResult {
6452                added,
6453                removed,
6454                changed_pending_reload,
6455                enabled_changes: enabled_changes.iter().cloned().collect(),
6456                unchanged,
6457                preview: false,
6458                // Filled by the caller, which is the only layer that can see the
6459                // previous config to diff against.
6460                restart_required: Vec::new(),
6461                capability_warnings: Vec::new(),
6462                module_actions: Vec::new(),
6463                earlier_pending: Vec::new(),
6464                expect: None,
6465            },
6466            changed,
6467        ))
6468    }
6469
6470    async fn handle_supervisor_set_enabled(
6471        &self,
6472        frame: Frame,
6473        module_id: String,
6474        enabled: bool,
6475        requester: Principal,
6476    ) -> Result<Vec<Frame>, RouterError> {
6477        let operation_lock = self.supervisor.operation_lock();
6478        let _operation_guard = operation_lock.lock().await;
6479        let Some(module) = self.supervisor.get(&module_id) else {
6480            return Ok(vec![control_error_frame(
6481                &frame,
6482                "unknown_module",
6483                format!("module_id '{module_id}' is not supervised"),
6484            )?]);
6485        };
6486
6487        // Enabling counts as well as disabling: a module an operator starts
6488        // is refused until it registers, and that wait was asked for.
6489        self.route_outages.mark_operator_action(&module_id);
6490        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6491            Ok(applied) => applied,
6492            Err(err) => {
6493                self.route_outages
6494                    .operator_action_ended_unrefused(&module_id);
6495                return Ok(vec![control_error_frame(
6496                    &frame,
6497                    "target_unavailable",
6498                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6499                )?]);
6500            }
6501        };
6502        if !applied {
6503            // Already in the requested state: nothing was made unavailable,
6504            // so the mark must not outlive this request.
6505            self.route_outages
6506                .operator_action_ended_unrefused(&module_id);
6507        }
6508
6509        self.capability_evaluator.wake_deadline_loop();
6510        self.refresh_capability_requirements();
6511        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6512        Ok(vec![control_response_body_frame(
6513            &frame,
6514            &response,
6515            "ClientControlResponse::SupervisorAck",
6516        )?])
6517    }
6518
6519    async fn handle_supervisor_health_probe(
6520        &self,
6521        frame: Frame,
6522        module_id: String,
6523    ) -> Result<Vec<Frame>, RouterError> {
6524        self.refresh_capability_requirements();
6525        let Some(registration) = self
6526            .registry
6527            .get_module(&module_id)
6528            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6529        else {
6530            return Ok(vec![control_error_frame(
6531                &frame,
6532                "unknown_module",
6533                format!("module_id '{module_id}' is not registered"),
6534            )?]);
6535        };
6536
6537        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6538        // named for it. Making `module_registration_grants_op` return false
6539        // unconditionally reddens five tests, and every one is named for something
6540        // else -- capability relay, probe/bind demultiplexing, supervision-only
6541        // probing. They exercise a successful advertisement check on the way to their
6542        // own subject.
6543        //
6544        // Real protection, fragile in a specific way: narrowing any of those tests to
6545        // focus on its stated subject would silently remove coverage nobody knows
6546        // they are carrying. Recorded here rather than as a sixth test, because the
6547        // useful fact is WHICH tests hold the guard up -- a new test would add
6548        // coverage without telling the next person what the existing ones quietly do.
6549        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6550        {
6551            return Ok(vec![control_error_frame(
6552                &frame,
6553                "health_not_advertised",
6554                format!("module_id '{module_id}' did not advertise health.check"),
6555            )?]);
6556        }
6557
6558        let deadline = Instant::now() + self.health_probe_timeout;
6559        let pending = match self.forwarding.begin_module_control_rpc_for(
6560            &module_id,
6561            MODULE_CONTROL_OP_HEALTH_CHECK,
6562            deadline,
6563        ) {
6564            Ok(pending) => pending,
6565            Err(err) => {
6566                return Ok(vec![control_error_frame(
6567                    &frame,
6568                    forwarding_error_code(&err),
6569                    err.to_string(),
6570                )?])
6571            }
6572        };
6573
6574        let PendingModuleControlRpc {
6575            endpoint,
6576            module_sink,
6577            negotiated_ver,
6578            corr: probe_corr,
6579            receiver,
6580        } = pending;
6581        let mut guard =
6582            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6583        let probe_body =
6584            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6585                RouterError::backend(
6586                    0,
6587                    frame.header.corr,
6588                    format!("failed to encode health.check request: {err}"),
6589                )
6590            })?;
6591        let probe_frame = Frame::build_with_version(
6592            negotiated_ver,
6593            FrameType::Request,
6594            control_flags(),
6595            0,
6596            0,
6597            probe_corr,
6598            probe_body,
6599        )
6600        .map_err(RouterError::FrameBuild)?;
6601
6602        // Queue admission and the reply share one deadline, so a module that
6603        // stops reading cannot park the requesting connection's reader forever.
6604        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6605            Ok(Ok(())) => {}
6606            Ok(Err(err)) => {
6607                return Ok(vec![control_error_frame(
6608                    &frame,
6609                    "target_unavailable",
6610                    err.to_string(),
6611                )?]);
6612            }
6613            Err(_) => {
6614                return Ok(vec![control_error_frame(
6615                    &frame,
6616                    "module_timeout",
6617                    format!(
6618                        "module_id '{module_id}' did not answer health.check within {:?}",
6619                        self.health_probe_timeout
6620                    ),
6621                )?]);
6622            }
6623        }
6624
6625        match timeout_at(deadline, receiver).await {
6626            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6627                guard.disarm();
6628                let Some(report) = response.health_report() else {
6629                    return Ok(vec![control_error_frame(
6630                        &frame,
6631                        "invalid_control_body",
6632                        "health.check RPC returned a non-health response",
6633                    )?]);
6634                };
6635                // Metrics go out whole here. The supervisor's cached snapshot
6636                // caps this blob (see truncate_health_metrics), and this path
6637                // exists precisely to answer without that cap -- so applying it
6638                // here would leave no way to see what the cached view drops.
6639                let HealthReport {
6640                    status,
6641                    detail,
6642                    metrics,
6643                } = report;
6644                let capability_detail = self
6645                    .capability_evaluator
6646                    .required_problem_detail(&module_id);
6647                let response = ClientControlResponse::SupervisorHealthProbe {
6648                    module_id,
6649                    status,
6650                    detail: append_capability_problem_detail(detail, capability_detail),
6651                    metrics,
6652                };
6653                Ok(vec![control_response_body_frame(
6654                    &frame,
6655                    &response,
6656                    "ClientControlResponse::SupervisorHealthProbe",
6657                )?])
6658            }
6659            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6660                guard.disarm();
6661                Ok(vec![control_error_body_frame(&frame, body)?])
6662            }
6663            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6664                guard.disarm();
6665                Ok(vec![control_error_frame(
6666                    &frame,
6667                    "target_unavailable",
6668                    message,
6669                )?])
6670            }
6671            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6672                guard.disarm();
6673                Ok(vec![control_error_frame(
6674                    &frame,
6675                    "invalid_control_body",
6676                    message,
6677                )?])
6678            }
6679            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6680                guard.disarm();
6681                Ok(vec![control_error_frame(
6682                    &frame,
6683                    "invalid_control_body",
6684                    format!("expected module-control op '{expected}', got '{actual}'"),
6685                )?])
6686            }
6687            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6688                guard.disarm();
6689                Ok(vec![control_error_frame(
6690                    &frame,
6691                    "module_timeout",
6692                    format!(
6693                        "module_id '{module_id}' answered health.check after {:?}",
6694                        self.health_probe_timeout
6695                    ),
6696                )?])
6697            }
6698            Ok(Err(_)) => Ok(vec![control_error_frame(
6699                &frame,
6700                "target_unavailable",
6701                "health.check waiter was canceled before the module responded",
6702            )?]),
6703            Err(_) => Ok(vec![control_error_frame(
6704                &frame,
6705                "module_timeout",
6706                format!(
6707                    "module_id '{module_id}' did not answer health.check within {:?}",
6708                    self.health_probe_timeout
6709                ),
6710            )?]),
6711        }
6712    }
6713
6714    fn supervisor_status(
6715        &self,
6716        module_id: &str,
6717        corr: u64,
6718    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6719        self.supervisor
6720            .get(module_id)
6721            .map(|module| {
6722                let warming = module.is_warming_for_control("status").map_err(|err| {
6723                    RouterError::backend(
6724                        0,
6725                        corr,
6726                        format!(
6727                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6728                        ),
6729                    )
6730                })?;
6731                module.status_for_control("status").map_err(|err| {
6732                    RouterError::backend(
6733                        0,
6734                        corr,
6735                        format!(
6736                            "failed to read supervisor status for module_id '{module_id}': {err}"
6737                        ),
6738                    )
6739                }).map(|status| (status, warming))
6740            })
6741            .transpose()
6742    }
6743
6744    fn guard_module_control_op(
6745        &self,
6746        frame: &Frame,
6747        module_id: &str,
6748        op: &str,
6749    ) -> Result<Option<Frame>, RouterError> {
6750        if self.module_grants_op(module_id, op, frame.header.corr)? {
6751            return Ok(None);
6752        }
6753
6754        Ok(Some(control_error_frame(
6755            frame,
6756            "op_not_allowed",
6757            format!("module_id '{module_id}' did not grant control op '{op}'"),
6758        )?))
6759    }
6760
6761    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6762        let Some(registration) = self
6763            .registry
6764            .get_module(module_id)
6765            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6766        else {
6767            return Ok(false);
6768        };
6769        Ok(module_registration_grants_op(&registration.control_ops, op))
6770    }
6771
6772    fn handle_status_update(
6773        &self,
6774        endpoint: ModuleEndpointId,
6775        frame: Frame,
6776    ) -> Result<Vec<Frame>, RouterError> {
6777        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6778            Ok(update) => update,
6779            Err(err) => {
6780                // Forward-compat: a newer module may push a channel-0 op this subc
6781                // version doesn't know. The control contract says unknown push ops
6782                // are IGNORED, never answered with an error. Only a malformed body
6783                // for an op we DO know is a real error worth surfacing.
6784                if is_known_module_push_op(&frame.body) {
6785                    return Ok(vec![control_error_frame(
6786                        &frame,
6787                        "invalid_control_body",
6788                        format!("malformed module control push body: {err}"),
6789                    )?]);
6790                }
6791                return Ok(Vec::new());
6792            }
6793        };
6794
6795        match update {
6796            ModuleControlPush::RouteStatus {
6797                route_channel,
6798                route_epoch,
6799                status,
6800            } => {
6801                self.forwarding
6802                    .cache_status(endpoint, route_channel, route_epoch, status)
6803                    .map_err(RouterError::Forwarding)?;
6804            }
6805        }
6806        Ok(Vec::new())
6807    }
6808
6809    fn handle_route_poll(
6810        &self,
6811        ctx: &RouteCtx,
6812        frame: Frame,
6813        route_channel: u16,
6814        route_epoch: u32,
6815        kind: PollKind,
6816    ) -> Result<Vec<Frame>, RouterError> {
6817        let snapshot = self
6818            .forwarding
6819            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6820            .map_err(RouterError::Forwarding)?;
6821        let response = match (kind, snapshot) {
6822            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6823                ClientControlResponse::RoutePoll {
6824                    route_channel,
6825                    route_epoch,
6826                    status,
6827                    live: None,
6828                }
6829            }
6830            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6831                route_channel,
6832                route_epoch,
6833                status: None,
6834                live: None,
6835            },
6836            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6837                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6838                // is what makes reporting `true` correct rather than a
6839                // confident guess. `process_live` returns None only when the
6840                // module id has no supervisor snapshot at all -- an
6841                // externally-started module the daemon did not spawn -- and
6842                // for those the supervisor has no opinion to offer, ever. It
6843                // is never None for a supervised module in an unknown state:
6844                // a supervised module always has a snapshot, and the answer
6845                // comes from `state == Running && process_alive`.
6846                //
6847                // The route is Bound, so the module completed a HELLO on a
6848                // live connection; "the process this route points at is
6849                // running" is therefore attested by the binding rather than
6850                // assumed. Reporting `false` for an unsupervised module would
6851                // be the actual lie -- it would tell a client its healthy
6852                // route is dead because the daemon does not manage the
6853                // process.
6854                //
6855                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6856                // module whose liveness is genuinely unknown, e.g. a snapshot
6857                // that has not been populated yet -- THIS DEFAULT BECOMES
6858                // WRONG and must split: unsupervised stays true, unknown
6859                // becomes null so the client can tell the two apart. The
6860                // response field is already `Option<bool>`, so the wire can
6861                // carry that distinction today.
6862                let live = self
6863                    .process_liveness
6864                    .as_ref()
6865                    .and_then(|source| source.process_live(&module_id))
6866                    .unwrap_or(true);
6867                ClientControlResponse::RoutePoll {
6868                    route_channel,
6869                    route_epoch,
6870                    status: None,
6871                    live: Some(live),
6872                }
6873            }
6874            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6875                route_channel,
6876                route_epoch,
6877                status: None,
6878                live: Some(false),
6879            },
6880        };
6881
6882        Ok(vec![control_response_body_frame(
6883            &frame,
6884            &response,
6885            "ClientControlResponse::RoutePoll",
6886        )?])
6887    }
6888
6889    pub(crate) fn observe_module_control_completion(
6890        &self,
6891        completion: ModuleControlRpcCompletion,
6892    ) -> bool {
6893        match completion {
6894            ModuleControlRpcCompletion::Unknown => false,
6895            ModuleControlRpcCompletion::Settled => true,
6896            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6897                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6898                info!(
6899                    module_id = %module_id,
6900                    latency_ms,
6901                    "late health.check answer proves the module is alive"
6902                );
6903                match self
6904                    .supervisor
6905                    .record_late_health_answer(&module_id, latency_ms)
6906                {
6907                    Ok(true) => {}
6908                    Ok(false) => debug!(
6909                        module_id = %module_id,
6910                        latency_ms,
6911                        "late health.check answer has no active supervisor snapshot"
6912                    ),
6913                    Err(err) => warn!(
6914                        module_id = %module_id,
6915                        latency_ms,
6916                        error = %err,
6917                        "failed to record late health.check answer"
6918                    ),
6919                }
6920                true
6921            }
6922        }
6923    }
6924
6925    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6926    /// the module connection whose frame is being handled, or to the client that
6927    /// relay was opened for.
6928    ///
6929    /// This runs on the MODULE connection's frame handler, where returning `Err`
6930    /// ends that connection -- and a module connection carries every client's
6931    /// routes to that module, so ending it costs the whole fleet its tools.
6932    /// `ConnectionClosing` carries the id of the connection that is closing, and
6933    /// when that id is a CLIENT's, the condition is entirely about that one
6934    /// client's route.open. A client-scoped condition has no authority over a
6935    /// shared module connection, so it is logged and the single relay is dropped:
6936    /// the client is going away, and `complete_pending_relay` already removed the
6937    /// relay before failing, so there is nothing left to settle. Anything that
6938    /// relay still reserved is released by that client's own connection teardown,
6939    /// which is already under way -- that is what "closing" means.
6940    ///
6941    /// Every other failure is a statement about THIS connection and stays fatal:
6942    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6943    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6944    /// frames correctly.
6945    fn refuse_to_end_module_connection_for_a_client(
6946        &self,
6947        module_connection_id: ConnectionId,
6948        corr: u64,
6949        err: ForwardingError,
6950    ) -> Result<(), RouterError> {
6951        if let ForwardingError::ConnectionClosing { connection_id } = err {
6952            if connection_id != module_connection_id {
6953                warn!(
6954                    module_connection_id = module_connection_id.get(),
6955                    client_connection_id = connection_id.get(),
6956                    corr,
6957                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6958                );
6959                return Ok(());
6960            }
6961        }
6962        Err(RouterError::Forwarding(err))
6963    }
6964
6965    fn handle_module_relay_response(
6966        &self,
6967        connection_id: ConnectionId,
6968        frame: Frame,
6969    ) -> Result<Vec<Frame>, RouterError> {
6970        let mut secondary_error = None;
6971        let outcome = match frame.header.ty {
6972            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6973                Ok(probe) if probe.op == "route.bind" => {
6974                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6975                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6976                            RouteBindRelayOutcome::Accepted
6977                        }
6978                        Ok(other) => {
6979                            let message =
6980                                format!("route.bind response carried unexpected body: {other:?}");
6981                            secondary_error = Some(control_error_frame(
6982                                &frame,
6983                                "invalid_control_body",
6984                                message.clone(),
6985                            )?);
6986                            RouteBindRelayOutcome::ModuleGone(message)
6987                        }
6988                        Err(err) => {
6989                            let message = format!("malformed route.bind response body: {err}");
6990                            secondary_error = Some(control_error_frame(
6991                                &frame,
6992                                "invalid_control_body",
6993                                message.clone(),
6994                            )?);
6995                            RouteBindRelayOutcome::ModuleGone(message)
6996                        }
6997                    }
6998                }
6999                Ok(probe) => {
7000                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7001                    {
7002                        Ok(response) => ModuleControlRpcOutcome::Response(response),
7003                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7004                            "malformed {} response body: {err}",
7005                            probe.op
7006                        )),
7007                    };
7008                    let completion = self
7009                        .forwarding
7010                        .complete_module_control_rpc(
7011                            connection_id,
7012                            frame.header.corr,
7013                            Some(&probe.op),
7014                            outcome,
7015                        )
7016                        .map_err(RouterError::Forwarding)?;
7017                    if !self.observe_module_control_completion(completion) {
7018                        debug!(
7019                            connection_id = connection_id.get(),
7020                            corr = frame.header.corr,
7021                            op = %probe.op,
7022                            "dropping late or unknown module-control RPC response"
7023                        );
7024                    }
7025                    return Ok(Vec::new());
7026                }
7027                Err(err) => {
7028                    if let Some(expected_op) = self
7029                        .forwarding
7030                        .pending_module_control_op(connection_id, frame.header.corr)
7031                        .map_err(RouterError::Forwarding)?
7032                    {
7033                        let completion = self
7034                            .forwarding
7035                            .complete_module_control_rpc(
7036                                connection_id,
7037                                frame.header.corr,
7038                                None,
7039                                ModuleControlRpcOutcome::MalformedResponse(format!(
7040                                    "malformed {expected_op} response body: {err}"
7041                                )),
7042                            )
7043                            .map_err(RouterError::Forwarding)?;
7044                        if !self.observe_module_control_completion(completion) {
7045                            debug!(
7046                                connection_id = connection_id.get(),
7047                                corr = frame.header.corr,
7048                                "dropping late malformed module-control RPC response"
7049                            );
7050                        }
7051                        return Ok(Vec::new());
7052                    }
7053                    let message = format!("malformed route.bind response body: {err}");
7054                    secondary_error = Some(control_error_frame(
7055                        &frame,
7056                        "invalid_control_body",
7057                        message.clone(),
7058                    )?);
7059                    RouteBindRelayOutcome::ModuleGone(message)
7060                }
7061            },
7062            FrameType::Error => {
7063                if self
7064                    .forwarding
7065                    .pending_module_control_op(connection_id, frame.header.corr)
7066                    .map_err(RouterError::Forwarding)?
7067                    .is_some()
7068                {
7069                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7070                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7071                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7072                            "malformed module-control ERROR body: {err}"
7073                        )),
7074                    };
7075                    let completion = self
7076                        .forwarding
7077                        .complete_module_control_rpc(
7078                            connection_id,
7079                            frame.header.corr,
7080                            None,
7081                            outcome,
7082                        )
7083                        .map_err(RouterError::Forwarding)?;
7084                    if !self.observe_module_control_completion(completion) {
7085                        debug!(
7086                            connection_id = connection_id.get(),
7087                            corr = frame.header.corr,
7088                            "dropping late or unknown module-control RPC error"
7089                        );
7090                    }
7091                    return Ok(Vec::new());
7092                }
7093                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7094                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7095                    Err(err) => {
7096                        let message = format!("malformed route.bind ERROR body: {err}");
7097                        secondary_error = Some(control_error_frame(
7098                            &frame,
7099                            "invalid_control_body",
7100                            message.clone(),
7101                        )?);
7102                        RouteBindRelayOutcome::ModuleGone(message)
7103                    }
7104                }
7105            }
7106            ty => {
7107                return Ok(vec![control_error_frame(
7108                    &frame,
7109                    "unsupported_control_frame",
7110                    format!("unsupported module channel-0 frame {ty:?}"),
7111                )?])
7112            }
7113        };
7114
7115        let settled =
7116            self.forwarding
7117                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7118        let completion = match settled {
7119            Ok(completion) => completion,
7120            Err(err) => {
7121                self.refuse_to_end_module_connection_for_a_client(
7122                    connection_id,
7123                    frame.header.corr,
7124                    err,
7125                )?;
7126                return Ok(secondary_error.into_iter().collect());
7127            }
7128        };
7129        if let Some(target) = completion.abandoned.as_ref() {
7130            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7131        }
7132        if !completion.settled {
7133            debug!(
7134                connection_id = connection_id.get(),
7135                corr = frame.header.corr,
7136                frame_type = ?frame.header.ty,
7137                "dropping late or unknown route.bind relay response"
7138            );
7139        }
7140        Ok(secondary_error.into_iter().collect())
7141    }
7142
7143    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7144        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7145        // GOODBYE ends the connection's logical session even when its socket
7146        // stays open. Use disconnect teardown so verdicts, client notices and
7147        // scope authority are released at the same lifecycle boundary.
7148        self.cleanup_connection_with_end_reason(
7149            connection_id,
7150            RegistrationEndReason::ExplicitGoodbye,
7151        )
7152        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7153        Ok(Vec::new())
7154    }
7155}
7156
7157impl Default for ControlHandler {
7158    fn default() -> Self {
7159        Self::new(Arc::new(Registry::default()))
7160    }
7161}
7162
7163impl crate::supervise::SwapPromotionObserver for ControlHandler {
7164    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7165        self.apply_registration_capabilities(registration);
7166    }
7167}
7168
7169fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7170    CapabilityRequirementStatus {
7171        consumer: status.consumer,
7172        capability: status.capability,
7173        need: match status.need {
7174            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7175            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7176        },
7177        verdict: status.verdict.as_str().to_string(),
7178        episode_seq: status.episode_seq,
7179        config_satisfiable: status.config_satisfiable,
7180        runtime_available: status.runtime_available,
7181        detail: status.detail,
7182    }
7183}
7184
7185fn append_capability_problem_detail(
7186    detail: Option<String>,
7187    capability_detail: Option<String>,
7188) -> Option<String> {
7189    match (detail, capability_detail) {
7190        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7191        (Some(detail), None) => Some(detail),
7192        (None, Some(capability_detail)) => Some(capability_detail),
7193        (None, None) => None,
7194    }
7195}
7196
7197fn subc_ops() -> Vec<String> {
7198    SUBC_CONTROL_OPS
7199        .iter()
7200        .map(|op| (*op).to_string())
7201        .collect()
7202}
7203
7204fn module_subc_ops() -> Vec<String> {
7205    SUBC_CONTROL_OPS
7206        .iter()
7207        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7208        .map(|op| (*op).to_string())
7209        .collect()
7210}
7211
7212#[cfg(test)]
7213fn module_baseline_control_ops() -> Vec<String> {
7214    MODULE_BASELINE_CONTROL_OPS
7215        .iter()
7216        .map(|op| (*op).to_string())
7217        .collect()
7218}
7219
7220fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7221    let mut seen = HashSet::new();
7222    let mut effective = Vec::new();
7223    for op in MODULE_BASELINE_CONTROL_OPS {
7224        if seen.insert((*op).to_string()) {
7225            effective.push((*op).to_string());
7226        }
7227    }
7228    for op in declared.unwrap_or_default() {
7229        if seen.insert(op.clone()) {
7230            effective.push(op);
7231        }
7232    }
7233    effective
7234}
7235
7236fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7237    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7238}
7239
7240fn target_module_id(target: &RouteTarget) -> &str {
7241    match target {
7242        RouteTarget::ToolProvider { module_id }
7243        | RouteTarget::ManagementSurface { module_id }
7244        | RouteTarget::InternalService { module_id, .. } => module_id,
7245    }
7246}
7247
7248fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7249    roles.iter().any(|role| match (target, role) {
7250        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7251        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7252        (
7253            RouteTarget::InternalService { service_id, .. },
7254            ProviderRole::InternalService {
7255                service_id: provided,
7256                ..
7257            },
7258        ) => service_id == provided,
7259        _ => false,
7260    })
7261}
7262
7263fn is_routable_role(role: &ProviderRole) -> bool {
7264    matches!(
7265        role,
7266        ProviderRole::ToolProvider { .. }
7267            | ProviderRole::ManagementSurface { .. }
7268            | ProviderRole::InternalService { .. }
7269    )
7270}
7271
7272#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7273enum ControlRequestBodyError {
7274    UnknownOp,
7275    InvalidBody,
7276}
7277
7278#[derive(Debug, Deserialize)]
7279struct ControlOpProbe {
7280    op: String,
7281}
7282
7283/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7284/// this set is treated as a forward-compat unknown and ignored rather than errored.
7285const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7286
7287fn is_known_module_push_op(body: &[u8]) -> bool {
7288    serde_json::from_slice::<ControlOpProbe>(body)
7289        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7290        .unwrap_or(false)
7291}
7292
7293fn is_known_module_request_op(body: &[u8]) -> bool {
7294    serde_json::from_slice::<ControlOpProbe>(body)
7295        .map(|probe| is_module_to_subc_op(&probe.op))
7296        .unwrap_or(false)
7297}
7298
7299fn is_module_to_subc_op(op: &str) -> bool {
7300    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7301}
7302
7303fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7304    debug!(
7305        op = %op,
7306        connection_id = connection_id.get(),
7307        corr,
7308        "control dispatch"
7309    );
7310}
7311
7312fn log_slow_control_dispatch(
7313    dispatch_started_at: Option<StdInstant>,
7314    op: &'static str,
7315    connection_id: ConnectionId,
7316    corr: u64,
7317    route_open_timings: Option<&RouteOpenTimings>,
7318) {
7319    let Some(dispatch_started_at) = dispatch_started_at else {
7320        return;
7321    };
7322    // Use the same monotonic clock as the phase timers and relay deadline.
7323    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7324    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7325        if let Some(timings) = route_open_timings {
7326            warn!(
7327                op = %op,
7328                connection_id = connection_id.get(),
7329                corr,
7330                elapsed_ms = elapsed.as_millis() as u64,
7331                module_id = ?timings.module_id,
7332                principal = %timings.principal,
7333                admission_ms = timings.admission.as_millis() as u64,
7334                project_root_ms = timings.project_root.as_millis() as u64,
7335                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7336                bind_send_ms = timings.bind_send.as_millis() as u64,
7337                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7338                "slow control dispatch"
7339            );
7340            return;
7341        }
7342        warn!(
7343            op = %op,
7344            connection_id = connection_id.get(),
7345            corr,
7346            elapsed_ms = elapsed.as_millis() as u64,
7347            "slow control dispatch"
7348        );
7349    }
7350}
7351
7352/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7353#[cfg(test)]
7354type TestCallerLookup =
7355    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7356
7357/// Every new client operation needs an explicit privilege decision here.
7358fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7359    match request {
7360        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7361            Some(PrivilegedOperation::Restart { module_id })
7362        }
7363        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7364            Some(PrivilegedOperation::Swap { module_id })
7365        }
7366        ClientControlRequest::SupervisorReload { module_id } => {
7367            Some(PrivilegedOperation::Reload { module_id })
7368        }
7369        ClientControlRequest::SupervisorRescan { preview } => {
7370            if *preview {
7371                None
7372            } else {
7373                Some(PrivilegedOperation::Rescan)
7374            }
7375        }
7376        // Only the well-formed apply asks for approval. A preview changes
7377        // nothing, and any other combination is refused before it touches
7378        // anything, so prompting the operator for it would ask about a request
7379        // that cannot run.
7380        ClientControlRequest::SupervisorRescanChecked {
7381            preview,
7382            candidate,
7383            expect,
7384        } => (!*preview && candidate.is_none() && expect.is_some())
7385            .then_some(PrivilegedOperation::Rescan),
7386        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7387            Some(PrivilegedOperation::SetEnabled {
7388                module_id,
7389                enabled: *enabled,
7390            })
7391        }
7392        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7393            Some(PrivilegedOperation::ReleaseReserved { module_id })
7394        }
7395        ClientControlRequest::ServerDescribe {}
7396        | ClientControlRequest::CatalogList { .. }
7397        | ClientControlRequest::RouteOpen { .. }
7398        | ClientControlRequest::RoutePoll { .. }
7399        | ClientControlRequest::SupervisorList {}
7400        | ClientControlRequest::SupervisorHealth {}
7401        | ClientControlRequest::SupervisorHealthProbe { .. }
7402        | ClientControlRequest::SupervisorRoutes { .. }
7403        | ClientControlRequest::SupervisorProvenance { .. }
7404        | ClientControlRequest::SupervisorStderrTail { .. }
7405        | ClientControlRequest::SupervisorTerminals { .. }
7406        | ClientControlRequest::SupervisorSpawnSnapshot {}
7407        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7408    }
7409}
7410
7411fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7412    match request {
7413        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7414        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7415        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7416        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7417        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7418        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7419        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7420        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7421        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7422        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7423        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7424        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7425        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7426        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7427        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7428        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7429        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7430        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7431        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7432        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7433    }
7434}
7435
7436/// The caller of one `route.open`, as its `route.open refused` line names it.
7437///
7438/// A connection id alone cannot name a caller after the fact: ids restart at
7439/// every daemon restart, so the same number belongs to different modules
7440/// before and after one. These three fields are what the refused line needs
7441/// to be attributable on its own, and they are rendered with the same helpers
7442/// as the `route.open accepted` line so both directions grep alike.
7443struct RouteOpenCaller {
7444    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
7445    /// when the request presented a consumer identity whose launch nonce did
7446    /// not verify. The claimed module id is never printed as a principal.
7447    principal: String,
7448    /// `none` for an unscoped open; the admitted stamp's label once scope
7449    /// admission issued one; before that, the requested selector's label.
7450    scope: String,
7451    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
7452    /// because it is a string the caller chose.
7453    harness: String,
7454}
7455
7456impl RouteOpenCaller {
7457    fn new(
7458        principal: Option<&Principal>,
7459        scope: Option<&ScopeSelector>,
7460        identity: &BindIdentity,
7461    ) -> Self {
7462        Self {
7463            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7464            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7465            harness: bounded_log_text(&identity.harness),
7466        }
7467    }
7468
7469    /// For a refusal that has no parsed request at hand: names whatever the
7470    /// frame still says, and `unknown` where it says nothing parseable.
7471    fn unknown() -> Self {
7472        Self {
7473            principal: "unknown".to_string(),
7474            scope: "unknown".to_string(),
7475            harness: "unknown".to_string(),
7476        }
7477    }
7478}
7479
7480/// Longest client-chosen text (a scope ref, a harness) a route.open log line
7481/// prints before cutting it.
7482const MAX_LOGGED_TEXT_CHARS: usize = 64;
7483
7484/// `value` Debug-quoted, so control characters land escaped and cannot forge
7485/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
7486/// `…` inside the quotes so one request cannot flood the line.
7487fn bounded_log_text(value: &str) -> String {
7488    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7489    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7490        bounded.push('…');
7491    }
7492    format!("{bounded:?}")
7493}
7494
7495/// The `scope=` field of the `route.open accepted` line, from the stamp that
7496/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7497/// so a ref holding control characters cannot forge a log line, and cut to 64
7498/// characters with a trailing `…` to bound the line. The bind proof and the
7499/// scope's attributes never appear.
7500fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7501    use subc_protocol::scope::ScopeKind;
7502    let kind = match stamp.kind {
7503        ScopeKind::Head => "head",
7504        ScopeKind::Worker => "worker",
7505        ScopeKind::Ephemeral => "ephemeral",
7506    };
7507    format!(
7508        "{kind}/{}/{}@{}",
7509        principal_label(&stamp.owner),
7510        bounded_log_text(&stamp.scope_ref),
7511        stamp.scope_epoch
7512    )
7513}
7514
7515/// The `scope=` field of a `route.open refused` line for an open refused
7516/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
7517/// The kind is not known yet, so the literal `requested` stands in its place.
7518/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
7519/// owner it has not been checked against the configured modules: a quoted
7520/// owner means the value was never validated. The bind proof never appears.
7521fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7522    let epoch = selector
7523        .scope_epoch
7524        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7525    format!(
7526        "requested/{}/{}@{epoch}",
7527        bounded_log_text(&principal_label(&selector.owner)),
7528        bounded_log_text(&selector.scope_ref),
7529    )
7530}
7531
7532fn principal_label(principal: &Principal) -> String {
7533    match principal {
7534        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7535        Principal::Direct => "direct".to_string(),
7536        other => format!("{other:?}"),
7537    }
7538}
7539
7540fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7541    match request {
7542        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7543        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7544        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7545        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7546        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7547    }
7548}
7549
7550fn parse_client_control_request(
7551    body: &[u8],
7552) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7553    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7554        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7555            Ok(probe)
7556                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7557                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7558            {
7559                ControlRequestBodyError::InvalidBody
7560            }
7561            Ok(_) => ControlRequestBodyError::UnknownOp,
7562            Err(_) => ControlRequestBodyError::InvalidBody,
7563        };
7564        (err, classification)
7565    })
7566}
7567
7568fn parse_module_control_request_from_module(
7569    body: &[u8],
7570) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7571    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7572        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7573            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7574            Ok(_) => ControlRequestBodyError::UnknownOp,
7575            Err(_) => ControlRequestBodyError::InvalidBody,
7576        };
7577        (err, classification)
7578    })
7579}
7580
7581#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7582enum ProviderRoleKind {
7583    ToolProvider,
7584    PipelineStage,
7585    ManagementSurface,
7586    InternalService,
7587}
7588
7589fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7590    match role {
7591        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7592        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7593        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7594        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7595    }
7596}
7597
7598fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7599    roles.iter().map(provider_role_kind).collect()
7600}
7601
7602/// Most refused scope records named individually in the log per sync; the
7603/// `refused` count on the accepted line is always complete.
7604const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7605
7606/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7607#[derive(Debug, Default, PartialEq, Eq)]
7608struct ScopeOutcomeCounts {
7609    created: usize,
7610    replaced: usize,
7611    updated: usize,
7612    unchanged: usize,
7613    refused: usize,
7614}
7615
7616impl ScopeOutcomeCounts {
7617    fn of(results: &[ScopeRecordResult]) -> Self {
7618        let mut counts = Self::default();
7619        for result in results {
7620            let slot = match result.outcome {
7621                ScopeRecordOutcome::Created => &mut counts.created,
7622                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7623                ScopeRecordOutcome::Updated => &mut counts.updated,
7624                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7625                ScopeRecordOutcome::Refused => &mut counts.refused,
7626            };
7627            *slot += 1;
7628        }
7629        counts
7630    }
7631}
7632
7633#[cfg(test)]
7634mod scope_outcome_count_tests {
7635    use super::*;
7636
7637    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7638        ScopeRecordResult {
7639            scope_ref: "r".to_string(),
7640            scope_epoch: 1,
7641            outcome,
7642            code: None,
7643            message: None,
7644            version: None,
7645            parent_state: None,
7646        }
7647    }
7648
7649    /// Each outcome lands in its own count, so a refused record can never be
7650    /// hidden inside the total the log already printed.
7651    #[test]
7652    fn every_outcome_is_counted_in_its_own_field() {
7653        let results = [
7654            result(ScopeRecordOutcome::Created),
7655            result(ScopeRecordOutcome::Created),
7656            result(ScopeRecordOutcome::Replaced),
7657            result(ScopeRecordOutcome::Updated),
7658            result(ScopeRecordOutcome::Unchanged),
7659            result(ScopeRecordOutcome::Refused),
7660            result(ScopeRecordOutcome::Refused),
7661            result(ScopeRecordOutcome::Refused),
7662        ];
7663        assert_eq!(
7664            ScopeOutcomeCounts::of(&results),
7665            ScopeOutcomeCounts {
7666                created: 2,
7667                replaced: 1,
7668                updated: 1,
7669                unchanged: 1,
7670                refused: 3,
7671            }
7672        );
7673    }
7674}
7675
7676/// Return whether a catalog change can create a newly violating live route.
7677/// Removing an attested claim is intentionally excluded: it makes fewer routes
7678/// forbidden and therefore must leave the existing route census untouched.
7679fn capability_census_trigger(
7680    old: Option<&CapabilityDeclarations>,
7681    new: Option<&CapabilityDeclarations>,
7682) -> bool {
7683    let old_provides = old
7684        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7685        .unwrap_or_default();
7686    let old_denies = old
7687        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7688        .unwrap_or_default();
7689    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7690        provides: Vec::new(),
7691        requires: Vec::new(),
7692        must_never_reach: Vec::new(),
7693    });
7694
7695    new.provides
7696        .iter()
7697        .any(|capability| !old_provides.contains(capability))
7698        || new
7699            .must_never_reach
7700            .iter()
7701            .any(|capability| !old_denies.contains(capability))
7702}
7703
7704/// Find the first capability an attested opener denies that an attested target
7705/// claims. Both manifests are live registry records, never cached or client data.
7706fn denied_capability<'a>(
7707    opening_manifest: &'a ModuleManifest,
7708    target_manifest: &ModuleManifest,
7709) -> Option<&'a str> {
7710    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7711    let target_capabilities = target_manifest.capabilities.as_ref()?;
7712    opening_capabilities
7713        .must_never_reach
7714        .iter()
7715        .find(|denied| {
7716            target_capabilities
7717                .provides
7718                .iter()
7719                .any(|provided| provided == *denied)
7720        })
7721        .map(String::as_str)
7722}
7723
7724fn catalog_update_frozen_field_message(
7725    registered: &ModuleManifest,
7726    provides: &[ProviderRole],
7727) -> Option<String> {
7728    let old_has_provides = !registered.provides.is_empty();
7729    let new_has_provides = !provides.is_empty();
7730    if old_has_provides != new_has_provides {
7731        return Some(format!(
7732            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7733            registered.module_id
7734        ));
7735    }
7736
7737    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7738        return Some(format!(
7739            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7740            registered.module_id
7741        ));
7742    }
7743
7744    let registered_concurrency = manifest_concurrency(registered);
7745    let mut candidate = registered.clone();
7746    candidate.provides = provides.to_vec();
7747    let candidate_concurrency = manifest_concurrency(&candidate);
7748    if candidate_concurrency != registered_concurrency {
7749        return Some(format!(
7750            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7751            registered.module_id, registered_concurrency, candidate_concurrency
7752        ));
7753    }
7754
7755    // control_ops live beside the manifest in the HELLO body, not inside
7756    // ModuleManifest, so a provides-only catalog.update cannot change them.
7757    None
7758}
7759
7760fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7761    manifest.provides.iter().any(is_routable_role)
7762}
7763
7764/// Returns the routable-provider concurrency subc should enforce for this manifest.
7765///
7766/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7767/// InternalService has no role-specific concurrency field, so it retains the
7768/// existing ModuleManaged default for backward compatibility.
7769fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7770    manifest
7771        .provides
7772        .iter()
7773        .find_map(|provider| match provider {
7774            ProviderRole::ToolProvider { concurrency, .. }
7775            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7776            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7777        })
7778        .unwrap_or(Concurrency::ModuleManaged)
7779}
7780
7781/// True when the manifest carries a ManagementSurface role whose concurrency
7782/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7783/// bytes because the typed manifest deliberately erases that distinction: the
7784/// default exists for wire compatibility, and this probe exists so the default
7785/// stays observable. Any parse irregularity returns false -- the caller only
7786/// logs, and a malformed body already failed registration upstream.
7787fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7788    let has_management_surface = manifest
7789        .provides
7790        .iter()
7791        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7792    if !has_management_surface {
7793        return false;
7794    }
7795    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7796        return false;
7797    };
7798    let Some(provides) = raw
7799        .get("manifest")
7800        .and_then(|manifest| manifest.get("provides"))
7801        .and_then(serde_json::Value::as_array)
7802    else {
7803        return false;
7804    };
7805    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7806    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7807    // against the management_surface_manifest_without_concurrency golden, not
7808    // recalled (the externally-tagged guess was this function's first bug).
7809    provides.iter().any(|role| {
7810        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7811            && role.get("concurrency").is_none()
7812    })
7813}
7814
7815fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7816    if peer_version != PROTOCOL_VERSION {
7817        return Err(format!(
7818            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7819        ));
7820    }
7821    Ok(PROTOCOL_VERSION)
7822}
7823
7824fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7825    Frame::build_with_version(
7826        response_version(frame),
7827        FrameType::Pong,
7828        frame.header.flags,
7829        0,
7830        0,
7831        frame.header.corr,
7832        Vec::new(),
7833    )
7834    .map_err(RouterError::FrameBuild)
7835}
7836
7837pub(crate) fn control_error_frame(
7838    frame: &Frame,
7839    code: &'static str,
7840    message: impl Into<String>,
7841) -> Result<Frame, RouterError> {
7842    control_error_body_frame(
7843        frame,
7844        ErrorBody {
7845            code: code.to_string(),
7846            message: message.into(),
7847            detail: None,
7848        },
7849    )
7850}
7851
7852fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7853    let body = serde_json::to_vec(&error).map_err(|err| {
7854        RouterError::backend(
7855            0,
7856            frame.header.corr,
7857            format!("failed to encode control ERROR: {err}"),
7858        )
7859    })?;
7860
7861    Frame::build_with_version(
7862        response_version(frame),
7863        FrameType::Error,
7864        control_flags(),
7865        0,
7866        0,
7867        frame.header.corr,
7868        body,
7869    )
7870    .map_err(RouterError::FrameBuild)
7871}
7872
7873fn control_response_body_frame<T: Serialize>(
7874    frame: &Frame,
7875    reply: &T,
7876    label: &'static str,
7877) -> Result<Frame, RouterError> {
7878    let body = serde_json::to_vec(reply).map_err(|err| {
7879        RouterError::backend(
7880            0,
7881            frame.header.corr,
7882            format!("failed to encode {label}: {err}"),
7883        )
7884    })?;
7885
7886    Frame::build_with_version(
7887        response_version(frame),
7888        FrameType::Response,
7889        control_flags(),
7890        0,
7891        0,
7892        frame.header.corr,
7893        body,
7894    )
7895    .map_err(RouterError::FrameBuild)
7896}
7897
7898/// Map a forwarding failure to the wire code a client sees.
7899///
7900/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7901/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7902/// chosen here decides whether a caller retries or gives up.
7903///
7904/// That makes attribution the load-bearing property, not merely having a code. A
7905/// permanent fault published as a retryable one produces a fleet-wide retry storm
7906/// against something that can never recover; a transient fault published as
7907/// permanent gives up on work that would have succeeded. Both look correct in a
7908/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7909/// the mapping per variant rather than merely asserting that some code exists.
7910///
7911/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7912/// two codes on the same side of the boundary passes it. Measured rather than
7913/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7914/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7915/// a test named for something else that happens to assert the string.
7916///
7917/// That accidental coverage is deliberately left alone rather than promoted to a
7918/// named test, because it guards a property this function does not promise.
7919/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7920/// specific code within a class, so identity is free to change and only the
7921/// partition is a contract. Splitting it out would assert a guarantee nothing
7922/// depends on — and a suite that promises more than the code does is the harder
7923/// thing to correct later, because the next reader cannot tell which assertions
7924/// are load-bearing.
7925///
7926/// Pin identity here the moment a consumer branches on a specific code.
7927fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7928    match err {
7929        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7930        ForwardingError::NoModuleConnection => "target_unavailable",
7931        ForwardingError::ModuleReloading { .. } => "module_reloading",
7932        ForwardingError::ClientRouteChannelExhausted { .. }
7933        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7934        ForwardingError::StaleModuleEndpoint
7935        | ForwardingError::UnknownReservation { .. }
7936        | ForwardingError::ConnectionClosing { .. }
7937        | ForwardingError::ClientEgressClosed { .. }
7938        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7939        // Only a swap candidate's registration can produce this, and it means
7940        // exactly what a second active HELLO for a live id means.
7941        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7942        ForwardingError::RelayCorrelationExhausted
7943        | ForwardingError::RouteOpenBuild(_)
7944        | ForwardingError::Poisoned => "forwarding_error",
7945    }
7946}
7947
7948fn response_version(frame: &Frame) -> u8 {
7949    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7950        frame.header.ver
7951    } else {
7952        PROTOCOL_VERSION
7953    }
7954}
7955
7956fn control_flags() -> Flags {
7957    Flags::new(false, Priority::Passive, false)
7958}
7959
7960/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7961/// channel. The target is the module (a client never saw the route), so this
7962/// takes the module path: delivered late rather than dropped when the module's
7963/// queue is momentarily full, and never closing its connection.
7964fn send_goodbye_target_best_effort(
7965    counters: &DaemonCounters,
7966    target: &GoodbyeTarget,
7967    context: &'static str,
7968) {
7969    let Ok(frame) = Frame::build_with_version(
7970        target.negotiated_ver,
7971        FrameType::Goodbye,
7972        control_flags(),
7973        target.channel,
7974        target.epoch,
7975        0,
7976        Vec::new(),
7977    ) else {
7978        return;
7979    };
7980    crate::forwarding::send_module_route_goodbye(
7981        counters,
7982        &target.sink,
7983        frame,
7984        target.module_id.as_deref(),
7985        context,
7986    );
7987}
7988
7989pub(crate) fn send_route_control_pushes(
7990    forwarding: &ForwardingTable,
7991    routes: Vec<EndpointRoute>,
7992    push: ClientControlPush,
7993) {
7994    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7995    for route in routes {
7996        let target = route.goodbye_target;
7997        if let Some((existing, channels)) = targets
7998            .iter_mut()
7999            .find(|(existing, _)| existing.connection_id == target.connection_id)
8000        {
8001            debug_assert_eq!(
8002                existing.negotiated_ver, target.negotiated_ver,
8003                "one connection cannot negotiate multiple frame versions"
8004            );
8005            if !channels.contains(&target.channel) {
8006                channels.push(target.channel);
8007            }
8008            continue;
8009        }
8010        let channel = target.channel;
8011        targets.push((target, vec![channel]));
8012    }
8013    for (target, mut channels) in targets {
8014        channels.sort_unstable();
8015        let mut push = push.clone();
8016        match &mut push {
8017            ClientControlPush::RouteClosing {
8018                channels: covered, ..
8019            }
8020            | ClientControlPush::RouteClosed {
8021                channels: covered, ..
8022            } => *covered = channels,
8023        }
8024        let body = match serde_json::to_vec(&push) {
8025            Ok(body) => body,
8026            Err(err) => {
8027                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8028                continue;
8029            }
8030        };
8031        let frame = match Frame::build_with_version(
8032            target.negotiated_ver,
8033            FrameType::Push,
8034            control_flags(),
8035            0,
8036            0,
8037            0,
8038            body.clone(),
8039        ) {
8040            Ok(frame) => frame,
8041            Err(err) => {
8042                warn!(
8043                    route_channel = target.channel,
8044                    error = %err,
8045                    "failed to build route lifecycle control PUSH frame"
8046                );
8047                continue;
8048            }
8049        };
8050        if let Err(err) = target.sink.try_send(frame) {
8051            if target.close_on_delivery_failure() {
8052                warn!(
8053                    target_connection_id = target.connection_id.get(),
8054                    route_channel = target.channel,
8055                    error = %err,
8056                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8057                );
8058                let _ = forwarding.escalate_client_delivery_failure(
8059                    target.connection_id,
8060                    target.channel,
8061                    target.epoch,
8062                    CloseReason::new(
8063                        "route_lifecycle_push_delivery_failed",
8064                        format!(
8065                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8066                            target.channel
8067                        ),
8068                    ),
8069                    crate::forwarding::UndeliveredFrame {
8070                        module_id: target.module_id.as_deref(),
8071                        sink: &target.sink,
8072                    },
8073                );
8074            }
8075        }
8076    }
8077}
8078
8079#[cfg(test)]
8080mod tests {
8081    mod activity_read {
8082        include!("activity_read_tests.rs");
8083    }
8084
8085    mod lifecycle_activity {
8086        include!("control_lifecycle_tests.rs");
8087    }
8088    #[cfg(unix)]
8089    #[tokio::test]
8090    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8091        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8092        let path = dir.join("subc.jsonc");
8093        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8094            "program":"/bin/sleep","args":["60"],"protocol":"none",
8095            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8096        }}}).to_string()).unwrap();
8097        let mut configured = crate::daemon_config::load(&path)
8098            .unwrap()
8099            .unwrap()
8100            .modules
8101            .pop()
8102            .unwrap();
8103        let registry = std::sync::Arc::new(crate::Registry::default());
8104        let handle = crate::SupervisorHandle::new();
8105        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8106            .with_handle(handle.clone());
8107        let module = supervisor
8108            .supervise_configured_with_health(
8109                configured.module_spec(),
8110                true,
8111                configured.health.clone(),
8112                None,
8113                configured.restart,
8114            )
8115            .unwrap();
8116        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8117        let before = module.status().unwrap().pid;
8118        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8119        configured.health.cadence = std::time::Duration::from_secs(3600);
8120        let health_only = handler
8121            .reconcile_supervised_modules(
8122                &supervisor,
8123                vec![configured.clone()],
8124                false,
8125                Principal::Direct,
8126            )
8127            .await
8128            .unwrap();
8129        assert!(
8130            health_only.0.changed_pending_reload.is_empty(),
8131            "health policy is already applied live"
8132        );
8133        assert_eq!(module.status().unwrap().pid, before);
8134        assert_eq!(
8135            module.configuration().unwrap().1.http,
8136            configured.health.http
8137        );
8138        configured.args = vec!["61".into()];
8139        let launch = handler
8140            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8141            .await
8142            .unwrap();
8143        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8144        assert_eq!(
8145            module.status().unwrap().pid,
8146            before,
8147            "a launch edit is stored until reload"
8148        );
8149        module.drain().await.unwrap();
8150    }
8151    use cortexkit_test_support::ScratchDir;
8152    use std::{
8153        collections::BTreeMap,
8154        fmt,
8155        path::PathBuf,
8156        sync::{Arc, Mutex},
8157        time::Duration,
8158    };
8159
8160    use serde_json::{json, Value};
8161    use subc_protocol::{
8162        manifest::{
8163            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8164            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8165        },
8166        session::HealthStatus,
8167        FrameType,
8168    };
8169
8170    use super::*;
8171    use crate::{
8172        forwarding::{DataRoute, DataRouteState},
8173        registry::ChannelState,
8174        router::FrameSink,
8175        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8176        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8177        RouteCtx, Router,
8178    };
8179    use tokio::{
8180        sync::mpsc,
8181        time::{sleep, Instant},
8182    };
8183    use tracing::{
8184        field::{Field, Visit},
8185        Event, Subscriber,
8186    };
8187    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8188
8189    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8190    ///
8191    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8192    /// is only populated for `tests/*.rs` integration test binaries -- this file
8193    /// compiles as part of the library target, which gets neither. This test's
8194    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8195    /// and the sibling binary lives two directories up at
8196    /// `<target-dir>/<profile>/fake-aft-stub`.
8197    ///
8198    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8199    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8200    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8201    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8202    /// `NotFound`, which reads as a broken test rather than an unbuilt
8203    /// dependency -- so state the cause and the remedy instead. Deliberately a
8204    /// panic and not a silent skip: a test that quietly passes when it could not
8205    /// run is worse than one that fails, because it reports health it never
8206    /// verified.
8207    fn fake_aft_stub_path() -> PathBuf {
8208        let mut path = std::env::current_exe().expect("current_exe available in tests");
8209        path.pop(); // .../deps/
8210        path.pop(); // .../<profile>/
8211        path.push(if cfg!(windows) {
8212            "fake-aft-stub.exe"
8213        } else {
8214            "fake-aft-stub"
8215        });
8216        assert!(
8217            path.exists(),
8218            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8219             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8220            path.display()
8221        );
8222        path
8223    }
8224
8225    /// Whether clients retry `code` in place: the predicate itself, never a copy
8226    /// of its set. A copied list breaks silently when a code is added to or
8227    /// removed from the real one, and a stale copy here would let exactly the
8228    /// failure this test exists to catch pass.
8229    fn client_retries(code: &str) -> bool {
8230        subc_protocol::error_codes::is_retryable_route_open(code)
8231    }
8232
8233    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8234    /// of failure is worse than publishing none. A permanent fault dressed as
8235    /// retryable makes every client in the fleet retry forever against something
8236    /// that cannot recover; a transient fault dressed as permanent abandons work
8237    /// that would have succeeded.
8238    ///
8239    /// Asserting "a code exists" cannot catch either, because the string is free
8240    /// to say anything. This enumerates every variant and pins which side of the
8241    /// retry boundary it lands on, so a new variant must be classified here
8242    /// deliberately rather than inheriting whichever arm it was appended to.
8243    #[test]
8244    fn retryability_of_forwarding_codes_matches_the_failure() {
8245        // Transient by nature: the target is booting, reloading, or its endpoint
8246        // was swapped mid-flight. Retrying is how these resolve.
8247        let transient = [
8248            ForwardingError::NoModuleConnection,
8249            ForwardingError::ModuleReloading {
8250                module_id: "m".into(),
8251            },
8252            ForwardingError::StaleModuleEndpoint,
8253            ForwardingError::UnknownReservation {
8254                client_channel: 1,
8255                module_channel: 1,
8256            },
8257            ForwardingError::ConnectionClosing {
8258                connection_id: ConnectionId::new(1),
8259            },
8260            ForwardingError::ClientEgressClosed {
8261                connection_id: ConnectionId::new(1),
8262            },
8263            ForwardingError::ModuleEgressUnavailable {
8264                connection_id: ConnectionId::new(1),
8265            },
8266        ];
8267        for err in transient {
8268            let code = forwarding_error_code(&err);
8269            assert!(
8270                client_retries(code),
8271                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8272            );
8273        }
8274
8275        // Not fixed by retrying. Channel and correlation exhaustion need the
8276        // caller to close routes, and a poisoned lock is a daemon that cannot
8277        // recover at all — the worst thing to advertise as retryable, since every
8278        // client would storm a daemon that will never answer.
8279        let permanent = [
8280            ForwardingError::ConnectionRoleConflict {
8281                connection_id: ConnectionId::new(1),
8282            },
8283            ForwardingError::ClientRouteChannelExhausted {
8284                connection_id: ConnectionId::new(1),
8285            },
8286            ForwardingError::ModuleRouteChannelExhausted {
8287                endpoint: ModuleEndpointId {
8288                    connection_id: ConnectionId::new(1),
8289                    generation: 1,
8290                },
8291            },
8292            ForwardingError::RelayCorrelationExhausted,
8293            ForwardingError::RouteOpenBuild("x".into()),
8294            ForwardingError::Poisoned,
8295        ];
8296        for err in permanent {
8297            let code = forwarding_error_code(&err);
8298            assert!(
8299                !client_retries(code),
8300                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8301            );
8302        }
8303    }
8304
8305    /// The principal is the daemon's answer to "who is calling", and modules
8306    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8307    /// plexus gates connector invocation. So a stamp is an authorization input in
8308    /// another process, not a label — and both possible answers SUCCEED, which is
8309    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8310    /// first-party capability to something that never proved it; a supervised one
8311    /// stamped `Direct` silently strips a module of capability it is entitled to.
8312    ///
8313    /// Neither shows up in a test that only checks the bind succeeded. Before this
8314    /// test the only coverage was accidental —
8315    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8316    /// stamped principal on its way past, so narrowing that wire-shape test to its
8317    /// stated subject would have deleted the last assertion on this value. It
8318    /// still asserts the stamp, which is now redundancy rather than the only
8319    /// guard: both fail under the same mutation, and this one names the reason.
8320    /// SCOPE: this handler's supervisor has spawned nothing, so
8321    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8322    /// is unreachable here. Both assertions below are refusals, and a mutant that
8323    /// refuses everything would satisfy them.
8324    ///
8325    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8326    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8327    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8328    /// at source rather than assumed, since a citation is a claim about another
8329    /// file and ages like one. Recorded because a harness that structurally
8330    /// cannot reach an arm reports "none" for that arm identically to one that
8331    /// covers it and found nothing.
8332    #[tokio::test]
8333    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8334        let handler = ControlHandler::default();
8335        let frame =
8336            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8337
8338        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8339        // any process holding the connection file. Nothing was proved, so nothing
8340        // may be granted beyond the unattested floor.
8341        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8342        assert_eq!(
8343            stamped,
8344            Principal::Direct,
8345            "a caller that proved nothing must not be stamped as a supervised module"
8346        );
8347
8348        // A claimed module_id with a nonce no supervised child was given is a
8349        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8350        // quietly demoted to Direct, or an impersonation attempt looks identical
8351        // to an ordinary unattested connection.
8352        let forged = handler
8353            .route_open_principal(
8354                &frame,
8355                Some(ConsumerIdentity {
8356                    module_id: "aft".to_string(),
8357                    launch_nonce: "not-a-real-nonce".to_string(),
8358                }),
8359            )
8360            .unwrap();
8361        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8362        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8363    }
8364
8365    /// The test above hands `route_open_principal` an identity it built itself,
8366    /// which proves the stamping rule and nothing about where the identity comes
8367    /// from. The real producer is a wire body, and the two are joined by a serde
8368    /// field name that nothing else asserts.
8369    ///
8370    /// That join fails quietly in one specific way: an unrecognised key is simply
8371    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8372    /// yields `None` and every supervised module silently drops to `Direct`.
8373    /// Capability-wise that is the safe direction, but it surfaces far from its
8374    /// cause — as a module mysteriously refused bash — and it would pass every
8375    /// test that builds its own input.
8376    ///
8377    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8378    /// would break every client the moment the daemon gains a field, trading a
8379    /// quiet demotion for a hard refusal on additive change. Asserting the join
8380    /// instead means a rename breaks a test here rather than the fleet.
8381    #[test]
8382    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8383        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8384        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8385        let ClientControlRequest::RouteOpen {
8386            consumer_identity, ..
8387        } = parsed
8388        else {
8389            panic!("route.open body must parse as RouteOpen");
8390        };
8391        assert_eq!(
8392            consumer_identity,
8393            Some(ConsumerIdentity {
8394                module_id: "aft".to_string(),
8395                launch_nonce: "n".to_string(),
8396            }),
8397            "the wire field name must reach the value route_open_principal reads"
8398        );
8399    }
8400
8401    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8402        ModuleManifest::builder(module_id, "0.1.0")
8403            .protocol_ver(protocol_ver)
8404            .provides(vec![ProviderRole::ToolProvider {
8405                tools: vec![Tool {
8406                    name: "read".to_string(),
8407                    description: None,
8408                    execution_mode: ExecutionMode::Pure,
8409                    schema: json!({"type": "object"}),
8410                }],
8411                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8412                concurrency: Concurrency::ModuleManaged,
8413                emits_push: true,
8414                sub_supervises: true,
8415            }])
8416            .build()
8417    }
8418
8419    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8420        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8421    }
8422
8423    fn hello_frame_with_control_ops(
8424        module_id: &str,
8425        protocol_ver: u8,
8426        corr: u64,
8427        control_ops: Option<Vec<String>>,
8428    ) -> Frame {
8429        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8430    }
8431
8432    fn hello_frame_with_nonce(
8433        module_id: &str,
8434        protocol_ver: u8,
8435        corr: u64,
8436        launch_nonce: Option<&str>,
8437    ) -> Frame {
8438        hello_frame_full(
8439            module_id,
8440            protocol_ver,
8441            corr,
8442            None,
8443            launch_nonce.map(ToOwned::to_owned),
8444        )
8445    }
8446
8447    fn hello_frame_full(
8448        module_id: &str,
8449        protocol_ver: u8,
8450        corr: u64,
8451        control_ops: Option<Vec<String>>,
8452        launch_nonce: Option<String>,
8453    ) -> Frame {
8454        let body = serde_json::to_vec(&ModuleHelloBody {
8455            manifest: manifest(module_id, protocol_ver),
8456            protocol_ver,
8457            control_ops,
8458            launch_nonce,
8459        })
8460        .unwrap();
8461        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8462    }
8463
8464    fn hello_frame_with_rich_ui(
8465        module_id: &str,
8466        protocol_ver: u8,
8467        corr: u64,
8468        launch_nonce: Option<&str>,
8469        rich_ui: Value,
8470    ) -> Frame {
8471        let mut body = serde_json::to_value(ModuleHelloBody {
8472            manifest: manifest(module_id, protocol_ver),
8473            protocol_ver,
8474            control_ops: None,
8475            launch_nonce: launch_nonce.map(str::to_string),
8476        })
8477        .unwrap();
8478        body["manifest"]["rich_ui"] = rich_ui;
8479        Frame::build(
8480            FrameType::Hello,
8481            control_flags(),
8482            0,
8483            0,
8484            corr,
8485            serde_json::to_vec(&body).unwrap(),
8486        )
8487        .unwrap()
8488    }
8489
8490    fn non_routable_hello_frame_with_control_ops(
8491        module_id: &str,
8492        corr: u64,
8493        control_ops: Option<Vec<String>>,
8494    ) -> Frame {
8495        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8496        manifest.provides.clear();
8497        let body = serde_json::to_vec(&ModuleHelloBody {
8498            manifest,
8499            protocol_ver: PROTOCOL_VERSION,
8500            control_ops,
8501            launch_nonce: None,
8502        })
8503        .unwrap();
8504        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8505    }
8506
8507    fn capability_grammar_hello_frame(
8508        capabilities: Value,
8509        runtime_computed: Option<Value>,
8510        corr: u64,
8511    ) -> Frame {
8512        let mut body = serde_json::to_value(ModuleHelloBody {
8513            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8514            protocol_ver: PROTOCOL_VERSION,
8515            control_ops: None,
8516            launch_nonce: None,
8517        })
8518        .expect("HELLO body serializes");
8519        body["manifest"]["capabilities"] = capabilities;
8520        if let Some(runtime_computed) = runtime_computed {
8521            body["runtime_computed"] = runtime_computed;
8522        }
8523        Frame::build(
8524            FrameType::Hello,
8525            control_flags(),
8526            0,
8527            0,
8528            corr,
8529            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8530        )
8531        .expect("HELLO frame builds")
8532    }
8533
8534    fn channel_request(channel: u16, corr: u64) -> Frame {
8535        Frame::build(
8536            FrameType::Request,
8537            Flags::new(true, Priority::Interactive, false),
8538            channel,
8539            0,
8540            corr,
8541            b"opaque".to_vec(),
8542        )
8543        .unwrap()
8544    }
8545
8546    fn route_ctx(
8547        connection_id: ConnectionId,
8548    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8549        let (tx, rx) = mpsc::channel(8);
8550        (
8551            RouteCtx {
8552                connection_id,
8553                egress: FrameSink::new(tx),
8554            },
8555            rx,
8556        )
8557    }
8558
8559    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8560        serde_json::from_slice(&frame.body).unwrap()
8561    }
8562
8563    /// Register a module over a connection that has a sink and return the
8564    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8565    /// module's own sink rather than returning it as a reply, so the ack is
8566    /// taken off `rx` here and whatever the test reads next is what followed it.
8567    async fn hello_via_sink(
8568        handler: &ControlHandler,
8569        ctx: &RouteCtx,
8570        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8571        hello: Frame,
8572    ) -> Frame {
8573        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8574        assert!(
8575            replies.is_empty(),
8576            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8577        );
8578        let ack = rx
8579            .try_recv()
8580            .expect("HELLO_ACK is queued on the module sink")
8581            .frame;
8582        assert_eq!(ack.header.ty, FrameType::HelloAck);
8583        ack
8584    }
8585
8586    fn parse_error(frame: &Frame) -> Value {
8587        serde_json::from_slice(&frame.body).unwrap()
8588    }
8589
8590    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8591        serde_json::from_slice(&frame.body).unwrap()
8592    }
8593
8594    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8595        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8596            route_channel,
8597            route_epoch: 0,
8598            kind,
8599        })
8600        .unwrap();
8601        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8602    }
8603
8604    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8605        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8606            module_id: module_id.to_string(),
8607        })
8608        .unwrap();
8609        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8610    }
8611
8612    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8613        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8614    }
8615
8616    fn route_open_frame_with_consumer_capabilities(
8617        corr: u64,
8618        module_id: &str,
8619        project_root: ScratchDir,
8620        consumer_capabilities: Option<Vec<String>>,
8621    ) -> Frame {
8622        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8623            target: RouteTarget::ToolProvider {
8624                module_id: module_id.to_string(),
8625            },
8626            identity: BindIdentity::new(
8627                project_root.path().to_path_buf(),
8628                "unit".to_string(),
8629                "session".to_string(),
8630            ),
8631            consumer_identity: None,
8632            consumer_capabilities,
8633            role_versions: None,
8634            admission_facts: None,
8635            scope: None,
8636        })
8637        .unwrap();
8638        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8639    }
8640
8641    fn route_open_frame_with_role_versions(
8642        corr: u64,
8643        module_id: &str,
8644        project_root: ScratchDir,
8645        role_versions: Option<BTreeMap<String, String>>,
8646    ) -> Frame {
8647        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8648            target: RouteTarget::ToolProvider {
8649                module_id: module_id.to_string(),
8650            },
8651            identity: BindIdentity::new(
8652                project_root.path().to_path_buf(),
8653                "unit".to_string(),
8654                format!("session-{corr}"),
8655            ),
8656            consumer_identity: None,
8657            consumer_capabilities: None,
8658            role_versions,
8659            admission_facts: None,
8660            scope: None,
8661        })
8662        .unwrap();
8663        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8664    }
8665
8666    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8667        entries
8668            .iter()
8669            .map(|(role, version)| (role.to_string(), version.to_string()))
8670            .collect()
8671    }
8672
8673    fn route_open_frame_with_admission_facts(
8674        corr: u64,
8675        module_id: &str,
8676        project_root: ScratchDir,
8677        consumer_identity: Option<subc_control::ConsumerIdentity>,
8678        facts: Option<Value>,
8679    ) -> Frame {
8680        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8681            target: RouteTarget::ToolProvider {
8682                module_id: module_id.to_string(),
8683            },
8684            identity: BindIdentity::new(
8685                project_root.path().to_path_buf(),
8686                "unit".to_string(),
8687                format!("session-{corr}"),
8688            ),
8689            consumer_identity,
8690            consumer_capabilities: None,
8691            role_versions: None,
8692            admission_facts: facts,
8693            scope: None,
8694        })
8695        .unwrap();
8696        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8697    }
8698
8699    #[derive(Clone, Default)]
8700    struct EventCapture {
8701        events: Arc<Mutex<Vec<CapturedEvent>>>,
8702    }
8703
8704    #[derive(Clone, Debug)]
8705    struct CapturedEvent {
8706        target: String,
8707        level: tracing::Level,
8708        fields: BTreeMap<String, String>,
8709    }
8710
8711    impl EventCapture {
8712        fn events(&self) -> Vec<CapturedEvent> {
8713            self.events.lock().unwrap().clone()
8714        }
8715    }
8716
8717    impl<S> Layer<S> for EventCapture
8718    where
8719        S: Subscriber,
8720    {
8721        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8722            let mut visitor = EventFieldVisitor::default();
8723            event.record(&mut visitor);
8724            self.events.lock().unwrap().push(CapturedEvent {
8725                target: event.metadata().target().to_string(),
8726                level: *event.metadata().level(),
8727                fields: visitor.fields,
8728            });
8729        }
8730    }
8731
8732    #[derive(Default)]
8733    struct EventFieldVisitor {
8734        fields: BTreeMap<String, String>,
8735    }
8736
8737    impl Visit for EventFieldVisitor {
8738        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8739            self.fields
8740                .insert(field.name().to_string(), format!("{value:?}"));
8741        }
8742    }
8743
8744    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8745        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8746            status,
8747            detail: Some("warming".to_string()),
8748            metrics: Some(json!({"queue_depth": 3})),
8749        })
8750        .unwrap();
8751        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8752    }
8753
8754    fn route_bind_ack(corr: u64) -> Frame {
8755        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8756        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8757    }
8758
8759    fn unique_project_root(label: &str) -> ScratchDir {
8760        ScratchDir::new(label)
8761    }
8762
8763    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8764        match parse_route_poll(frame) {
8765            ClientControlResponse::RoutePoll {
8766                status: None,
8767                live: Some(live),
8768                ..
8769            } => assert_eq!(live, expected_live),
8770            other => panic!("unexpected route.poll response: {other:?}"),
8771        }
8772    }
8773
8774    fn bind_liveness_route(
8775        registry: &Registry,
8776        forwarding: &ForwardingTable,
8777        module_id: &str,
8778    ) -> (RouteCtx, u16, u32) {
8779        let module_connection = ConnectionId::new(101);
8780        let client_connection = ConnectionId::new(202);
8781        let registration = registry
8782            .register_with_control_ops(
8783                manifest(module_id, PROTOCOL_VERSION),
8784                PROTOCOL_VERSION,
8785                module_connection,
8786                module_baseline_control_ops(),
8787            )
8788            .unwrap();
8789        let (module_tx, _module_rx) = mpsc::channel(8);
8790        let endpoint = forwarding
8791            .register_module_connection(
8792                module_connection,
8793                module_id.to_string(),
8794                PROTOCOL_VERSION,
8795                manifest_concurrency(&registration.manifest),
8796                FrameSink::new(module_tx),
8797            )
8798            .unwrap();
8799        let (client_ctx, _client_rx) = route_ctx(client_connection);
8800        let pending = forwarding
8801            .begin_route_bind_relay_for_test(
8802                client_connection,
8803                client_ctx.egress.clone(),
8804                1,
8805                module_id,
8806            )
8807            .unwrap();
8808        assert_eq!(pending.endpoint, endpoint);
8809        let route_channel = pending.client_channel;
8810        let route_epoch = pending.client_epoch;
8811        forwarding
8812            .complete_pending_relay(
8813                module_connection,
8814                pending.corr,
8815                RouteBindRelayOutcome::Accepted,
8816            )
8817            .unwrap();
8818        (client_ctx, route_channel, route_epoch)
8819    }
8820
8821    struct FakeProcessLiveness {
8822        live: Option<bool>,
8823    }
8824
8825    impl ModuleProcessLiveness for FakeProcessLiveness {
8826        fn process_live(&self, _module_id: &str) -> Option<bool> {
8827            self.live
8828        }
8829    }
8830
8831    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8832    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8833    {
8834        let registry = Arc::new(Registry::default());
8835        let supervisor_handle = SupervisorHandle::new();
8836        let supervisor = Supervisor::new_for_test(
8837            Arc::clone(&registry),
8838            RestartPolicy::new(1, Duration::from_millis(10)),
8839        )
8840        .with_handle(supervisor_handle.clone());
8841        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8842        let module = supervisor
8843            .spawn(ModuleSpec {
8844                module_id: "stderr-tail-wire".to_string(),
8845                program: fake_aft_stub_path(),
8846                args: Vec::new(),
8847                env: vec![
8848                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8849                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8850                ],
8851                reserved: false,
8852                reserved_prefixes: Vec::new(),
8853                protocol: ModuleProtocol::Subc,
8854                overlap: Default::default(),
8855            })
8856            .unwrap();
8857
8858        let deadline = Instant::now() + Duration::from_secs(5);
8859        loop {
8860            let tail = module.stderr_tail(None, None);
8861            if tail
8862                .entries
8863                .iter()
8864                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8865                && tail.entries.iter().any(|entry| {
8866                    matches!(
8867                        entry,
8868                        TailEntry::Line {
8869                            truncated: true,
8870                            ..
8871                        }
8872                    )
8873                })
8874            {
8875                break;
8876            }
8877            assert!(
8878                Instant::now() < deadline,
8879                "module did not produce a truncated line and restart boundary: {tail:?}"
8880            );
8881            sleep(Duration::from_millis(10)).await;
8882        }
8883
8884        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8885        let request = ClientControlRequest::SupervisorStderrTail {
8886            module_id: "stderr-tail-wire".to_string(),
8887            max_lines: None,
8888            max_bytes: None,
8889        };
8890        let frame = Frame::build(
8891            FrameType::Request,
8892            control_flags(),
8893            0,
8894            0,
8895            1,
8896            serde_json::to_vec(&request).unwrap(),
8897        )
8898        .unwrap();
8899        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8900        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8901        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8902            serde_json::from_slice(&responses[0].body).unwrap()
8903        else {
8904            panic!("expected supervisor.stderr_tail response");
8905        };
8906
8907        assert!(
8908            tail.entries
8909                .iter()
8910                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8911            "the control response lost the restart boundary"
8912        );
8913        let Some(StderrTailEntry::Line {
8914            text,
8915            truncated,
8916            at_ms,
8917        }) = tail.entries.iter().find(|entry| {
8918            matches!(
8919                entry,
8920                StderrTailEntry::Line {
8921                    truncated: true,
8922                    ..
8923                }
8924            )
8925        })
8926        else {
8927            panic!("the control response lost the truncated line");
8928        };
8929        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8930        assert!(*truncated);
8931        assert!(
8932            at_ms.is_some(),
8933            "the control response lost the line's capture time"
8934        );
8935    }
8936
8937    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8938    /// read done on the worker thread would stall every other task until it
8939    /// finished; the read must run off the worker so this test's own task keeps
8940    /// running while the read is paused.
8941    #[tokio::test(flavor = "current_thread")]
8942    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8943        let dir = ScratchDir::new("terminals-off-worker");
8944        let journal_path = dir.join("terminals.jsonl");
8945        let registry = Arc::new(Registry::default());
8946        let supervisor_handle = SupervisorHandle::new();
8947        let supervisor =
8948            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8949                .with_handle(supervisor_handle.clone())
8950                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8951        let module = supervisor
8952            .spawn(ModuleSpec {
8953                module_id: "terminal-off-worker".to_string(),
8954                program: fake_aft_stub_path(),
8955                args: Vec::new(),
8956                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8957                reserved: false,
8958                reserved_prefixes: Vec::new(),
8959                protocol: ModuleProtocol::Subc,
8960                overlap: Default::default(),
8961            })
8962            .unwrap();
8963        let deadline = Instant::now() + Duration::from_secs(5);
8964        while module.terminal_history().entries.len() != 2 {
8965            assert!(Instant::now() < deadline, "module did not record two exits");
8966            sleep(Duration::from_millis(10)).await;
8967        }
8968
8969        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8970        let handler =
8971            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8972        let frame = Frame::build(
8973            FrameType::Request,
8974            control_flags(),
8975            0,
8976            0,
8977            1,
8978            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8979                module_id: "terminal-off-worker".to_string(),
8980            })
8981            .unwrap(),
8982        )
8983        .unwrap();
8984        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8985        let spawned_at = std::time::Instant::now();
8986        let read = tokio::spawn({
8987            let handler = Arc::clone(&handler);
8988            async move { handler.handle_control_frame(&ctx, frame).await }
8989        });
8990        // Waiting for the pause from a blocking thread keeps this task pending,
8991        // so the runtime's single worker is free to run the read task.
8992        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8993            .await
8994            .unwrap()
8995            .expect("the history read reached its pause");
8996        let elapsed = spawned_at.elapsed();
8997        assert!(
8998            elapsed < Duration::from_secs(2) && !read.is_finished(),
8999            "this task could not run while the history read was paused \
9000             (resumed after {elapsed:?}, read finished: {})",
9001            read.is_finished()
9002        );
9003
9004        drop(release);
9005        let responses = read.await.unwrap().unwrap();
9006        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9007        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9008            panic!("expected supervisor.terminals response");
9009        };
9010        assert_eq!(terminals.entries.len(), 2);
9011        assert_eq!(terminals.journal_skipped_lines, 0);
9012        assert_eq!(terminals.journal_read_errors, 0);
9013    }
9014
9015    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9016    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9017        let registry = Arc::new(Registry::default());
9018        let supervisor_handle = SupervisorHandle::new();
9019        let supervisor =
9020            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9021                .with_handle(supervisor_handle.clone());
9022        let module = supervisor
9023            .spawn(ModuleSpec {
9024                module_id: "terminal-golden".to_string(),
9025                program: fake_aft_stub_path(),
9026                args: Vec::new(),
9027                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9028                reserved: false,
9029                reserved_prefixes: Vec::new(),
9030                protocol: ModuleProtocol::Subc,
9031                overlap: Default::default(),
9032            })
9033            .unwrap();
9034
9035        let deadline = Instant::now() + Duration::from_secs(5);
9036        while module.terminal_history().entries.len() != 2 {
9037            assert!(
9038                Instant::now() < deadline,
9039                "module did not retain two terminal exits: {:?}",
9040                module.terminal_history()
9041            );
9042            sleep(Duration::from_millis(10)).await;
9043        }
9044
9045        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9046        let request = ClientControlRequest::SupervisorTerminals {
9047            module_id: "terminal-golden".to_string(),
9048        };
9049        let frame = Frame::build(
9050            FrameType::Request,
9051            control_flags(),
9052            0,
9053            0,
9054            1,
9055            serde_json::to_vec(&request).unwrap(),
9056        )
9057        .unwrap();
9058        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9059        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9060        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9061        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9062            panic!("expected supervisor.terminals response");
9063        };
9064        assert_eq!(terminals.entries.len(), 2);
9065        assert_eq!(terminals.dropped, 0);
9066
9067        let mut rendered = serde_json::to_value(response).unwrap();
9068        // Wall-clock fields are the observation contract, but not stable fixture
9069        // bytes; normalize only them after the real handler has shaped the response.
9070        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9071        for (index, entry) in rendered["entries"]
9072            .as_array_mut()
9073            .expect("terminal response entries array")
9074            .iter_mut()
9075            .enumerate()
9076        {
9077            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9078        }
9079
9080        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9081            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9082        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9083        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9084            std::fs::write(&golden_path, &serialized).unwrap();
9085        }
9086        let expected: Value =
9087            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9088        assert_eq!(rendered, expected);
9089    }
9090
9091    #[test]
9092    fn hello_registers_manifest_and_returns_ack() {
9093        let registry = Arc::new(Registry::default());
9094        let handler = ControlHandler::new(Arc::clone(&registry));
9095        let conn = ConnectionId::new(1);
9096
9097        let responses = handler
9098            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9099            .unwrap();
9100
9101        assert_eq!(responses.len(), 1);
9102        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9103        assert_eq!(responses[0].header.channel, 0);
9104        assert_eq!(responses[0].header.corr, 7);
9105        let ack = parse_ack(&responses[0]);
9106        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9107        assert!(ack
9108            .subc_capabilities
9109            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9110        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9111        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9112        assert!(ack
9113            .subc_ops
9114            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9115        assert!(ack
9116            .subc_ops
9117            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9118
9119        let registration = registry.get_module("aft").unwrap().unwrap();
9120        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9121        assert_eq!(registration.state, ChannelState::Active);
9122        assert_eq!(registration.connection_id, conn);
9123        assert_eq!(registration.control_ops, module_baseline_control_ops());
9124    }
9125
9126    #[test]
9127    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9128        let invalid_identifiers = [
9129            ("case_change", "credentials-Provider/v1"),
9130            ("leading_zero", "credentials-provider/v01"),
9131            ("trailing_hyphen", "credentials-provider-/v1"),
9132            ("consecutive_hyphens", "credentials--provider/v1"),
9133            ("uppercase", "Credentials-provider/v1"),
9134            ("missing_v", "credentials-provider/1"),
9135            ("whitespace", "credentials provider/v1"),
9136            ("zero_version", "credentials-provider/v0"),
9137            ("out_of_range_version", "credentials-provider/v4294967296"),
9138            (
9139                "overlength_name",
9140                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9141            ),
9142        ];
9143        let mut cases = invalid_identifiers
9144            .into_iter()
9145            .map(|(name, identifier)| {
9146                (
9147                    format!("identifier_{name}"),
9148                    "capabilities.provides[0]".to_string(),
9149                    identifier.to_string(),
9150                    json!({ "provides": [identifier] }),
9151                    None,
9152                )
9153            })
9154            .collect::<Vec<_>>();
9155        cases.extend([
9156            (
9157                "unknown_need".to_string(),
9158                "capabilities.requires[0].need".to_string(),
9159                "deferred".to_string(),
9160                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9161                None,
9162            ),
9163            (
9164                "duplicate_provides".to_string(),
9165                "capabilities.provides[1]".to_string(),
9166                "credentials-provider/v1".to_string(),
9167                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9168                None,
9169            ),
9170            (
9171                "duplicate_must_never_reach".to_string(),
9172                "capabilities.must_never_reach[1]".to_string(),
9173                "credentials-provider/v1".to_string(),
9174                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9175                None,
9176            ),
9177            (
9178                "duplicate_requires_same_need".to_string(),
9179                "capabilities.requires[1]".to_string(),
9180                "credentials-provider/v1".to_string(),
9181                json!({ "requires": [
9182                    { "capability": "credentials-provider/v1", "need": "required" },
9183                    { "capability": "credentials-provider/v1", "need": "required" }
9184                ] }),
9185                None,
9186            ),
9187            (
9188                "duplicate_requires_conflicting_need".to_string(),
9189                "capabilities.requires[1]".to_string(),
9190                "credentials-provider/v1".to_string(),
9191                json!({ "requires": [
9192                    { "capability": "credentials-provider/v1", "need": "required" },
9193                    { "capability": "credentials-provider/v1", "need": "optional" }
9194                ] }),
9195                None,
9196            ),
9197            (
9198                "capabilities_root_pointer".to_string(),
9199                "runtime_computed[0]".to_string(),
9200                "/capabilities".to_string(),
9201                json!({}),
9202                Some(json!(["/capabilities"])),
9203            ),
9204            (
9205                "capabilities_descendant_pointer".to_string(),
9206                "runtime_computed[0]".to_string(),
9207                "/capabilities/provides".to_string(),
9208                json!({}),
9209                Some(json!(["/capabilities/provides"])),
9210            ),
9211            (
9212                "malformed_pointer_without_leading_slash".to_string(),
9213                "runtime_computed[0]".to_string(),
9214                "capabilities".to_string(),
9215                json!({}),
9216                Some(json!(["capabilities"])),
9217            ),
9218            (
9219                "malformed_pointer_escape".to_string(),
9220                "runtime_computed[0]".to_string(),
9221                "/roles/~2/tools".to_string(),
9222                json!({}),
9223                Some(json!(["/roles/~2/tools"])),
9224            ),
9225            (
9226                "unknown_capabilities_field".to_string(),
9227                "capabilities.future".to_string(),
9228                "<array>".to_string(),
9229                json!({ "future": [] }),
9230                None,
9231            ),
9232        ]);
9233
9234        for (index, (name, field, value, capabilities, runtime_computed)) in
9235            cases.into_iter().enumerate()
9236        {
9237            let registry = Arc::new(Registry::default());
9238            let handler = ControlHandler::new(Arc::clone(&registry));
9239            let response = handler
9240                .handle_control(
9241                    ConnectionId::new((index + 1) as u64),
9242                    capability_grammar_hello_frame(
9243                        capabilities,
9244                        runtime_computed,
9245                        index as u64 + 1,
9246                    ),
9247                )
9248                .expect("invalid HELLO returns a refusal");
9249
9250            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9251            let error = parse_error(&response[0]);
9252            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9253            let message = error["message"]
9254                .as_str()
9255                .expect("error message is a string");
9256            assert!(
9257                message.contains(&field),
9258                "{name}: field missing from {message}"
9259            );
9260            assert!(
9261                message.contains(&value),
9262                "{name}: value missing from {message}"
9263            );
9264            assert_eq!(
9265                registry
9266                    .active_registration_count()
9267                    .expect("registry reads"),
9268                0,
9269                "{name}: refused HELLO must not create a catalog entry"
9270            );
9271        }
9272    }
9273
9274    #[test]
9275    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9276        let registry = Arc::new(Registry::default());
9277        let handler = ControlHandler::new(Arc::clone(&registry));
9278        let capabilities = json!({
9279            "provides": ["credentials-provider/v1"],
9280            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9281            "must_never_reach": ["federation-transport/v1"]
9282        });
9283        let response = handler
9284            .handle_control(
9285                ConnectionId::new(99),
9286                capability_grammar_hello_frame(
9287                    capabilities.clone(),
9288                    Some(json!(["/roles/0/tools"])),
9289                    99,
9290                ),
9291            )
9292            .expect("valid HELLO registers");
9293        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9294
9295        let request = Frame::build(
9296            FrameType::Request,
9297            control_flags(),
9298            0,
9299            0,
9300            100,
9301            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9302                .expect("catalog request serializes"),
9303        )
9304        .expect("catalog request frame builds");
9305        let response = handler
9306            .handle_catalog_list(request, None)
9307            .expect("catalog list succeeds");
9308        let ClientControlResponse::CatalogList { modules, .. } =
9309            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9310        else {
9311            panic!("catalog request must return catalog.list");
9312        };
9313        assert_eq!(modules.len(), 1);
9314        assert_eq!(
9315            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9316            capabilities
9317        );
9318    }
9319
9320    #[test]
9321    fn catalog_list_mirrors_management_operation_description() {
9322        let registry = Arc::new(Registry::default());
9323        let handler = ControlHandler::new(Arc::clone(&registry));
9324        let description = "List managed records and return their identifiers and metadata.";
9325        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9326        manifest.provides = vec![ProviderRole::ManagementSurface {
9327            operations: vec![ManagementOperation {
9328                name: "records.list".to_string(),
9329                kind: ManagementOperationKind::Query,
9330                description: Some(description.to_string()),
9331            }],
9332            config_schema: json!({"type": "object"}),
9333            observability: vec![ObservabilitySurface {
9334                name: "records.stats".to_string(),
9335                kind: ObservabilityKind::Snapshot,
9336            }],
9337            identity_scope: vec![IdentityScope::Project],
9338            concurrency: Concurrency::ModuleManaged,
9339        }];
9340        registry
9341            .register_with_control_ops(
9342                manifest,
9343                PROTOCOL_VERSION,
9344                ConnectionId::new(99),
9345                Vec::new(),
9346            )
9347            .expect("described management manifest registers");
9348
9349        let request = Frame::build(
9350            FrameType::Request,
9351            control_flags(),
9352            0,
9353            0,
9354            100,
9355            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9356                .expect("catalog request serializes"),
9357        )
9358        .expect("catalog request frame builds");
9359        let response = handler
9360            .handle_catalog_list(request, None)
9361            .expect("catalog list succeeds");
9362        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9363        assert_eq!(
9364            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9365            "catalog.list must preserve the declared operation description verbatim"
9366        );
9367    }
9368
9369    #[test]
9370    fn catalog_list_mirrors_registered_operation_delivery() {
9371        let registry = Arc::new(Registry::default());
9372        let handler = ControlHandler::new(Arc::clone(&registry));
9373        let delivery =
9374            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9375        let mut body = serde_json::to_value(ModuleHelloBody {
9376            manifest: manifest("watch-management", PROTOCOL_VERSION),
9377            protocol_ver: PROTOCOL_VERSION,
9378            control_ops: None,
9379            launch_nonce: None,
9380        })
9381        .unwrap();
9382        body["manifest"]["provides"] = json!([{
9383            "role": "management_surface",
9384            "operations": [
9385                {"name": "status", "kind": "query"},
9386                {"name": "future", "kind": "query"},
9387                {"name": "reset", "kind": "mutate"}
9388            ],
9389            "config_schema": {}, "observability": [], "identity_scope": []
9390        }]);
9391        body["manifest"]["operation_delivery"] = delivery.clone();
9392        let hello = Frame::build(
9393            FrameType::Hello,
9394            control_flags(),
9395            0,
9396            0,
9397            99,
9398            serde_json::to_vec(&body).unwrap(),
9399        )
9400        .unwrap();
9401        let response = handler
9402            .handle_control(ConnectionId::new(99), hello)
9403            .unwrap();
9404        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9405
9406        let (_, registrations) = registry.list_modules().unwrap();
9407        let registered = &registrations[0].manifest;
9408        assert!(registered.is_watch_operation("status"));
9409        assert!(!registered.is_watch_operation("future"));
9410        assert!(!registered.is_watch_operation("reset"));
9411        assert!(!registered.is_watch_operation("missing"));
9412
9413        let request = Frame::build(
9414            FrameType::Request,
9415            control_flags(),
9416            0,
9417            0,
9418            100,
9419            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9420        )
9421        .unwrap();
9422        let response = handler.handle_catalog_list(request, None).unwrap();
9423        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9424        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9425        let ClientControlResponse::CatalogList { modules, .. } =
9426            serde_json::from_slice(&response[0].body).unwrap()
9427        else {
9428            panic!("expected catalog.list");
9429        };
9430        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9431    }
9432
9433    #[test]
9434    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9435        let registry = Arc::new(Registry::default());
9436        let supervisor = SupervisorHandle::new();
9437        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9438        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9439        let descriptor = json!({
9440            "schema": "ckios.richui.manifest/1",
9441            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9442            "bytes": 4096,
9443            "chunks": 4
9444        });
9445        let connection = ConnectionId::new(201);
9446        let hello = handler
9447            .handle_control(
9448                connection,
9449                hello_frame_with_rich_ui(
9450                    "vault",
9451                    PROTOCOL_VERSION,
9452                    201,
9453                    Some("the-real-nonce"),
9454                    descriptor.clone(),
9455                ),
9456            )
9457            .unwrap();
9458        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9459
9460        let request =
9461            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9462        let provides = registry
9463            .get_module_by_connection(connection)
9464            .unwrap()
9465            .unwrap()
9466            .manifest
9467            .provides;
9468        handler
9469            .handle_catalog_update(connection, request, provides, None, Some(false))
9470            .unwrap();
9471
9472        let request = Frame::build(
9473            FrameType::Request,
9474            control_flags(),
9475            0,
9476            0,
9477            203,
9478            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9479        )
9480        .unwrap();
9481        let response = handler.handle_catalog_list(request, None).unwrap();
9482        let ClientControlResponse::CatalogList { modules, .. } =
9483            serde_json::from_slice(&response[0].body).unwrap()
9484        else {
9485            panic!("expected catalog.list");
9486        };
9487        assert_eq!(
9488            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9489            descriptor
9490        );
9491        assert!(modules[0].nonce_verified);
9492    }
9493
9494    fn hello_with_role_declarations(declarations: Value) -> Frame {
9495        let mut body = serde_json::to_value(ModuleHelloBody {
9496            manifest: manifest("role-test", PROTOCOL_VERSION),
9497            protocol_ver: PROTOCOL_VERSION,
9498            control_ops: None,
9499            launch_nonce: None,
9500        })
9501        .unwrap();
9502        body["manifest"]["role_declarations"] = declarations;
9503        Frame::build(
9504            FrameType::Hello,
9505            control_flags(),
9506            0,
9507            0,
9508            301,
9509            serde_json::to_vec(&body).unwrap(),
9510        )
9511        .unwrap()
9512    }
9513
9514    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9515        let request = Frame::build(
9516            FrameType::Request,
9517            control_flags(),
9518            0,
9519            0,
9520            302,
9521            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9522        )
9523        .unwrap();
9524        let response = handler.handle_catalog_list(request, None).unwrap();
9525        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9526        let ClientControlResponse::CatalogList { mut modules, .. } =
9527            serde_json::from_slice(&response[0].body).unwrap()
9528        else {
9529            panic!("expected catalog.list");
9530        };
9531        (raw["modules"][0].clone(), modules.remove(0))
9532    }
9533
9534    #[test]
9535    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9536        let registry = Arc::new(Registry::default());
9537        let handler = ControlHandler::new(Arc::clone(&registry));
9538        let declarations = json!({
9539            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9540            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9541        });
9542        let connection = ConnectionId::new(301);
9543        let response = handler
9544            .handle_control(
9545                connection,
9546                hello_with_role_declarations(declarations.clone()),
9547            )
9548            .unwrap();
9549        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9550        let registered = registry
9551            .get_module_by_connection(connection)
9552            .unwrap()
9553            .unwrap()
9554            .manifest;
9555        // Compared as JSON values, not bytes: object key order is not part of
9556        // the relay's promise, and workspace builds unify serde_json's
9557        // preserve_order feature, which changes the order a literal keeps.
9558        assert_eq!(
9559            serde_json::to_value(&registered.role_declarations).unwrap(),
9560            declarations
9561        );
9562        let (raw, entry) = role_declarations_catalog(&handler);
9563        assert_eq!(raw["role_declarations"], declarations);
9564        assert_eq!(entry.role_declarations, registered.role_declarations);
9565
9566        let request =
9567            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9568        handler
9569            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9570            .unwrap();
9571        let (raw, entry) = role_declarations_catalog(&handler);
9572        assert_eq!(raw["role_declarations"], declarations);
9573        assert_eq!(entry.role_declarations, registered.role_declarations);
9574    }
9575
9576    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9577        let registry = Arc::new(Registry::default());
9578        let handler = ControlHandler::new(Arc::clone(&registry));
9579        let capture = EventCapture::default();
9580        let _subscriber =
9581            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9582        let response = handler
9583            .handle_control(
9584                ConnectionId::new(301),
9585                hello_with_role_declarations(declarations),
9586            )
9587            .unwrap();
9588        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9589        assert!(registry
9590            .get_module_by_connection(ConnectionId::new(301))
9591            .unwrap()
9592            .unwrap()
9593            .manifest
9594            .role_declarations
9595            .is_empty());
9596        let (raw, entry) = role_declarations_catalog(&handler);
9597        assert!(raw.get("role_declarations").is_none());
9598        assert!(entry.role_declarations.is_empty());
9599        let warnings: Vec<_> = capture
9600            .events()
9601            .into_iter()
9602            .filter(|event| {
9603                event.fields.get("message").map(String::as_str)
9604                    == Some("dropping whole role_declarations map from HELLO")
9605            })
9606            .collect();
9607        assert_eq!(warnings.len(), 1);
9608        assert_eq!(warnings[0].level, tracing::Level::WARN);
9609        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9610        assert_eq!(warnings[0].fields["reason"], reason);
9611    }
9612
9613    #[test]
9614    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9615        for value in [
9616            json!(null),
9617            json!(false),
9618            json!(0),
9619            json!("opaque"),
9620            json!([]),
9621        ] {
9622            assert_hello_drops_role_declarations_with_warning(
9623                json!({"routing/v1": {}, "broken/v1": value}),
9624                "role_declarations values must be objects",
9625            );
9626        }
9627        assert_hello_drops_role_declarations_with_warning(
9628            json!([]),
9629            "role_declarations must be an object",
9630        );
9631    }
9632
9633    #[test]
9634    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9635        assert_hello_drops_role_declarations_with_warning(
9636            json!({"routing/v1": {}, "bad key": {}}),
9637            "role_declarations key must use capability-name grammar",
9638        );
9639    }
9640
9641    #[test]
9642    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9643        assert_hello_drops_role_declarations_with_warning(
9644            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9645            "role_declarations exceeds 16384 compact JSON bytes",
9646        );
9647    }
9648
9649    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
9650        let mut body = serde_json::to_value(ModuleHelloBody {
9651            manifest: manifest("flow-ops", PROTOCOL_VERSION),
9652            protocol_ver: PROTOCOL_VERSION,
9653            control_ops: None,
9654            launch_nonce: None,
9655        })
9656        .unwrap();
9657        body["manifest"]["provides"] = json!([{
9658            "role": "management_surface",
9659            "operations": [
9660                {"name": "diagnostics.findings", "kind": "query"},
9661                {"name": "reset", "kind": "mutate"}
9662            ],
9663            "config_schema": {}, "observability": [], "identity_scope": []
9664        }]);
9665        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
9666        if let Some(flow_operations) = flow_operations {
9667            body["manifest"]["flow_operations"] = flow_operations;
9668        }
9669        Frame::build(
9670            FrameType::Hello,
9671            control_flags(),
9672            0,
9673            0,
9674            401,
9675            serde_json::to_vec(&body).unwrap(),
9676        )
9677        .unwrap()
9678    }
9679
9680    #[test]
9681    fn catalog_list_mirrors_flow_operations_unchanged() {
9682        // Order is kept as declared (not sorted) and an empty list stays an
9683        // affirmative "none", distinct from an absent declaration.
9684        for (capability, flow_operations) in [
9685            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
9686            ("call-modes/v1", json!(["diagnostics.findings"])),
9687            ("flow-scopes/v1", json!([])),
9688        ] {
9689            let registry = Arc::new(Registry::default());
9690            let handler = ControlHandler::new(Arc::clone(&registry));
9691            let connection = ConnectionId::new(401);
9692            let response = handler
9693                .handle_control(
9694                    connection,
9695                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
9696                )
9697                .unwrap();
9698            assert_eq!(response[0].header.ty, FrameType::HelloAck);
9699            let registered = registry
9700                .get_module_by_connection(connection)
9701                .unwrap()
9702                .unwrap()
9703                .manifest;
9704            assert_eq!(
9705                serde_json::to_value(&registered.flow_operations).unwrap(),
9706                flow_operations
9707            );
9708            let (raw, entry) = role_declarations_catalog(&handler);
9709            assert_eq!(raw["flow_operations"], flow_operations);
9710            assert_eq!(entry.flow_operations, registered.flow_operations);
9711        }
9712
9713        let registry = Arc::new(Registry::default());
9714        let handler = ControlHandler::new(Arc::clone(&registry));
9715        let response = handler
9716            .handle_control(
9717                ConnectionId::new(401),
9718                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
9719            )
9720            .unwrap();
9721        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9722        let (raw, entry) = role_declarations_catalog(&handler);
9723        assert!(raw.get("flow_operations").is_none());
9724        assert_eq!(entry.flow_operations, None);
9725    }
9726
9727    fn assert_hello_drops_flow_operations_with_warning(
9728        capabilities: Value,
9729        flow_operations: Value,
9730        reason: &str,
9731    ) {
9732        let registry = Arc::new(Registry::default());
9733        let handler = ControlHandler::new(Arc::clone(&registry));
9734        let capture = EventCapture::default();
9735        let _subscriber =
9736            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9737        let response = handler
9738            .handle_control(
9739                ConnectionId::new(401),
9740                hello_with_flow_operations(capabilities, Some(flow_operations)),
9741            )
9742            .unwrap();
9743        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9744        let registered = registry
9745            .get_module_by_connection(ConnectionId::new(401))
9746            .unwrap()
9747            .expect("the module still registers")
9748            .manifest;
9749        assert_eq!(registered.module_id, "flow-ops");
9750        assert_eq!(registered.flow_operations, None);
9751        let (raw, entry) = role_declarations_catalog(&handler);
9752        assert_eq!(raw["module_id"], "flow-ops");
9753        assert!(raw.get("flow_operations").is_none());
9754        assert_eq!(entry.flow_operations, None);
9755        let warnings: Vec<_> = capture
9756            .events()
9757            .into_iter()
9758            .filter(|event| {
9759                event.fields.get("message").map(String::as_str)
9760                    == Some("dropping flow_operations from HELLO")
9761            })
9762            .collect();
9763        assert_eq!(warnings.len(), 1);
9764        assert_eq!(warnings[0].level, tracing::Level::WARN);
9765        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
9766        assert_eq!(warnings[0].fields["reason"], reason);
9767    }
9768
9769    #[test]
9770    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
9771        assert_hello_drops_flow_operations_with_warning(
9772            json!(["flow-scopes/v1"]),
9773            json!(["diagnostics.findings", "missing"]),
9774            "flow_operations names 'missing', which is not a declared management operation",
9775        );
9776    }
9777
9778    #[test]
9779    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
9780        assert_hello_drops_flow_operations_with_warning(
9781            json!([]),
9782            json!(["diagnostics.findings"]),
9783            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9784        );
9785        // A run-only capability does not admit flows either.
9786        assert_hello_drops_flow_operations_with_warning(
9787            json!(["agent-run-scopes/v1"]),
9788            json!([]),
9789            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9790        );
9791    }
9792
9793    #[test]
9794    fn hello_malformed_flow_operations_is_dropped_with_warning() {
9795        assert_hello_drops_flow_operations_with_warning(
9796            json!(["flow-scopes/v1"]),
9797            json!(["reset", "reset"]),
9798            "flow_operations names 'reset' more than once",
9799        );
9800        assert_hello_drops_flow_operations_with_warning(
9801            json!(["flow-scopes/v1"]),
9802            json!(vec!["reset"; 257]),
9803            "flow_operations lists 257 names, more than 256",
9804        );
9805        assert_hello_drops_flow_operations_with_warning(
9806            json!(["flow-scopes/v1"]),
9807            json!("reset"),
9808            "flow_operations must be an array of operation names",
9809        );
9810    }
9811
9812    #[test]
9813    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9814        let registry = Arc::new(Registry::default());
9815        let handler = ControlHandler::new(registry.clone());
9816        let descriptor = json!({
9817            "schema": "ckios.richui.manifest/1",
9818            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9819            "bytes": 128,
9820            "chunks": 2
9821        });
9822        let malformed = [
9823            ("schema", json!("")),
9824            ("schema", json!("a".repeat(65))),
9825            ("schema", json!("Bad schema")),
9826            (
9827                "manifest_digest",
9828                json!(format!("sha256:{}", "A".repeat(64))),
9829            ),
9830            ("manifest_digest", json!("sha256:abc")),
9831            ("bytes", json!(0)),
9832            ("bytes", json!(16_777_217)),
9833            ("bytes", json!("128")),
9834            ("chunks", json!(0)),
9835            ("chunks", json!(1025)),
9836            ("chunks", json!("2")),
9837        ];
9838        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9839            let mut invalid = descriptor.clone();
9840            invalid[field] = invalid_value;
9841            let response = handler
9842                .handle_control(
9843                    ConnectionId::new(210 + index as u64),
9844                    hello_frame_with_rich_ui(
9845                        &format!("bad-rich-ui-{index}"),
9846                        PROTOCOL_VERSION,
9847                        210 + index as u64,
9848                        None,
9849                        invalid,
9850                    ),
9851                )
9852                .unwrap();
9853            assert_eq!(response[0].header.ty, FrameType::Error);
9854            let error = parse_error(&response[0]);
9855            assert_eq!(error["code"], "invalid_manifest");
9856            assert!(error["message"]
9857                .as_str()
9858                .unwrap()
9859                .contains(&format!("rich_ui.{field}")));
9860            assert!(registry
9861                .get_module(&format!("bad-rich-ui-{index}"))
9862                .unwrap()
9863                .is_none());
9864        }
9865    }
9866
9867    #[test]
9868    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9869        let registry = Arc::new(Registry::default());
9870        let handler = ControlHandler::new(Arc::clone(&registry));
9871        let connection = ConnectionId::new(219);
9872        registry
9873            .register_with_control_ops(
9874                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9875                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9876                        "bad schema",
9877                        format!("sha256:{}", "a".repeat(64)),
9878                        128,
9879                        2,
9880                    )))
9881                    .build(),
9882                PROTOCOL_VERSION,
9883                connection,
9884                Vec::new(),
9885            )
9886            .unwrap();
9887        let request =
9888            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9889        let response = handler
9890            .handle_catalog_update(connection, request, Vec::new(), None, None)
9891            .unwrap();
9892        assert_eq!(response[0].header.ty, FrameType::Error);
9893        let error = parse_error(&response[0]);
9894        assert_eq!(error["code"], "invalid_manifest");
9895        assert!(error["message"]
9896            .as_str()
9897            .unwrap()
9898            .contains("rich_ui.schema"));
9899    }
9900
9901    #[test]
9902    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9903        let registry = Arc::new(Registry::default());
9904        let supervisor = SupervisorHandle::new();
9905        supervisor.open_swap("provider", "candidate-nonce".to_string());
9906        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9907        let connection = ConnectionId::new(218);
9908        let hello = handler
9909            .handle_control(
9910                connection,
9911                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9912            )
9913            .unwrap();
9914        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9915        assert!(registry.list_modules().unwrap().1.is_empty());
9916
9917        registry.promote_candidate("provider").unwrap().unwrap();
9918        let request = Frame::build(
9919            FrameType::Request,
9920            control_flags(),
9921            0,
9922            0,
9923            219,
9924            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9925        )
9926        .unwrap();
9927        let response = handler.handle_catalog_list(request, None).unwrap();
9928        let ClientControlResponse::CatalogList { modules, .. } =
9929            serde_json::from_slice(&response[0].body).unwrap()
9930        else {
9931            panic!("expected catalog.list");
9932        };
9933        assert_eq!(modules.len(), 1);
9934        assert!(modules[0].nonce_verified);
9935    }
9936
9937    #[test]
9938    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9939        let registry = Arc::new(Registry::default());
9940        let handler = ControlHandler::new(Arc::clone(&registry));
9941        let hello = Frame::build(
9942            FrameType::Hello,
9943            control_flags(),
9944            0,
9945            0,
9946            220,
9947            serde_json::to_vec(&ModuleHelloBody {
9948                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9949                protocol_ver: PROTOCOL_VERSION,
9950                control_ops: None,
9951                launch_nonce: Some("unverified".to_string()),
9952            })
9953            .unwrap(),
9954        )
9955        .unwrap();
9956        assert_eq!(
9957            handler
9958                .handle_control(ConnectionId::new(220), hello)
9959                .unwrap()[0]
9960                .header
9961                .ty,
9962            FrameType::HelloAck
9963        );
9964        let request = Frame::build(
9965            FrameType::Request,
9966            control_flags(),
9967            0,
9968            0,
9969            221,
9970            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9971        )
9972        .unwrap();
9973        let response = handler.handle_catalog_list(request, None).unwrap();
9974        let ClientControlResponse::CatalogList { modules, .. } =
9975            serde_json::from_slice(&response[0].body).unwrap()
9976        else {
9977            panic!("expected catalog.list");
9978        };
9979        assert_eq!(
9980            serde_json::to_vec(&modules[0]).unwrap(),
9981            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9982        );
9983        assert!(!modules[0].nonce_verified);
9984    }
9985
9986    #[test]
9987    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9988        let registry = Arc::new(Registry::default());
9989        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9990            [("vault".to_string(), true), ("squatter".to_string(), true)],
9991            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9992        );
9993        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9994        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9995            provides: vec!["credentials-provider/v1".to_string()],
9996            requires: Vec::new(),
9997            must_never_reach: Vec::new(),
9998        });
9999        let frame = Frame::build(
10000            FrameType::Hello,
10001            control_flags(),
10002            0,
10003            0,
10004            77,
10005            serde_json::to_vec(&ModuleHelloBody {
10006                manifest: squatter,
10007                protocol_ver: PROTOCOL_VERSION,
10008                control_ops: None,
10009                launch_nonce: None,
10010            })
10011            .expect("HELLO serializes"),
10012        )
10013        .expect("HELLO frame builds");
10014        let response = handler
10015            .handle_control(ConnectionId::new(77), frame)
10016            .expect("reserved claim receives a typed refusal");
10017        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10018        assert_eq!(
10019            registry
10020                .active_registration_count()
10021                .expect("registry reads"),
10022            0,
10023            "a reserved capability refusal must not leave a catalog entry"
10024        );
10025    }
10026
10027    #[test]
10028    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10029        for settlement in ["timeout", "inconclusive", "drop"] {
10030            let breakers = RouteBindBreakers::default();
10031            let RouteBindAdmission::Admitted {
10032                guard: mut old,
10033                probe: false,
10034            } = breakers.admit("prov")
10035            else {
10036                panic!("ordinary relay admitted")
10037            };
10038            let RouteBindAdmission::Admitted {
10039                guard: mut opener, ..
10040            } = breakers.admit("prov")
10041            else {
10042                panic!("second relay admitted")
10043            };
10044            assert!(
10045                !opener
10046                    .record_timeout(1, Duration::ZERO)
10047                    .unwrap()
10048                    .reopened_after_probe
10049            );
10050            let RouteBindAdmission::Admitted {
10051                guard: mut probe,
10052                probe: true,
10053            } = breakers.admit("prov")
10054            else {
10055                panic!("one cooldown probe admitted")
10056            };
10057            match settlement {
10058                "timeout" => assert!(
10059                    !old.record_timeout(1, Duration::ZERO)
10060                        .unwrap()
10061                        .reopened_after_probe
10062                ),
10063                "inconclusive" => old.record_inconclusive(),
10064                "drop" => drop(old),
10065                _ => unreachable!(),
10066            }
10067            assert!(
10068                matches!(
10069                    breakers.admit("prov"),
10070                    RouteBindAdmission::Refused {
10071                        probe_in_flight: true,
10072                        ..
10073                    }
10074                ),
10075                "{settlement} of a pre-open relay cannot release the real probe"
10076            );
10077            assert!(
10078                probe
10079                    .record_timeout(1, Duration::ZERO)
10080                    .unwrap()
10081                    .reopened_after_probe
10082            );
10083            assert!(matches!(
10084                breakers.admit("prov"),
10085                RouteBindAdmission::Admitted { probe: true, .. }
10086            ));
10087        }
10088        let breakers = RouteBindBreakers::default();
10089        let admit = || match breakers.admit("prov") {
10090            RouteBindAdmission::Admitted { guard, .. } => guard,
10091            _ => panic!("relay admitted"),
10092        };
10093        admit().record_timeout(1, Duration::ZERO);
10094        let mut old_probe = admit();
10095        breakers.reset_for_new_module_connection("prov");
10096        admit().record_timeout(1, Duration::ZERO);
10097        let _new_probe = admit();
10098        old_probe.record_inconclusive();
10099        assert!(matches!(
10100            breakers.admit("prov"),
10101            RouteBindAdmission::Refused {
10102                probe_in_flight: true,
10103                ..
10104            }
10105        ));
10106    }
10107
10108    #[tokio::test]
10109    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10110        for candidate in [false, true] {
10111            let registry = Arc::new(Registry::default());
10112            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10113                [("vault".to_string(), true), ("squatter".to_string(), true)],
10114                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10115            );
10116            let conn = ConnectionId::new(77);
10117            let (ctx, mut rx) = route_ctx(conn);
10118            let initial = capability_manifest("squatter", &[], &[]);
10119            if candidate {
10120                registry
10121                    .register_candidate_with_control_ops(
10122                        initial.clone(),
10123                        PROTOCOL_VERSION,
10124                        conn,
10125                        module_baseline_control_ops(),
10126                    )
10127                    .unwrap();
10128                handler
10129                    .forwarding
10130                    .register_candidate_module_connection(
10131                        conn,
10132                        "squatter".to_string(),
10133                        PROTOCOL_VERSION,
10134                        manifest_concurrency(&initial),
10135                        ctx.egress.clone(),
10136                    )
10137                    .unwrap();
10138            } else {
10139                hello_via_sink(
10140                    &handler,
10141                    &ctx,
10142                    &mut rx,
10143                    hello_frame_with_manifest(initial.clone(), 1),
10144                )
10145                .await;
10146            }
10147            let response = handler
10148                .handle_control_frame(
10149                    &ctx,
10150                    catalog_update_with_capabilities_frame(
10151                        2,
10152                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
10153                            .capabilities
10154                            .unwrap(),
10155                    ),
10156                )
10157                .await
10158                .unwrap();
10159            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10160            assert_eq!(
10161                registry
10162                    .get_module_by_connection(conn)
10163                    .unwrap()
10164                    .unwrap()
10165                    .manifest,
10166                initial
10167            );
10168        }
10169    }
10170
10171    #[test]
10172    fn server_describe_surfaces_required_capability_verdict_fields() {
10173        let registry = Arc::new(Registry::default());
10174        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10175            [
10176                ("consumer".to_string(), true),
10177                ("provider".to_string(), false),
10178            ],
10179            BTreeMap::new(),
10180        );
10181        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10182        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10183            provides: Vec::new(),
10184            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10185                capability: "credentials-provider/v1".to_string(),
10186                need: subc_protocol::manifest::CapabilityNeed::Required,
10187            }],
10188            must_never_reach: Vec::new(),
10189        });
10190        let hello = Frame::build(
10191            FrameType::Hello,
10192            control_flags(),
10193            0,
10194            0,
10195            78,
10196            serde_json::to_vec(&ModuleHelloBody {
10197                manifest: consumer,
10198                protocol_ver: PROTOCOL_VERSION,
10199                control_ops: None,
10200                launch_nonce: None,
10201            })
10202            .expect("HELLO serializes"),
10203        )
10204        .expect("HELLO frame builds");
10205        handler
10206            .handle_control(ConnectionId::new(78), hello)
10207            .expect("consumer registers");
10208        let describe = Frame::build(
10209            FrameType::Request,
10210            control_flags(),
10211            0,
10212            0,
10213            79,
10214            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10215                .expect("request serializes"),
10216        )
10217        .expect("describe frame builds");
10218        let response = handler
10219            .handle_server_describe(describe)
10220            .expect("server.describe succeeds");
10221        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10222        let requirement = &rendered["capability_requirements"][0];
10223        assert_eq!(requirement["consumer"], "consumer");
10224        assert_eq!(requirement["verdict"], "never_provided");
10225        assert_eq!(requirement["episode_seq"], 1);
10226        assert_eq!(requirement["config_satisfiable"], false);
10227        assert_eq!(requirement["runtime_available"], false);
10228        assert!(requirement["detail"]
10229            .as_str()
10230            .expect("detail string")
10231            .contains("credentials-provider/v1"));
10232    }
10233
10234    /// Register a consumer that requires `project-identity/v1` and the provider
10235    /// of it, optionally mark daemon shutdown as begun, then deregister the
10236    /// provider as its connection closes. Returns the `capability.requirement`
10237    /// events that deregistration logged and the verdict the evaluator then
10238    /// holds for the requirement (the one `route.open` refusals read).
10239    fn deregister_required_provider(
10240        daemon_shutdown_begun: bool,
10241    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10242        const CAPABILITY: &str = "project-identity/v1";
10243        let registry = Arc::new(Registry::default());
10244        let daemon_shutdown = DaemonShutdownFlag::default();
10245        let handler = ControlHandler::new(Arc::clone(&registry))
10246            .with_capability_config(
10247                [
10248                    ("consumer".to_string(), true),
10249                    ("provider".to_string(), true),
10250                ],
10251                BTreeMap::new(),
10252            )
10253            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10254        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10255        consumer.capabilities = Some(CapabilityDeclarations {
10256            provides: Vec::new(),
10257            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10258                capability: CAPABILITY.to_string(),
10259                need: subc_protocol::manifest::CapabilityNeed::Required,
10260            }],
10261            must_never_reach: Vec::new(),
10262        });
10263        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10264        let provider_conn = ConnectionId::new(182);
10265        for (conn, manifest, corr) in [
10266            (ConnectionId::new(181), consumer, 181),
10267            (provider_conn, provider, 182),
10268        ] {
10269            let hello = handler
10270                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10271                .expect("module registers");
10272            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10273        }
10274        assert_eq!(
10275            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10276            Some(CapabilityVerdict::Provided),
10277            "the registered provider must satisfy the requirement first"
10278        );
10279        if daemon_shutdown_begun {
10280            daemon_shutdown.set_for_test();
10281        }
10282
10283        let capture = EventCapture::default();
10284        let _subscriber =
10285            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10286        handler
10287            .cleanup_connection(provider_conn)
10288            .expect("provider deregisters");
10289        let events = capture
10290            .events()
10291            .into_iter()
10292            .filter(|event| {
10293                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10294            })
10295            .collect();
10296        (
10297            events,
10298            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10299        )
10300    }
10301
10302    #[test]
10303    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10304        let (events, verdict) = deregister_required_provider(true);
10305        assert_eq!(
10306            verdict,
10307            Some(CapabilityVerdict::NeverProvided),
10308            "shutdown must not change the verdict route.open refusals read"
10309        );
10310        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10311        let event = &events[0];
10312        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10313        assert_eq!(
10314            event.fields.get("cause").map(String::as_str),
10315            Some("\"daemon_shutdown\""),
10316            "{event:?}"
10317        );
10318        assert_eq!(
10319            event.fields.get("verdict").map(String::as_str),
10320            Some("\"never_provided\""),
10321            "{event:?}"
10322        );
10323    }
10324
10325    #[test]
10326    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10327        let (events, verdict) = deregister_required_provider(false);
10328        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10329        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10330        let event = &events[0];
10331        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10332        assert_eq!(event.fields.get("cause"), None, "{event:?}");
10333        assert_eq!(
10334            event.fields.get("verdict").map(String::as_str),
10335            Some("\"never_provided\""),
10336            "{event:?}"
10337        );
10338    }
10339
10340    #[test]
10341    fn catalog_list_omits_capabilities_for_legacy_manifest() {
10342        let registry = Arc::new(Registry::default());
10343        let handler = ControlHandler::new(Arc::clone(&registry));
10344        let hello = handler
10345            .handle_control(
10346                ConnectionId::new(101),
10347                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10348            )
10349            .expect("legacy HELLO registers");
10350        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10351
10352        let request = Frame::build(
10353            FrameType::Request,
10354            control_flags(),
10355            0,
10356            0,
10357            102,
10358            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10359                .expect("catalog request serializes"),
10360        )
10361        .expect("catalog request frame builds");
10362        let response = handler
10363            .handle_catalog_list(request, None)
10364            .expect("catalog list succeeds");
10365        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10366        assert!(
10367            body["modules"][0].get("capabilities").is_none(),
10368            "legacy manifest must retain an absent capabilities field on catalog.list"
10369        );
10370    }
10371
10372    #[test]
10373    fn hello_ack_omits_storage_when_no_storage_config() {
10374        let registry = Arc::new(Registry::default());
10375        let handler = ControlHandler::new(Arc::clone(&registry));
10376        let responses = handler
10377            .handle_control(
10378                ConnectionId::new(1),
10379                hello_frame("aft", PROTOCOL_VERSION, 7),
10380            )
10381            .unwrap();
10382        let ack = parse_ack(&responses[0]);
10383        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10384        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10385    }
10386
10387    #[tokio::test]
10388    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10389        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10390        let registry = Arc::new(Registry::default());
10391        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
10392        let responses = handler
10393            .handle_control(
10394                ConnectionId::new(1),
10395                hello_frame("aft", PROTOCOL_VERSION, 7),
10396            )
10397            .unwrap();
10398        let ack = parse_ack(&responses[0]);
10399        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10400
10401        let described = handler
10402            .handle_control_frame(
10403                &route_ctx(ConnectionId::new(2)).0,
10404                Frame::build(
10405                    FrameType::Request,
10406                    control_flags(),
10407                    0,
10408                    0,
10409                    9,
10410                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10411                )
10412                .unwrap(),
10413            )
10414            .await
10415            .unwrap();
10416        let ClientControlResponse::ServerDescribe { machine_id, .. } =
10417            serde_json::from_slice(&described[0].body).unwrap()
10418        else {
10419            panic!("server.describe answered with another shape");
10420        };
10421        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10422    }
10423
10424    #[test]
10425    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10426        // With a central sqlite storage policy, each registering module gets its
10427        // own resolved descriptor in HELLO_ACK, keyed by its module id.
10428        let registry = Arc::new(Registry::default());
10429        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
10430            crate::daemon_config::StorageConfig::Sqlite {
10431                data_home: std::path::PathBuf::from("/data"),
10432            },
10433        ));
10434
10435        let responses = handler
10436            .handle_control(
10437                ConnectionId::new(1),
10438                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10439            )
10440            .unwrap();
10441        let ack = parse_ack(&responses[0]);
10442        assert_eq!(
10443            ack.storage,
10444            Some(serde_json::json!({
10445                "module_id": "alfonso-routing",
10446                "storage_namespace": "default",
10447                "isolation": { "kind": "module" },
10448                "backend": {
10449                    "backend": "sqlite",
10450                    "path": "/data/cortexkit/alfonso-routing/store.db"
10451                }
10452            })),
10453            "the delivered descriptor is the module's own sqlite store path"
10454        );
10455    }
10456
10457    #[test]
10458    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10459        let registry = Arc::new(Registry::default());
10460        let handler = ControlHandler::new(Arc::clone(&registry));
10461        let conn = ConnectionId::new(1);
10462        let responses = handler
10463            .handle_control(
10464                conn,
10465                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10466            )
10467            .unwrap();
10468        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10469        let registration = registry.get_module("aft").unwrap().unwrap();
10470        assert_eq!(registration.control_ops, module_baseline_control_ops());
10471
10472        let frame =
10473            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10474        assert!(handler
10475            .guard_module_control_op(&frame, "aft", "route.bind")
10476            .unwrap()
10477            .is_none());
10478        let error = handler
10479            .guard_module_control_op(&frame, "aft", "test.synthetic")
10480            .unwrap()
10481            .expect("synthetic ungranted op should be rejected");
10482        assert_eq!(error.header.ty, FrameType::Error);
10483        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10484    }
10485
10486    #[test]
10487    fn hello_control_ops_some_adds_optional_grants() {
10488        let registry = Arc::new(Registry::default());
10489        let handler = ControlHandler::new(Arc::clone(&registry));
10490        handler
10491            .handle_control(
10492                ConnectionId::new(1),
10493                hello_frame_with_control_ops(
10494                    "aft",
10495                    PROTOCOL_VERSION,
10496                    7,
10497                    Some(vec![
10498                        "future.synthetic".to_string(),
10499                        "route.bind".to_string(),
10500                    ]),
10501                ),
10502            )
10503            .unwrap();
10504        let registration = registry.get_module("aft").unwrap().unwrap();
10505        assert_eq!(
10506            registration.control_ops,
10507            vec![
10508                "route.bind".to_string(),
10509                "route.status".to_string(),
10510                "future.synthetic".to_string(),
10511            ]
10512        );
10513        let frame =
10514            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10515        assert!(handler
10516            .guard_module_control_op(&frame, "aft", "future.synthetic")
10517            .unwrap()
10518            .is_none());
10519    }
10520
10521    #[tokio::test]
10522    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10523        let registry = Arc::new(Registry::default());
10524        let forwarding = Arc::new(ForwardingTable::default());
10525        let handler =
10526            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10527        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10528        hello_via_sink(
10529            &handler,
10530            &module_ctx,
10531            &mut module_rx,
10532            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10533        )
10534        .await;
10535
10536        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10537        let responses = handler
10538            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10539            .await
10540            .unwrap();
10541        assert_eq!(responses.len(), 1);
10542        assert_eq!(responses[0].header.ty, FrameType::Error);
10543        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10544        assert!(module_rx.try_recv().is_err());
10545    }
10546
10547    #[tokio::test]
10548    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10549        let registry = Arc::new(Registry::default());
10550        let forwarding = Arc::new(ForwardingTable::default());
10551        let handler =
10552            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10553        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10554        hello_via_sink(
10555            &handler,
10556            &module_ctx,
10557            &mut module_rx,
10558            hello_frame_with_control_ops(
10559                "aft",
10560                PROTOCOL_VERSION,
10561                7,
10562                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10563            ),
10564        )
10565        .await;
10566
10567        let project_root = unique_project_root("demux");
10568        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10569        let route_handler = handler.clone();
10570        let route_task = tokio::spawn(async move {
10571            route_handler
10572                .handle_control_frame(
10573                    &route_client_ctx,
10574                    route_open_frame(100, "aft", project_root),
10575                )
10576                .await
10577                .unwrap()
10578        });
10579        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10580            .await
10581            .unwrap()
10582            .unwrap();
10583        assert!(matches!(
10584            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10585            ModuleControlRequest::RouteBind { .. }
10586        ));
10587
10588        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10589        let health_handler = handler.clone();
10590        let health_task = tokio::spawn(async move {
10591            health_handler
10592                .handle_control_frame(
10593                    &health_client_ctx,
10594                    supervisor_health_probe_frame(101, "aft"),
10595                )
10596                .await
10597                .unwrap()
10598        });
10599        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10600            .await
10601            .unwrap()
10602            .unwrap();
10603        assert_eq!(
10604            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10605            ModuleControlRequest::HealthCheck {}
10606        );
10607
10608        handler
10609            .handle_control_frame(
10610                &module_ctx,
10611                health_response(health_frame.header.corr, HealthStatus::Degraded),
10612            )
10613            .await
10614            .unwrap();
10615        let health_response = health_task.await.unwrap();
10616        assert_eq!(health_response.len(), 1);
10617        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10618            ClientControlResponse::SupervisorHealthProbe {
10619                module_id,
10620                status,
10621                detail,
10622                metrics,
10623            } => {
10624                assert_eq!(module_id, "aft");
10625                assert_eq!(status, HealthStatus::Degraded);
10626                assert_eq!(detail.as_deref(), Some("warming"));
10627                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10628            }
10629            other => panic!("unexpected health response: {other:?}"),
10630        }
10631
10632        handler
10633            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10634            .await
10635            .unwrap();
10636        let route_response = route_task.await.unwrap();
10637        assert!(route_response.is_empty());
10638        let published = route_client_rx.recv().await.unwrap();
10639        assert!(matches!(
10640            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10641            ClientControlResponse::RouteOpen { .. }
10642        ));
10643    }
10644
10645    /// Start one `route.open` on `client_connection` and return its still-running
10646    /// handler task together with the `route.bind` the module received for it.
10647    /// The handler blocks until the module answers, so it has to run as a task
10648    /// while the test drives the module side.
10649    async fn relay_route_open(
10650        handler: &ControlHandler,
10651        client_connection: ConnectionId,
10652        client_egress: &FrameSink,
10653        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10654        corr: u64,
10655        module_id: &str,
10656        project_root_label: &str,
10657    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10658        let ctx = RouteCtx {
10659            connection_id: client_connection,
10660            egress: client_egress.clone(),
10661        };
10662        let handler = handler.clone();
10663        let project_root = unique_project_root(project_root_label);
10664        let module_id = module_id.to_string();
10665        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10666        let task = tokio::spawn(async move {
10667            let _guard = tracing::dispatcher::set_default(&dispatch);
10668            handler
10669                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10670                .await
10671                .unwrap()
10672        });
10673        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10674            .await
10675            .expect("module receives the relayed route.bind")
10676            .expect("module egress is open");
10677        (task, bind.frame)
10678    }
10679
10680    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10681        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10682            ModuleControlRequest::RouteBind {
10683                route_channel,
10684                epoch,
10685                ..
10686            } => (route_channel, epoch),
10687            other => panic!("expected a route.bind request, got {other:?}"),
10688        }
10689    }
10690
10691    fn published_route(frame: &Frame) -> (u16, u32) {
10692        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10693            ClientControlResponse::RouteOpen {
10694                route_channel,
10695                route_epoch,
10696            } => (route_channel, route_epoch),
10697            other => panic!("expected a route.open response, got {other:?}"),
10698        }
10699    }
10700
10701    /// Reproduction of a production outage. A client had `route.open`s in
10702    /// flight to a module and was already marked closing -- its egress had refused a
10703    /// module frame, so the daemon asked its connection to end -- while its sink
10704    /// was still open. When the module acked those binds, the daemon refused to
10705    /// commit a route for a closing client, and that refusal was returned from
10706    /// the MODULE connection's frame handler, where a router error that has no
10707    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
10708    /// the supervisor correctly did not respawn a clean exit, and every seat lost
10709    /// its tools for hours -- one client's teardown took down a connection
10710    /// carrying ~170 other routes.
10711    ///
10712    /// The window is opened here by calling the production path that opens it
10713    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
10714    /// state that matters is "in `closing_connections`, sink still open, relay
10715    /// still pending", and it lasts only from the close request until the
10716    /// connection loop reacts to it; a socket-level test can flood a client into
10717    /// that escalation but cannot pin the module's ack inside the window. Closing
10718    /// the socket instead takes the other path entirely -- connection teardown
10719    /// removes the pending relay under the same lock, so the ack finds nothing.
10720    #[tokio::test]
10721    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10722        let registry = Arc::new(Registry::default());
10723        let forwarding = Arc::new(ForwardingTable::default());
10724        let handler =
10725            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10726
10727        let module_connection = ConnectionId::new(30);
10728        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10729        hello_via_sink(
10730            &handler,
10731            &module_ctx,
10732            &mut module_rx,
10733            hello_frame("aft", PROTOCOL_VERSION, 7),
10734        )
10735        .await;
10736
10737        let dying_client = ConnectionId::new(31);
10738        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10739
10740        // A published route on the dying client. The escalation below only marks
10741        // a connection closing for a route it has already published.
10742        let (first_task, first_bind) = relay_route_open(
10743            &handler,
10744            dying_client,
10745            &dying_ctx.egress,
10746            &mut module_rx,
10747            100,
10748            "aft",
10749            "closing-first",
10750        )
10751        .await;
10752        handler
10753            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10754            .await
10755            .unwrap();
10756        assert!(first_task.await.unwrap().is_empty());
10757        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10758
10759        // A second route.open from the same client, relayed and awaiting its ack.
10760        let (second_task, second_bind) = relay_route_open(
10761            &handler,
10762            dying_client,
10763            &dying_ctx.egress,
10764            &mut module_rx,
10765            101,
10766            "aft",
10767            "closing-second",
10768        )
10769        .await;
10770        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10771
10772        // The window: the client is closing, its sink is still open, and its
10773        // second bind is still pending.
10774        assert!(forwarding
10775            .escalate_client_delivery_failure(
10776                dying_client,
10777                first_channel,
10778                first_epoch,
10779                CloseReason::new(
10780                    "module_to_client_delivery_failed",
10781                    "client egress refused a module frame",
10782                ),
10783                crate::forwarding::UndeliveredFrame {
10784                    module_id: None,
10785                    sink: &dying_ctx.egress,
10786                },
10787            )
10788            .unwrap());
10789        assert!(!dying_ctx.egress.is_closed());
10790
10791        // The frame that used to end the module connection.
10792        let ack = handler
10793            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10794            .await;
10795        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10796        if module_loop_error.is_some() {
10797            // What the server's connection loop does with a router error that has
10798            // no ERROR-frame translation: end the connection, which releases the
10799            // module's registration and every route on it.
10800            handler.cleanup_connection(module_connection).unwrap();
10801        }
10802        // Read the module's next frame before opening the co-tenant's route, so
10803        // the GOODBYE assertion below is about THIS ack and not about later
10804        // traffic. `None` means the module was told nothing.
10805        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10806            .await
10807            .ok()
10808            .flatten();
10809
10810        // 1. The module connection is still registered.
10811        assert!(
10812            registry
10813                .get_module_by_connection(module_connection)
10814                .unwrap()
10815                .is_some(),
10816            "one client's closing connection ended the shared module connection: \
10817             {module_loop_error:?}"
10818        );
10819        // ...and still serving: another client can open and use a route on it.
10820        let cotenant = ConnectionId::new(32);
10821        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10822        let (cotenant_task, cotenant_bind) = relay_route_open(
10823            &handler,
10824            cotenant,
10825            &cotenant_ctx.egress,
10826            &mut module_rx,
10827            102,
10828            "aft",
10829            "closing-cotenant",
10830        )
10831        .await;
10832        handler
10833            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10834            .await
10835            .unwrap();
10836        assert!(cotenant_task.await.unwrap().is_empty());
10837        let (cotenant_channel, cotenant_epoch) =
10838            published_route(&cotenant_rx.recv().await.unwrap());
10839        assert!(matches!(
10840            forwarding
10841                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10842                .unwrap(),
10843            DataRoute::Client(DataRouteState::Bound(_))
10844        ));
10845
10846        // 2. The module was told to drop the binding it created for the route
10847        //    that will never be published.
10848        let goodbye = post_ack_module_frame
10849            .expect("module receives a GOODBYE for the abandoned route channel");
10850        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10851        assert_eq!(goodbye.header.channel, abandoned_channel);
10852        assert_eq!(goodbye.header.epoch, abandoned_epoch);
10853
10854        // 3. The dying client received nothing: no route was ever published to
10855        //    it. Its route.open is answered as unavailable, which the connection
10856        //    loop would write to a socket that is already going away.
10857        assert!(dying_rx.try_recv().is_err());
10858        let second_response = second_task.await.unwrap();
10859        assert_eq!(second_response.len(), 1);
10860        assert_eq!(
10861            parse_error(&second_response[0])["code"],
10862            "target_unavailable"
10863        );
10864    }
10865
10866    /// One `catalog.list` as a client sees it: the generation and the listed
10867    /// module ids with their readiness.
10868    async fn catalog_snapshot(
10869        handler: &ControlHandler,
10870        client_ctx: &RouteCtx,
10871        corr: u64,
10872    ) -> (u64, Vec<(String, bool)>) {
10873        let body =
10874            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10875        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10876        let reply = handler
10877            .handle_control_frame(client_ctx, request)
10878            .await
10879            .unwrap()
10880            .pop()
10881            .unwrap();
10882        let ClientControlResponse::CatalogList {
10883            generation,
10884            modules,
10885            ..
10886        } = serde_json::from_slice(&reply.body).unwrap()
10887        else {
10888            panic!("not a catalog.list reply");
10889        };
10890        (
10891            generation,
10892            modules
10893                .into_iter()
10894                .map(|entry| (entry.module_id, entry.ready))
10895                .collect(),
10896        )
10897    }
10898
10899    /// A handler whose HELLO for module "aft" on connection 40 is parked
10900    /// between reserving its registry entry and installing its forwarding
10901    /// endpoint. Returns the handler, the two-party barrier (wait once to know
10902    /// the HELLO is parked, once more to release it), the HELLO's thread, and
10903    /// the module's context and receiver.
10904    fn hello_held_before_forwarding() -> (
10905        ControlHandler,
10906        Arc<std::sync::Barrier>,
10907        std::thread::JoinHandle<Vec<Frame>>,
10908        RouteCtx,
10909        mpsc::Receiver<crate::router::OutboundFrame>,
10910    ) {
10911        let hold = Arc::new(std::sync::Barrier::new(2));
10912        let handler = ControlHandler::with_forwarding(
10913            Arc::new(Registry::default()),
10914            Arc::new(ForwardingTable::default()),
10915        )
10916        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10917        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10918        let hello = {
10919            let handler = handler.clone();
10920            let module_ctx = module_ctx.clone();
10921            std::thread::spawn(move || {
10922                handler
10923                    .handle_hello(
10924                        module_ctx.connection_id,
10925                        Some(module_ctx.egress.clone()),
10926                        hello_frame("aft", PROTOCOL_VERSION, 7),
10927                    )
10928                    .unwrap()
10929            })
10930        };
10931        (handler, hold, hello, module_ctx, module_rx)
10932    }
10933
10934    /// While a HELLO has reserved its registry entry but its forwarding
10935    /// endpoint is not installed yet, `route.open` to the module can only be
10936    /// refused. `catalog.list` must not list the module in that window: a
10937    /// client that opens a route as soon as it sees a module listed and ready
10938    /// would be refused for a module the daemon had just advertised. The
10939    /// refusal in the window stays the retryable `target_unavailable`, never
10940    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10941    /// Once the HELLO completes, the module is listed and a route to it opens.
10942    #[tokio::test]
10943    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10944        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10945        let client = ConnectionId::new(41);
10946        let (client_ctx, mut client_rx) = route_ctx(client);
10947
10948        hold.wait();
10949        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10950        assert!(handler.registry.get_module("aft").unwrap().is_some());
10951        assert!(!handler
10952            .forwarding
10953            .has_live_module_connection("aft")
10954            .unwrap());
10955        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10956        assert!(
10957            listed.is_empty(),
10958            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10959        );
10960        let refused = handler
10961            .handle_control_frame(
10962                &client_ctx,
10963                route_open_frame(301, "aft", unique_project_root("listed-window")),
10964            )
10965            .await
10966            .unwrap();
10967        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10968        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10969        let duplicate = handler
10970            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10971            .await
10972            .unwrap();
10973        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10974
10975        hold.wait();
10976        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10977        let ack = module_rx.try_recv().unwrap().frame;
10978        assert_eq!(ack.header.ty, FrameType::HelloAck);
10979        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10980        assert_eq!(listed, vec![("aft".to_string(), true)]);
10981
10982        let (open_task, bind) = relay_route_open(
10983            &handler,
10984            client,
10985            &client_ctx.egress,
10986            &mut module_rx,
10987            303,
10988            "aft",
10989            "listed-after",
10990        )
10991        .await;
10992        assert!(handler
10993            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10994            .await
10995            .unwrap()
10996            .is_empty());
10997        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10998        published_route(&client_rx.try_recv().unwrap().frame);
10999    }
11000
11001    /// A catalog watcher keys on the generation: when it changes, it re-reads
11002    /// the listing. So the generation must advance exactly when the module
11003    /// appears. If it advanced while the module was still hidden, a watcher
11004    /// would read the new generation without the module, and the module would
11005    /// later appear under that same generation with nothing telling the
11006    /// watcher to look again.
11007    #[tokio::test]
11008    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11009        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11010        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11011        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11012
11013        hold.wait();
11014        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11015        assert!(listed.is_empty());
11016        assert_eq!(
11017            during, before,
11018            "the generation moved while the module was not listed"
11019        );
11020
11021        hold.wait();
11022        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11023        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11024        assert_eq!(listed, vec![("aft".to_string(), true)]);
11025        assert_eq!(
11026            after,
11027            before + 1,
11028            "the module appeared without exactly one generation bump"
11029        );
11030    }
11031
11032    /// A HELLO whose forwarding endpoint cannot be installed (here the
11033    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
11034    /// refused and leaves nothing behind: no registry entry, no catalog entry,
11035    /// and the catalog generation where it was.
11036    #[tokio::test]
11037    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11038        let registry = Arc::new(Registry::default());
11039        let handler = ControlHandler::with_forwarding(
11040            Arc::clone(&registry),
11041            Arc::new(ForwardingTable::default()),
11042        );
11043        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11044        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11045        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11046        drop(module_rx);
11047
11048        let replies = handler
11049            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11050            .await
11051            .unwrap();
11052        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11053        assert!(registry.get_module("aft").unwrap().is_none());
11054        assert!(registry
11055            .get_module_by_connection(module_ctx.connection_id)
11056            .unwrap()
11057            .is_none());
11058        assert_eq!(
11059            catalog_snapshot(&handler, &client_ctx, 301).await,
11060            (before, Vec::new())
11061        );
11062    }
11063
11064    /// A handler whose `route.open` parks after queueing the route.bind and
11065    /// before marking it enqueued, with module "aft" registered on connection
11066    /// 40. Returns the two-party barrier that parks and releases it.
11067    async fn route_open_held_after_bind_send() -> (
11068        ControlHandler,
11069        Arc<tokio::sync::Barrier>,
11070        RouteCtx,
11071        mpsc::Receiver<crate::router::OutboundFrame>,
11072    ) {
11073        let hold = Arc::new(tokio::sync::Barrier::new(2));
11074        let handler = ControlHandler::with_forwarding(
11075            Arc::new(Registry::default()),
11076            Arc::new(ForwardingTable::default()),
11077        )
11078        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11079        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11080        hello_via_sink(
11081            &handler,
11082            &module_ctx,
11083            &mut module_rx,
11084            hello_frame("aft", PROTOCOL_VERSION, 7),
11085        )
11086        .await;
11087        (handler, hold, module_ctx, module_rx)
11088    }
11089
11090    /// A fast module acks the route.bind, and the reader commits the route,
11091    /// before `route.open` has marked the relay enqueued. The daemon used to
11092    /// read the missing relay as an abandoned bind and send the module a
11093    /// GOODBYE for the route it had just committed: the client was told its
11094    /// route was open, the module unbound the channel, and the client's first
11095    /// request was dropped on the module side as unbound.
11096    #[tokio::test]
11097    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11098        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11099        let client = ConnectionId::new(41);
11100        let (client_ctx, mut client_rx) = route_ctx(client);
11101
11102        let (open_task, bind) = relay_route_open(
11103            &handler,
11104            client,
11105            &client_ctx.egress,
11106            &mut module_rx,
11107            200,
11108            "aft",
11109            "acked-before-mark",
11110        )
11111        .await;
11112        let (module_channel, module_epoch) = route_bind_channel(&bind);
11113        // route.open is now parked between queueing the bind and marking it.
11114        hold.wait().await;
11115
11116        // The module answers inside that window, and the route commits.
11117        assert!(handler
11118            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11119            .await
11120            .unwrap()
11121            .is_empty());
11122        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11123
11124        hold.wait().await;
11125        assert!(
11126            open_task.await.unwrap().is_empty(),
11127            "route.open answers accepted"
11128        );
11129
11130        // Every frame route.open could have sent is already queued, because it
11131        // has returned; the module must have been sent nothing.
11132        match module_rx.try_recv() {
11133            Err(mpsc::error::TryRecvError::Empty) => {}
11134            Ok(outbound) => panic!(
11135                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11136                outbound.frame.header.ty,
11137                outbound.frame.header.channel,
11138                outbound.frame.header.epoch
11139            ),
11140            Err(err) => panic!("module egress closed: {err}"),
11141        }
11142
11143        // The client's first request on the route is the next thing the module
11144        // reads, on the binding it just created.
11145        let router = Router::with_control_handler(Arc::new(handler.clone()));
11146        let request = Frame::build(
11147            FrameType::Request,
11148            Flags::new(true, Priority::Interactive, false),
11149            client_channel,
11150            client_epoch,
11151            201,
11152            b"first request".to_vec(),
11153        )
11154        .unwrap();
11155        router
11156            .route_for_connection(&client_ctx, request)
11157            .await
11158            .unwrap();
11159        let delivered = module_rx.try_recv().unwrap().frame;
11160        assert_eq!(delivered.header.ty, FrameType::Request);
11161        assert_eq!(
11162            (delivered.header.channel, delivered.header.epoch),
11163            (module_channel, module_epoch)
11164        );
11165        assert_eq!(delivered.body, b"first request");
11166    }
11167
11168    /// The client goes away while `route.open` is still queueing the bind, so
11169    /// the relay is dropped with no module reply. The teardown cannot send the
11170    /// GOODBYE itself (it could reach the module ahead of the bind), so
11171    /// `route.open` must send it once the bind is queued; otherwise the module
11172    /// keeps a binding the daemon will never commit.
11173    #[tokio::test]
11174    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11175        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11176        let client = ConnectionId::new(42);
11177        let (client_ctx, _client_rx) = route_ctx(client);
11178
11179        let (open_task, bind) = relay_route_open(
11180            &handler,
11181            client,
11182            &client_ctx.egress,
11183            &mut module_rx,
11184            300,
11185            "aft",
11186            "released-before-mark",
11187        )
11188        .await;
11189        let (module_channel, module_epoch) = route_bind_channel(&bind);
11190        hold.wait().await;
11191
11192        handler.cleanup_connection(client).unwrap();
11193        assert!(
11194            module_rx.try_recv().is_err(),
11195            "client teardown must not GOODBYE a bind route.open is still queueing"
11196        );
11197
11198        hold.wait().await;
11199        let response = open_task.await.unwrap();
11200        assert_eq!(response.len(), 1);
11201        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11202
11203        let goodbye = module_rx
11204            .try_recv()
11205            .expect("module is told to drop the binding the daemon abandoned")
11206            .frame;
11207        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11208        assert_eq!(
11209            (goodbye.header.channel, goodbye.header.epoch),
11210            (module_channel, module_epoch)
11211        );
11212        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11213    }
11214
11215    /// The module acks inside the same window, but the client's egress closed
11216    /// first, so the reader refuses to commit. The ack proves the module holds
11217    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
11218    /// send a second one.
11219    #[tokio::test]
11220    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11221        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11222        let client = ConnectionId::new(43);
11223        let (client_ctx, client_rx) = route_ctx(client);
11224
11225        let (open_task, bind) = relay_route_open(
11226            &handler,
11227            client,
11228            &client_ctx.egress,
11229            &mut module_rx,
11230            400,
11231            "aft",
11232            "gone-client-before-mark",
11233        )
11234        .await;
11235        let (module_channel, module_epoch) = route_bind_channel(&bind);
11236        hold.wait().await;
11237
11238        drop(client_rx);
11239        assert!(client_ctx.egress.is_closed());
11240        handler
11241            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11242            .await
11243            .unwrap();
11244
11245        hold.wait().await;
11246        let response = open_task.await.unwrap();
11247        assert_eq!(response.len(), 1);
11248        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11249
11250        let goodbye = module_rx
11251            .try_recv()
11252            .expect("module is told to drop the binding it acked")
11253            .frame;
11254        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11255        assert_eq!(
11256            (goodbye.header.channel, goodbye.header.epoch),
11257            (module_channel, module_epoch)
11258        );
11259        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11260        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11261    }
11262
11263    /// The fence at the module-loop boundary, stated as its own contract: which
11264    /// forwarding failures are allowed to end the module connection that is being
11265    /// served. A `ConnectionClosing` naming some client is about that client, and
11266    /// a module connection is shared; the same error naming the module's own
11267    /// connection is about this connection and must stay fatal, as must failures
11268    /// that are about the forwarding table itself.
11269    #[test]
11270    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11271        let handler = ControlHandler::default();
11272        let module_connection = ConnectionId::new(30);
11273        let client_connection = ConnectionId::new(31);
11274
11275        handler
11276            .refuse_to_end_module_connection_for_a_client(
11277                module_connection,
11278                77,
11279                ForwardingError::ConnectionClosing {
11280                    connection_id: client_connection,
11281                },
11282            )
11283            .expect("a closing client must never end the module connection");
11284
11285        assert!(matches!(
11286            handler.refuse_to_end_module_connection_for_a_client(
11287                module_connection,
11288                78,
11289                ForwardingError::ConnectionClosing {
11290                    connection_id: module_connection,
11291                },
11292            ),
11293            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11294                connection_id
11295            })) if connection_id == module_connection
11296        ));
11297        assert!(matches!(
11298            handler.refuse_to_end_module_connection_for_a_client(
11299                module_connection,
11300                79,
11301                ForwardingError::Poisoned,
11302            ),
11303            Err(RouterError::Forwarding(ForwardingError::Poisoned))
11304        ));
11305        assert!(matches!(
11306            handler.refuse_to_end_module_connection_for_a_client(
11307                module_connection,
11308                80,
11309                ForwardingError::StaleModuleEndpoint,
11310            ),
11311            Err(RouterError::Forwarding(
11312                ForwardingError::StaleModuleEndpoint
11313            ))
11314        ));
11315    }
11316
11317    /// The spawn-attestation guard is what stops a connected module from claiming
11318    /// another module's identity and being stamped `Reserved` for it. Every other
11319    /// test that supplies a consumer_identity supplies a CORRECT one, because a
11320    /// correct one is what the rest of the flow needs -- so the guard's rejection
11321    /// branch was never the subject of an assertion, only its acceptance branch.
11322    ///
11323    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
11324    /// whole subc-core library suite green; only the forwarding integration tests
11325    /// notice, and they notice for unrelated reasons. This test exists so the
11326    /// refusal itself is asserted where the guard lives: it fails if the identity
11327    /// check stops refusing, which is the direction that matters, since a guard
11328    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
11329    #[tokio::test]
11330    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11331        let registry = Arc::new(Registry::default());
11332        let forwarding = Arc::new(ForwardingTable::default());
11333        let supervisor = SupervisorHandle::new();
11334        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11335        let handler =
11336            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11337                .with_supervisor(supervisor);
11338
11339        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11340        hello_via_sink(
11341            &handler,
11342            &target_ctx,
11343            &mut target_rx,
11344            hello_frame("target", PROTOCOL_VERSION, 1),
11345        )
11346        .await;
11347
11348        // A real supervised module id presenting the wrong nonce. This is the
11349        // impersonation case: the attacker knows a privileged module_id, which is
11350        // public, and guesses at the nonce, which is not.
11351        let wrong_nonce = handler
11352            .handle_control_frame(
11353                &route_ctx(ConnectionId::new(91)).0,
11354                route_open_frame_with_admission_facts(
11355                    20,
11356                    "target",
11357                    unique_project_root("admission-facts"),
11358                    Some(subc_control::ConsumerIdentity {
11359                        module_id: "fed".to_string(),
11360                        launch_nonce: "not-the-real-nonce".to_string(),
11361                    }),
11362                    None,
11363                ),
11364            )
11365            .await
11366            .unwrap();
11367        assert_eq!(
11368            parse_error(&wrong_nonce[0])["code"],
11369            "bad_consumer_identity",
11370            "a mismatched launch nonce must be refused, not stamped Reserved"
11371        );
11372
11373        // A module id the supervisor never spawned at all, so no nonce exists to
11374        // compare against. An implementation that treats "no record" as "nothing
11375        // to check" fails open here while passing the case above.
11376        let never_spawned = handler
11377            .handle_control_frame(
11378                &route_ctx(ConnectionId::new(92)).0,
11379                route_open_frame_with_admission_facts(
11380                    21,
11381                    "target",
11382                    unique_project_root("admission-facts"),
11383                    Some(subc_control::ConsumerIdentity {
11384                        module_id: "never-spawned".to_string(),
11385                        launch_nonce: "any-nonce".to_string(),
11386                    }),
11387                    None,
11388                ),
11389            )
11390            .await
11391            .unwrap();
11392        assert_eq!(
11393            parse_error(&never_spawned[0])["code"],
11394            "bad_consumer_identity",
11395            "an unspawned module_id must be refused rather than accepted for lack of a record"
11396        );
11397    }
11398
11399    /// The refusal test above proves the guard says NO. Nothing proved it can say
11400    /// YES, and the difference is not academic: replacing the whole authorization
11401    /// with `false` -- admitting no consumer identity at all, revoking Reserved
11402    /// standing for every supervised module in the fleet -- leaves 110 of the 111
11403    /// library tests GREEN. The one that notices does so by HANGING, because it
11404    /// waits for a bind that can no longer happen.
11405    ///
11406    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
11407    /// or flaky test, invites a RETRY rather than an investigation, and the retry
11408    /// hangs too and gets blamed on the runner. So a total revocation of the
11409    /// daemon's trust grant would have shipped behind a symptom nobody attributes
11410    /// to code.
11411    ///
11412    /// The bias is structural rather than accidental. A REFUSAL looks like a
11413    /// failure someone writes a test for; a GRANT looks like the happy path. Every
11414    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
11415    /// suite for that reason, and this one is the purest case in the daemon.
11416    ///
11417    /// This test asserts the EFFECT rather than the absence of an error: the module
11418    /// receives a RouteBind and it carries `Reserved` naming the attested module.
11419    /// A guard that admitted nobody would produce no bind at all; one that admitted
11420    /// everybody would stamp the wrong principal, which the refusal test catches.
11421    #[tokio::test]
11422    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11423        let registry = Arc::new(Registry::default());
11424        let forwarding = Arc::new(ForwardingTable::default());
11425        let supervisor = SupervisorHandle::new();
11426        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11427        let handler =
11428            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11429                .with_supervisor(supervisor);
11430
11431        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11432        hello_via_sink(
11433            &handler,
11434            &target_ctx,
11435            &mut target_rx,
11436            hello_frame("target", PROTOCOL_VERSION, 1),
11437        )
11438        .await;
11439
11440        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11441        let route_handler = handler.clone();
11442        let route_task = tokio::spawn(async move {
11443            route_handler
11444                .handle_control_frame(
11445                    &client_ctx,
11446                    route_open_frame_with_admission_facts(
11447                        30,
11448                        "target",
11449                        unique_project_root("admission-facts"),
11450                        Some(subc_control::ConsumerIdentity {
11451                            module_id: "fed".to_string(),
11452                            launch_nonce: "fed-nonce".to_string(),
11453                        }),
11454                        None,
11455                    ),
11456                )
11457                .await
11458                .unwrap()
11459        });
11460
11461        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
11462        // the very mutation it exists to catch -- a guard that admits nobody -- no
11463        // bind is ever sent, so it HUNG rather than failing. That reproduces the
11464        // exact defect being fixed: a total revocation detected only as a stalled
11465        // suite, which reads as flakiness and invites a retry. An acceptance test
11466        // that waits for an effect must bound the wait, or a red becomes a hang.
11467        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11468            .await
11469            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11470            .expect("module control channel closed before route.bind");
11471        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11472        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11473            panic!("expected route.bind")
11474        };
11475        assert_eq!(
11476            principal,
11477            Some(Principal::Reserved {
11478                module_id: "fed".to_string()
11479            }),
11480            "a correctly attested consumer must be stamped Reserved for its own id"
11481        );
11482
11483        handler
11484            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11485            .await
11486            .unwrap();
11487        assert!(route_task.await.unwrap().is_empty());
11488        assert!(
11489            matches!(
11490                serde_json::from_slice::<ClientControlResponse>(
11491                    &client_rx.recv().await.unwrap().body
11492                )
11493                .unwrap(),
11494                ClientControlResponse::RouteOpen { .. }
11495            ),
11496            "the route must actually open, not merely avoid an error"
11497        );
11498    }
11499
11500    #[tokio::test(start_paused = true)]
11501    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11502        let registry = Arc::new(Registry::default());
11503        let forwarding = Arc::new(ForwardingTable::default());
11504        let supervisor = SupervisorHandle::new();
11505        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11506        let handler =
11507            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11508                .with_supervisor(supervisor);
11509
11510        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11511        hello_via_sink(
11512            &handler,
11513            &target_ctx,
11514            &mut target_rx,
11515            hello_frame("target", PROTOCOL_VERSION, 1),
11516        )
11517        .await;
11518
11519        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11520        let direct_handler = handler.clone();
11521        let direct_open = tokio::spawn(async move {
11522            direct_handler
11523                .handle_control_frame(
11524                    &direct_ctx,
11525                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
11526                )
11527                .await
11528                .unwrap()
11529        });
11530        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11531            .await
11532            .expect("no direct route.bind within 5s")
11533            .expect("target control channel closed before direct route.bind");
11534        handler
11535            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11536            .await
11537            .unwrap();
11538        assert!(direct_open.await.unwrap().is_empty());
11539        let _ = direct_rx.recv().await.unwrap();
11540
11541        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11542        let reserved_handler = handler.clone();
11543        let reserved_open = tokio::spawn(async move {
11544            reserved_handler
11545                .handle_control_frame(
11546                    &reserved_ctx,
11547                    route_open_frame_with_admission_facts(
11548                        3,
11549                        "target",
11550                        unique_project_root("admission-facts"),
11551                        Some(ConsumerIdentity {
11552                            module_id: "fed".to_string(),
11553                            launch_nonce: "fed-nonce".to_string(),
11554                        }),
11555                        None,
11556                    ),
11557                )
11558                .await
11559                .unwrap()
11560        });
11561        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11562            .await
11563            .expect("no reserved route.bind within 5s")
11564            .expect("target control channel closed before reserved route.bind");
11565        handler
11566            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11567            .await
11568            .unwrap();
11569        assert!(reserved_open.await.unwrap().is_empty());
11570        let _ = reserved_rx.recv().await.unwrap();
11571
11572        forwarding
11573            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11574            .unwrap();
11575        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11576        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11577            module_id: Some("target".to_string()),
11578        })
11579        .unwrap();
11580        let census_frame =
11581            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11582        let response = handler
11583            .handle_control_frame(&census_ctx, census_frame)
11584            .await
11585            .unwrap()
11586            .pop()
11587            .unwrap();
11588        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11589        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11590        assert!(matches!(
11591            decoded,
11592            ClientControlResponse::SupervisorRoutes { .. }
11593        ));
11594        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11595        assert_eq!(routes.len(), 2);
11596        assert!(routes.iter().all(|route| route["draining"] == true));
11597        // The census carries WHY: the reason the drain was begun with, in the
11598        // route.closing vocabulary, on every draining route this drain marked.
11599        assert!(
11600            routes.iter().all(|route| route["drain_reason"] == "reload"),
11601            "draining routes must name the drain's reason: {routes:?}"
11602        );
11603        assert!(routes.iter().any(|route| {
11604            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11605        }));
11606        assert!(routes.iter().any(|route| {
11607            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11608        }));
11609
11610        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11611            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11612        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11613            std::fs::write(
11614                &golden_path,
11615                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11616            )
11617            .unwrap();
11618        }
11619        let expected: Value =
11620            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11621        assert_eq!(actual, expected);
11622    }
11623
11624    async fn query_live_roots(
11625        handler: &ControlHandler,
11626        module_ctx: &RouteCtx,
11627    ) -> ModuleControlResponseToModule {
11628        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11629        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11630        let response = handler
11631            .handle_control_frame(module_ctx, frame)
11632            .await
11633            .unwrap()
11634            .pop()
11635            .unwrap();
11636        serde_json::from_slice(&response.body).unwrap()
11637    }
11638
11639    #[tokio::test(start_paused = true)]
11640    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11641        let registry = Arc::new(Registry::default());
11642        let forwarding = Arc::new(ForwardingTable::default());
11643        let handler = ControlHandler::with_forwarding(registry, forwarding);
11644        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11645        hello_via_sink(
11646            &handler,
11647            &target_ctx,
11648            &mut target_rx,
11649            hello_frame("target", PROTOCOL_VERSION, 1),
11650        )
11651        .await;
11652        let root = unique_project_root("live-roots-known");
11653        let path = ProjectRootId::from_path_allowing_missing(root.path())
11654            .unwrap()
11655            .as_path()
11656            .to_path_buf();
11657        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11658        let open_handler = handler.clone();
11659        let opened = tokio::spawn(async move {
11660            open_handler
11661                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11662                .await
11663                .unwrap()
11664        });
11665        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11666            .await
11667            .unwrap()
11668            .unwrap();
11669        handler
11670            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11671            .await
11672            .unwrap();
11673        assert!(opened.await.unwrap().is_empty());
11674        let _ = client_rx.recv().await.unwrap();
11675
11676        let root = unique_project_root("live-roots-pending");
11677        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11678            .unwrap()
11679            .as_path()
11680            .to_path_buf();
11681        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11682        let open_handler = handler.clone();
11683        let pending = tokio::spawn(async move {
11684            open_handler
11685                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11686                .await
11687                .unwrap()
11688        });
11689        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11690            .await
11691            .unwrap()
11692            .unwrap();
11693        let actual = query_live_roots(&handler, &target_ctx).await;
11694        let ModuleControlResponseToModule::LiveRoots {
11695            roots,
11696            unknown_root_bindings,
11697            total_bindings,
11698        } = actual
11699        else {
11700            panic!("expected live roots")
11701        };
11702        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11703        assert_eq!(unknown_root_bindings, 0);
11704        assert_eq!(
11705            roots.len(),
11706            2,
11707            "root-known arm must retain each canonical root"
11708        );
11709        assert_eq!(
11710            total_bindings,
11711            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11712        );
11713        let counts = roots
11714            .iter()
11715            .map(|root| (root.project_root.clone(), root.bound, root.pending))
11716            .collect::<Vec<_>>();
11717        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11718        expected.sort_by(|a, b| a.0.cmp(&b.0));
11719        assert_eq!(
11720            counts, expected,
11721            "roots must sort by path and count pending separately"
11722        );
11723        handler
11724            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11725            .await
11726            .unwrap();
11727        assert!(pending.await.unwrap().is_empty());
11728    }
11729
11730    #[tokio::test(start_paused = true)]
11731    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11732        let forwarding = Arc::new(ForwardingTable::default());
11733        let handler =
11734            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11735        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11736        hello_via_sink(
11737            &handler,
11738            &target_ctx,
11739            &mut target_rx,
11740            hello_frame("target", PROTOCOL_VERSION, 1),
11741        )
11742        .await;
11743        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11744        let pending = forwarding
11745            .begin_route_bind_relay_for_test(
11746                client_ctx.connection_id,
11747                client_ctx.egress.clone(),
11748                2,
11749                "target",
11750            )
11751            .unwrap();
11752        forwarding
11753            .complete_pending_relay(
11754                target_ctx.connection_id,
11755                pending.corr,
11756                RouteBindRelayOutcome::Accepted,
11757            )
11758            .unwrap();
11759        let actual = query_live_roots(&handler, &target_ctx).await;
11760        let ModuleControlResponseToModule::LiveRoots {
11761            roots,
11762            unknown_root_bindings,
11763            total_bindings,
11764        } = actual
11765        else {
11766            panic!("expected live roots")
11767        };
11768        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11769        assert_eq!(
11770            unknown_root_bindings, 1,
11771            "unknown-root arm must not read as no bindings"
11772        );
11773        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11774        assert_eq!(
11775            total_bindings,
11776            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11777        );
11778    }
11779
11780    /// A module reads HELLO_ACK as its first frame and exits on anything else,
11781    /// so the ack has to be on its outbound queue before the module is
11782    /// routable. The connection loop writes a handler's replies only after the
11783    /// handler returns; this test stops in exactly that gap, runs a real
11784    /// route.open from another connection, and only then writes whatever the
11785    /// HELLO handler returned, the way the loop would. If the ack were still a
11786    /// reply, the route.bind request would reach the module first.
11787    #[tokio::test(start_paused = true)]
11788    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11789        let forwarding = Arc::new(ForwardingTable::default());
11790        let handler =
11791            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11792        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11793        let replies = handler
11794            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11795            .await
11796            .unwrap();
11797        let queued_by_hello = module_rx.len();
11798
11799        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11800        let open_handler = handler.clone();
11801        let open = tokio::spawn(async move {
11802            open_handler
11803                .handle_control_frame(
11804                    &client_ctx,
11805                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11806                )
11807                .await
11808                .unwrap()
11809        });
11810        // Let the route.open run until its route.bind is on the module's queue.
11811        let mut spins = 0;
11812        while module_rx.len() == queued_by_hello {
11813            spins += 1;
11814            assert!(spins < 10_000, "route.open never queued a route.bind");
11815            tokio::task::yield_now().await;
11816        }
11817
11818        // Now the connection loop's half: write the HELLO handler's replies.
11819        for reply in replies {
11820            module_ctx.egress.send(reply).await.unwrap();
11821        }
11822
11823        let first = module_rx.recv().await.unwrap().frame;
11824        assert_eq!(
11825            first.header.ty,
11826            FrameType::HelloAck,
11827            "the first frame a registering module reads must be its HELLO_ACK"
11828        );
11829        assert_eq!(first.header.corr, 7);
11830        let second = module_rx.recv().await.unwrap().frame;
11831        assert_eq!(second.header.ty, FrameType::Request);
11832        assert!(
11833            matches!(
11834                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11835                ModuleControlRequest::RouteBind { .. }
11836            ),
11837            "the route.bind follows the ack"
11838        );
11839        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11840
11841        handler
11842            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11843            .await
11844            .unwrap();
11845        assert!(open.await.unwrap().is_empty());
11846        let _ = client_rx.recv().await.unwrap();
11847    }
11848
11849    #[tokio::test(start_paused = true)]
11850    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11851        let handler = ControlHandler::with_forwarding(
11852            Arc::new(Registry::default()),
11853            Arc::new(ForwardingTable::default()),
11854        );
11855        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11856        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11857        hello_via_sink(
11858            &handler,
11859            &first_ctx,
11860            &mut first_rx,
11861            hello_frame("first", PROTOCOL_VERSION, 1),
11862        )
11863        .await;
11864        hello_via_sink(
11865            &handler,
11866            &second_ctx,
11867            &mut second_rx,
11868            hello_frame("second", PROTOCOL_VERSION, 2),
11869        )
11870        .await;
11871        let root = unique_project_root("second-only");
11872        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11873        let cloned = handler.clone();
11874        let open = tokio::spawn(async move {
11875            cloned
11876                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11877                .await
11878                .unwrap()
11879        });
11880        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11881            .await
11882            .unwrap()
11883            .unwrap();
11884        let first = query_live_roots(&handler, &first_ctx).await;
11885        let second = query_live_roots(&handler, &second_ctx).await;
11886        assert!(
11887            matches!(
11888                first,
11889                ModuleControlResponseToModule::LiveRoots {
11890                    total_bindings: 0,
11891                    ..
11892                }
11893            ),
11894            "cross-module scope must not expose another module's roots"
11895        );
11896        assert!(
11897            matches!(
11898                second,
11899                ModuleControlResponseToModule::LiveRoots {
11900                    total_bindings: 1,
11901                    ..
11902                }
11903            ),
11904            "second module must see its pending route"
11905        );
11906        handler
11907            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11908            .await
11909            .unwrap();
11910        assert!(open.await.unwrap().is_empty());
11911    }
11912
11913    #[tokio::test(start_paused = true)]
11914    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11915        let handler = ControlHandler::with_forwarding(
11916            Arc::new(Registry::default()),
11917            Arc::new(ForwardingTable::default()),
11918        );
11919        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11920        hello_via_sink(
11921            &handler,
11922            &target_ctx,
11923            &mut target_rx,
11924            hello_frame("target", PROTOCOL_VERSION, 1),
11925        )
11926        .await;
11927        let actual = query_live_roots(&handler, &target_ctx).await;
11928        let ModuleControlResponseToModule::LiveRoots {
11929            roots,
11930            unknown_root_bindings,
11931            total_bindings,
11932        } = actual
11933        else {
11934            panic!("expected live roots")
11935        };
11936        assert!(roots.is_empty());
11937        assert_eq!(unknown_root_bindings, 0);
11938        assert_eq!(total_bindings, 0);
11939        assert_eq!(
11940            total_bindings,
11941            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11942        );
11943    }
11944
11945    /// Read the vendored fed corpus rather than hand-building a package.
11946    ///
11947    /// A hand-built object encodes what the test author believed the carrier
11948    /// emits. These vectors are what it actually emits, and one of them exists
11949    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11950    /// ignores additive unknown fields at the traversal emit terminus."
11951    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11952        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11953            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11954        let text = std::fs::read_to_string(&path)
11955            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11956        let vectors: Vec<(String, Value)> = text
11957            .lines()
11958            .filter(|line| !line.trim().is_empty())
11959            .map(|line| {
11960                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11961                let id = entry["corpus_id"]
11962                    .as_str()
11963                    .expect("every vector carries a corpus_id")
11964                    .to_string();
11965                (id, entry["package"].clone())
11966            })
11967            .collect();
11968        // Pin the count: a corpus that silently shrinks would take its coverage
11969        // with it, and a suite reading N-1 vectors reports the same clean pass
11970        // as one reading N.
11971        assert_eq!(
11972            vectors.len(),
11973            3,
11974            "vendored fed corpus changed size; re-sync from subc-federation"
11975        );
11976
11977        // Pin what makes the corpus DISCRIMINATING, not just present.
11978        //
11979        // The relay test below takes its expected value from the corpus, so the
11980        // corpus supplies the test's power to detect a lossy relay rather than
11981        // its correctness. A relay that dropped unrecognised fields would still
11982        // be caught -- but only by a package carrying fields it does not know.
11983        // Shrink every package to the handful of keys any implementation would
11984        // recognise and the test keeps passing over an input that can no longer
11985        // fail, which is the same clean green as a corpus that shrank away.
11986        //
11987        // So assert the precondition rather than duplicating the packages here:
11988        // at least one vector must carry a field beyond the small common set.
11989        // That is one claim to maintain instead of nine, and it fails loudly if
11990        // a re-sync ever flattens the corpus.
11991        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11992        let richest = vectors
11993            .iter()
11994            .filter_map(|(_, package)| package.as_object())
11995            .map(|object| {
11996                object
11997                    .keys()
11998                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11999                    .count()
12000            })
12001            .max()
12002            .unwrap_or(0);
12003        assert!(
12004            richest >= 2,
12005            "vendored corpus no longer carries a package with unmodelled fields, \
12006             so the relay test can no longer distinguish a verbatim relay from a lossy one"
12007        );
12008
12009        vectors
12010    }
12011
12012    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
12013    ///
12014    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
12015    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
12016    /// three-key object, so a relay that quietly dropped fields it did not
12017    /// recognise would satisfy it. These vectors carry nine keys including ones
12018    /// this crate has no type for, so a typed relay fails here and only here.
12019    #[tokio::test]
12020    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12021        for (corpus_id, package) in fed_admission_facts_vectors() {
12022            let registry = Arc::new(Registry::default());
12023            let forwarding = Arc::new(ForwardingTable::default());
12024            let supervisor = SupervisorHandle::new();
12025            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12026            let handler =
12027                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12028                    .with_supervisor(supervisor)
12029                    .with_admission_facts_config(
12030                        Some("fed".to_string()),
12031                        Some(vec!["target".to_string()]),
12032                    );
12033
12034            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12035            hello_via_sink(
12036                &handler,
12037                &target_ctx,
12038                &mut target_rx,
12039                hello_frame("target", PROTOCOL_VERSION, 1),
12040            )
12041            .await;
12042
12043            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12044            let route_handler = handler.clone();
12045            let expected = package.clone();
12046            let route_task = tokio::spawn(async move {
12047                route_handler
12048                    .handle_control_frame(
12049                        &client_ctx,
12050                        route_open_frame_with_admission_facts(
12051                            20,
12052                            "target",
12053                            unique_project_root("admission-facts"),
12054                            Some(subc_control::ConsumerIdentity {
12055                                module_id: "fed".to_string(),
12056                                launch_nonce: "fed-nonce".to_string(),
12057                            }),
12058                            Some(package),
12059                        ),
12060                    )
12061                    .await
12062                    .unwrap()
12063            });
12064
12065            let bind_frame = target_rx.recv().await.unwrap();
12066            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12067            let ModuleControlRequest::RouteBind {
12068                admission_facts, ..
12069            } = bind
12070            else {
12071                panic!("{corpus_id}: expected route.bind")
12072            };
12073            assert_eq!(
12074                admission_facts,
12075                Some(expected),
12076                "{corpus_id}: relay must not add, drop or reshape any field"
12077            );
12078
12079            handler
12080                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12081                .await
12082                .unwrap();
12083            route_task.await.unwrap();
12084        }
12085    }
12086
12087    #[tokio::test]
12088    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12089        let registry = Arc::new(Registry::default());
12090        let forwarding = Arc::new(ForwardingTable::default());
12091        let supervisor = SupervisorHandle::new();
12092        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12093        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12094        let handler =
12095            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12096                .with_supervisor(supervisor)
12097                .with_admission_facts_config(
12098                    Some("fed".to_string()),
12099                    Some(vec!["target".to_string()]),
12100                );
12101
12102        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12103        hello_via_sink(
12104            &handler,
12105            &target_ctx,
12106            &mut target_rx,
12107            hello_frame("target", PROTOCOL_VERSION, 1),
12108        )
12109        .await;
12110        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12111        hello_via_sink(
12112            &handler,
12113            &other_ctx,
12114            &mut other_rx,
12115            hello_frame("other", PROTOCOL_VERSION, 2),
12116        )
12117        .await;
12118
12119        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12120        let expected_facts = facts.clone();
12121        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12122        let route_handler = handler.clone();
12123        let route_task = tokio::spawn(async move {
12124            route_handler
12125                .handle_control_frame(
12126                    &client_ctx,
12127                    route_open_frame_with_admission_facts(
12128                        10,
12129                        "target",
12130                        unique_project_root("admission-facts"),
12131                        Some(subc_control::ConsumerIdentity {
12132                            module_id: "fed".to_string(),
12133                            launch_nonce: "fed-nonce".to_string(),
12134                        }),
12135                        Some(facts.clone()),
12136                    ),
12137                )
12138                .await
12139                .unwrap()
12140        });
12141        let bind_frame = target_rx.recv().await.unwrap();
12142        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12143        let ModuleControlRequest::RouteBind {
12144            admission_facts, ..
12145        } = bind
12146        else {
12147            panic!("expected route.bind")
12148        };
12149        assert_eq!(admission_facts, Some(expected_facts));
12150        handler
12151            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12152            .await
12153            .unwrap();
12154        assert!(route_task.await.unwrap().is_empty());
12155        assert!(matches!(
12156            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12157                .unwrap(),
12158            ClientControlResponse::RouteOpen { .. }
12159        ));
12160
12161        let direct = handler
12162            .handle_control_frame(
12163                &route_ctx(ConnectionId::new(73)).0,
12164                route_open_frame_with_admission_facts(
12165                    11,
12166                    "target",
12167                    unique_project_root("admission-facts"),
12168                    None,
12169                    Some(json!({"x": 1})),
12170                ),
12171            )
12172            .await
12173            .unwrap();
12174        assert_eq!(
12175            parse_error(&direct[0])["code"],
12176            "admission_facts_not_permitted"
12177        );
12178
12179        let different_reserved = handler
12180            .handle_control_frame(
12181                &route_ctx(ConnectionId::new(77)).0,
12182                route_open_frame_with_admission_facts(
12183                    15,
12184                    "target",
12185                    unique_project_root("admission-facts"),
12186                    Some(subc_control::ConsumerIdentity {
12187                        module_id: "other".to_string(),
12188                        launch_nonce: "other-nonce".to_string(),
12189                    }),
12190                    Some(json!({"x": 1})),
12191                ),
12192            )
12193            .await
12194            .unwrap();
12195        assert_eq!(
12196            parse_error(&different_reserved[0])["code"],
12197            "admission_facts_not_permitted"
12198        );
12199
12200        let other_target = handler
12201            .handle_control_frame(
12202                &route_ctx(ConnectionId::new(74)).0,
12203                route_open_frame_with_admission_facts(
12204                    12,
12205                    "other",
12206                    unique_project_root("admission-facts"),
12207                    Some(subc_control::ConsumerIdentity {
12208                        module_id: "fed".to_string(),
12209                        launch_nonce: "fed-nonce".to_string(),
12210                    }),
12211                    Some(json!({"x": 1})),
12212                ),
12213            )
12214            .await
12215            .unwrap();
12216        assert_eq!(
12217            parse_error(&other_target[0])["code"],
12218            "admission_facts_target_not_allowed"
12219        );
12220
12221        let nonexistent = handler
12222            .handle_control_frame(
12223                &route_ctx(ConnectionId::new(75)).0,
12224                route_open_frame_with_admission_facts(
12225                    13,
12226                    "missing",
12227                    unique_project_root("admission-facts"),
12228                    None,
12229                    Some(json!({"x": 1})),
12230                ),
12231            )
12232            .await
12233            .unwrap();
12234        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12235
12236        let described = handler
12237            .handle_control_frame(
12238                &route_ctx(ConnectionId::new(76)).0,
12239                Frame::build(
12240                    FrameType::Request,
12241                    control_flags(),
12242                    0,
12243                    0,
12244                    14,
12245                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12246                )
12247                .unwrap(),
12248            )
12249            .await
12250            .unwrap();
12251        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12252            serde_json::from_slice(&described[0].body).unwrap()
12253        else {
12254            panic!("expected server.describe response")
12255        };
12256        assert!(capabilities
12257            .iter()
12258            .any(|cap| cap == "admission_facts_relay_v1"));
12259    }
12260
12261    #[tokio::test]
12262    async fn admission_facts_without_configured_carrier_are_rejected() {
12263        let registry = Arc::new(Registry::default());
12264        let forwarding = Arc::new(ForwardingTable::default());
12265        let handler = ControlHandler::with_forwarding(registry, forwarding);
12266        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12267        hello_via_sink(
12268            &handler,
12269            &target_ctx,
12270            &mut target_rx,
12271            hello_frame("target", PROTOCOL_VERSION, 1),
12272        )
12273        .await;
12274
12275        let responses = handler
12276            .handle_control_frame(
12277                &route_ctx(ConnectionId::new(79)).0,
12278                route_open_frame_with_admission_facts(
12279                    16,
12280                    "target",
12281                    unique_project_root("admission-facts"),
12282                    None,
12283                    Some(json!({"x": 1})),
12284                ),
12285            )
12286            .await
12287            .unwrap();
12288        assert_eq!(
12289            parse_error(&responses[0])["code"],
12290            "admission_facts_not_permitted"
12291        );
12292    }
12293
12294    #[tokio::test]
12295    async fn route_open_relays_consumer_capabilities_verbatim() {
12296        let registry = Arc::new(Registry::default());
12297        let forwarding = Arc::new(ForwardingTable::default());
12298        let handler =
12299            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12300        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12301        hello_via_sink(
12302            &handler,
12303            &module_ctx,
12304            &mut module_rx,
12305            hello_frame("aft", PROTOCOL_VERSION, 7),
12306        )
12307        .await;
12308
12309        let expected = vec!["elicitation".to_string(), "roots".to_string()];
12310        let expected_for_request = expected.clone();
12311        let project_root = unique_project_root("consumer-capabilities-present");
12312        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12313        let route_handler = handler.clone();
12314        let route_task = tokio::spawn(async move {
12315            route_handler
12316                .handle_control_frame(
12317                    &client_ctx,
12318                    route_open_frame_with_consumer_capabilities(
12319                        401,
12320                        "aft",
12321                        project_root,
12322                        Some(expected_for_request),
12323                    ),
12324                )
12325                .await
12326                .unwrap()
12327        });
12328        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12329            .await
12330            .unwrap()
12331            .unwrap();
12332        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12333        let ModuleControlRequest::RouteBind {
12334            consumer_capabilities,
12335            ..
12336        } = bind
12337        else {
12338            panic!("expected route.bind request, got {bind:?}");
12339        };
12340        assert_eq!(consumer_capabilities, Some(expected.clone()));
12341
12342        handler
12343            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12344            .await
12345            .unwrap();
12346        let route_response = route_task.await.unwrap();
12347        assert!(route_response.is_empty());
12348        let published = client_rx.recv().await.unwrap();
12349        assert!(matches!(
12350            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12351            ClientControlResponse::RouteOpen { .. }
12352        ));
12353    }
12354
12355    #[tokio::test]
12356    async fn route_open_without_consumer_capabilities_relays_none() {
12357        let registry = Arc::new(Registry::default());
12358        let forwarding = Arc::new(ForwardingTable::default());
12359        let handler =
12360            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12361        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12362        hello_via_sink(
12363            &handler,
12364            &module_ctx,
12365            &mut module_rx,
12366            hello_frame("aft", PROTOCOL_VERSION, 7),
12367        )
12368        .await;
12369
12370        let project_root = unique_project_root("consumer-capabilities-absent");
12371        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12372        let route_handler = handler.clone();
12373        let route_task = tokio::spawn(async move {
12374            route_handler
12375                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12376                .await
12377                .unwrap()
12378        });
12379        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12380            .await
12381            .unwrap()
12382            .unwrap();
12383        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12384        let ModuleControlRequest::RouteBind {
12385            consumer_capabilities,
12386            ..
12387        } = bind
12388        else {
12389            panic!("expected route.bind request, got {bind:?}");
12390        };
12391        assert_eq!(consumer_capabilities, None);
12392
12393        handler
12394            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12395            .await
12396            .unwrap();
12397        let route_response = route_task.await.unwrap();
12398        assert!(route_response.is_empty());
12399        let published = client_rx.recv().await.unwrap();
12400        assert!(matches!(
12401            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12402            ClientControlResponse::RouteOpen { .. }
12403        ));
12404    }
12405
12406    /// Opens a route to a freshly registered `aft` with `sent` as the
12407    /// route.open's role_versions, acks the bind, and returns the role_versions
12408    /// the module's bind carried.
12409    async fn bind_role_versions_for(
12410        sent: Option<BTreeMap<String, String>>,
12411        connection: u64,
12412    ) -> Option<BTreeMap<String, String>> {
12413        let registry = Arc::new(Registry::default());
12414        let forwarding = Arc::new(ForwardingTable::default());
12415        let handler =
12416            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12417        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12418        hello_via_sink(
12419            &handler,
12420            &module_ctx,
12421            &mut module_rx,
12422            hello_frame("aft", PROTOCOL_VERSION, 7),
12423        )
12424        .await;
12425        let project_root = unique_project_root("role-versions");
12426        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12427        let route_handler = handler.clone();
12428        let route_task = tokio::spawn(async move {
12429            route_handler
12430                .handle_control_frame(
12431                    &client_ctx,
12432                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
12433                )
12434                .await
12435                .unwrap()
12436        });
12437        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12438            .await
12439            .expect("a well-formed route.open reaches the module as a bind")
12440            .unwrap();
12441        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12442        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12443            panic!("expected route.bind request, got {bind:?}");
12444        };
12445        handler
12446            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12447            .await
12448            .unwrap();
12449        assert!(route_task.await.unwrap().is_empty());
12450        let published = client_rx.recv().await.unwrap();
12451        assert!(matches!(
12452            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12453            ClientControlResponse::RouteOpen { .. }
12454        ));
12455        role_versions
12456    }
12457
12458    #[tokio::test]
12459    async fn route_open_relays_role_versions_verbatim() {
12460        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12461        assert_eq!(
12462            bind_role_versions_for(Some(sent.clone()), 141).await,
12463            Some(sent)
12464        );
12465    }
12466
12467    /// An empty map declares nothing, so the provider sees no field rather
12468    /// than an empty object it would have to treat as a second "none".
12469    #[tokio::test]
12470    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12471        assert_eq!(bind_role_versions_for(None, 143).await, None);
12472        assert_eq!(
12473            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12474            None
12475        );
12476    }
12477
12478    #[tokio::test]
12479    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12480        let registry = Arc::new(Registry::default());
12481        let forwarding = Arc::new(ForwardingTable::default());
12482        let handler =
12483            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12484        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12485        hello_via_sink(
12486            &handler,
12487            &module_ctx,
12488            &mut module_rx,
12489            hello_frame("aft", PROTOCOL_VERSION, 7),
12490        )
12491        .await;
12492
12493        let nine: BTreeMap<String, String> = (0..9)
12494            .map(|index| (format!("role-{index}"), "v1".to_string()))
12495            .collect();
12496        for (label, malformed) in [
12497            (
12498                "invalid role name",
12499                role_versions(&[("Tool_Provider", "v1")]),
12500            ),
12501            ("invalid version", role_versions(&[("tool-provider", "v0")])),
12502            ("nine entries", nine),
12503        ] {
12504            // A refused open answers at once; one that reached the module
12505            // would wait for its bind ack and trip this timeout.
12506            let responses = tokio::time::timeout(
12507                Duration::from_secs(1),
12508                handler.handle_control_frame(
12509                    &route_ctx(ConnectionId::new(148)).0,
12510                    route_open_frame_with_role_versions(
12511                        404,
12512                        "aft",
12513                        unique_project_root("role-versions-malformed"),
12514                        Some(malformed),
12515                    ),
12516                ),
12517            )
12518            .await
12519            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12520            .unwrap();
12521            assert_eq!(responses.len(), 1, "{label}");
12522            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12523            let error = parse_error(&responses[0]);
12524            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12525            assert_eq!(
12526                error["detail"]["field"], "role_versions",
12527                "{label}: {error}"
12528            );
12529            assert!(
12530                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12531                "{label}: a malformed declaration is terminal"
12532            );
12533            assert!(
12534                module_rx.try_recv().is_err(),
12535                "{label}: the module must never see a bind"
12536            );
12537        }
12538    }
12539
12540    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
12541    /// consumer can tell this daemon forwards the field from one that would
12542    /// drop it.
12543    #[tokio::test]
12544    async fn route_role_versions_capability_is_advertised() {
12545        let handler = ControlHandler::new(Arc::new(Registry::default()));
12546        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12547        let ack = hello_via_sink(
12548            &handler,
12549            &ctx,
12550            &mut rx,
12551            hello_frame("m", PROTOCOL_VERSION, 1),
12552        )
12553        .await;
12554        let ack = parse_ack(&ack);
12555        assert!(
12556            ack.subc_capabilities
12557                .iter()
12558                .any(|c| c == "route-role-versions/v1"),
12559            "{:?}",
12560            ack.subc_capabilities
12561        );
12562
12563        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12564        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12565        let reply = handler
12566            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12567            .await
12568            .unwrap()
12569            .pop()
12570            .unwrap();
12571        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12572            serde_json::from_slice(&reply.body).unwrap()
12573        else {
12574            panic!("not a server.describe reply");
12575        };
12576        assert!(
12577            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12578            "{capabilities:?}"
12579        );
12580    }
12581
12582    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12583    /// catalog entry without `events` (the module declares none) from a daemon
12584    /// that never reports declared events.
12585    #[tokio::test]
12586    async fn catalog_events_capability_is_advertised() {
12587        let handler = ControlHandler::new(Arc::new(Registry::default()));
12588        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12589        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12590        let reply = handler
12591            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12592            .await
12593            .unwrap()
12594            .pop()
12595            .unwrap();
12596        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12597            serde_json::from_slice(&reply.body).unwrap()
12598        else {
12599            panic!("not a server.describe reply");
12600        };
12601        assert!(
12602            capabilities
12603                .iter()
12604                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12605            "{capabilities:?}"
12606        );
12607    }
12608
12609    #[tokio::test]
12610    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12611        let handler = ControlHandler::new(Arc::new(Registry::default()));
12612        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12613        let ack = hello_via_sink(
12614            &handler,
12615            &module,
12616            &mut module_rx,
12617            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12618        )
12619        .await;
12620        let ack = parse_ack(&ack);
12621        assert!(
12622            ack.subc_capabilities
12623                .iter()
12624                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12625            "{:?}",
12626            ack.subc_capabilities
12627        );
12628        assert!(ack
12629            .subc_capabilities
12630            .iter()
12631            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12632
12633        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12634        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12635        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12636        let reply = handler
12637            .handle_control_frame(&client, frame)
12638            .await
12639            .unwrap()
12640            .pop()
12641            .unwrap();
12642        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12643            serde_json::from_slice(&reply.body).unwrap()
12644        else {
12645            panic!("not a server.describe reply");
12646        };
12647        assert!(
12648            capabilities
12649                .iter()
12650                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12651            "{capabilities:?}"
12652        );
12653        assert!(capabilities
12654            .iter()
12655            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12656    }
12657
12658    #[tokio::test]
12659    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12660        let handler = ControlHandler::new(Arc::new(Registry::default()));
12661        let (module, mut rx) = route_ctx(ConnectionId::new(156));
12662        let ack = hello_via_sink(
12663            &handler,
12664            &module,
12665            &mut rx,
12666            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12667        )
12668        .await;
12669        assert!(parse_ack(&ack)
12670            .subc_capabilities
12671            .iter()
12672            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12673        assert!(parse_ack(&ack)
12674            .subc_capabilities
12675            .iter()
12676            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12677        let (client, _rx) = route_ctx(ConnectionId::new(157));
12678        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12679        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12680        let reply = handler
12681            .handle_control_frame(&client, frame)
12682            .await
12683            .unwrap()
12684            .pop()
12685            .unwrap();
12686        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12687            serde_json::from_slice(&reply.body).unwrap()
12688        else {
12689            panic!("server.describe reply")
12690        };
12691        assert!(capabilities
12692            .iter()
12693            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12694        assert!(capabilities
12695            .iter()
12696            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12697    }
12698
12699    #[tokio::test]
12700    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12701        let handler = ControlHandler::new(Arc::new(Registry::default()));
12702        let (module, mut rx) = route_ctx(ConnectionId::new(158));
12703        let ack = hello_via_sink(
12704            &handler,
12705            &module,
12706            &mut rx,
12707            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12708        )
12709        .await;
12710        assert!(parse_ack(&ack)
12711            .subc_capabilities
12712            .iter()
12713            .any(|cap| cap == "scope-workdir/v1"));
12714        let (client, _rx) = route_ctx(ConnectionId::new(159));
12715        let frame = Frame::build(
12716            FrameType::Request,
12717            control_flags(),
12718            0,
12719            0,
12720            5,
12721            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12722        )
12723        .unwrap();
12724        let reply = handler
12725            .handle_control_frame(&client, frame)
12726            .await
12727            .unwrap()
12728            .remove(0);
12729        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12730            serde_json::from_slice(&reply.body).unwrap()
12731        else {
12732            panic!("server.describe reply")
12733        };
12734        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12735    }
12736
12737    #[tokio::test]
12738    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
12739        let handler = ControlHandler::new(Arc::new(Registry::default()));
12740        let (module, mut rx) = route_ctx(ConnectionId::new(160));
12741        let ack = hello_via_sink(
12742            &handler,
12743            &module,
12744            &mut rx,
12745            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
12746        )
12747        .await;
12748        assert!(parse_ack(&ack)
12749            .subc_capabilities
12750            .iter()
12751            .any(|cap| cap == "worker-runs/v1"));
12752        let (client, _rx) = route_ctx(ConnectionId::new(161));
12753        let frame = Frame::build(
12754            FrameType::Request,
12755            control_flags(),
12756            0,
12757            0,
12758            5,
12759            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12760        )
12761        .unwrap();
12762        let reply = handler
12763            .handle_control_frame(&client, frame)
12764            .await
12765            .unwrap()
12766            .remove(0);
12767        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12768            serde_json::from_slice(&reply.body).unwrap()
12769        else {
12770            panic!("server.describe reply")
12771        };
12772        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
12773    }
12774
12775    #[tokio::test]
12776    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12777        let handler = ControlHandler::new(Arc::new(Registry::default()));
12778        let (module, mut rx) = route_ctx(ConnectionId::new(154));
12779        let ack = hello_via_sink(
12780            &handler,
12781            &module,
12782            &mut rx,
12783            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12784        )
12785        .await;
12786        assert!(parse_ack(&ack)
12787            .subc_capabilities
12788            .iter()
12789            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12790        let (client, _rx) = route_ctx(ConnectionId::new(155));
12791        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12792        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12793        let reply = handler
12794            .handle_control_frame(&client, frame)
12795            .await
12796            .unwrap()
12797            .pop()
12798            .unwrap();
12799        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12800            serde_json::from_slice(&reply.body).unwrap()
12801        else {
12802            panic!("not a server.describe reply")
12803        };
12804        assert!(capabilities
12805            .iter()
12806            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12807    }
12808
12809    #[tokio::test]
12810    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12811        let registry = Arc::new(Registry::default());
12812        let forwarding = Arc::new(ForwardingTable::default());
12813        let handler =
12814            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12815                .with_health_probe_timeout(Duration::from_secs(5));
12816        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12817        hello_via_sink(
12818            &handler,
12819            &module_ctx,
12820            &mut module_rx,
12821            non_routable_hello_frame_with_control_ops(
12822                "mcp",
12823                300,
12824                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12825            ),
12826        )
12827        .await;
12828        assert!(registry
12829            .get_module("mcp")
12830            .unwrap()
12831            .unwrap()
12832            .manifest
12833            .provides
12834            .is_empty());
12835
12836        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12837        let route_response = handler
12838            .handle_control_frame(
12839                &route_client_ctx,
12840                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12841            )
12842            .await
12843            .unwrap();
12844        assert_eq!(route_response[0].header.ty, FrameType::Error);
12845        assert_eq!(
12846            parse_error(&route_response[0])["code"],
12847            "target_unavailable"
12848        );
12849        assert!(parse_error(&route_response[0])["message"]
12850            .as_str()
12851            .unwrap()
12852            .contains("does not provide the requested target"));
12853        assert!(module_rx.try_recv().is_err());
12854
12855        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12856        let health_handler = handler.clone();
12857        let health_task = tokio::spawn(async move {
12858            health_handler
12859                .handle_control_frame(
12860                    &health_client_ctx,
12861                    supervisor_health_probe_frame(302, "mcp"),
12862                )
12863                .await
12864                .unwrap()
12865        });
12866        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12867            .await
12868            .unwrap()
12869            .unwrap();
12870        assert_eq!(
12871            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12872            ModuleControlRequest::HealthCheck {}
12873        );
12874        handler
12875            .handle_control_frame(
12876                &module_ctx,
12877                health_response(health_frame.header.corr, HealthStatus::Ok),
12878            )
12879            .await
12880            .unwrap();
12881        let health_response = health_task.await.unwrap();
12882        assert_eq!(health_response[0].header.ty, FrameType::Response);
12883        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12884            ClientControlResponse::SupervisorHealthProbe {
12885                module_id, status, ..
12886            } => {
12887                assert_eq!(module_id, "mcp");
12888                assert_eq!(status, HealthStatus::Ok);
12889            }
12890            other => panic!("unexpected health response: {other:?}"),
12891        }
12892
12893        // Exercise the forwarding cleanup path directly while leaving the registry
12894        // advertisement in place. If cleanup leaves a stale control sink behind,
12895        // the next probe will enqueue onto it and wait for the long probe timeout
12896        // instead of returning an immediate no-connection error.
12897        forwarding
12898            .cleanup_connection(module_ctx.connection_id)
12899            .unwrap();
12900        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12901        let cleanup_response = tokio::time::timeout(
12902            Duration::from_millis(200),
12903            handler.handle_control_frame(
12904                &cleanup_probe_ctx,
12905                supervisor_health_probe_frame(303, "mcp"),
12906            ),
12907        )
12908        .await
12909        .expect("probe should fail immediately when the control lane is gone")
12910        .unwrap();
12911        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12912        assert_eq!(
12913            parse_error(&cleanup_response[0])["code"],
12914            "target_unavailable"
12915        );
12916        assert!(parse_error(&cleanup_response[0])["message"]
12917            .as_str()
12918            .unwrap()
12919            .contains("no module connection"));
12920
12921        handler
12922            .cleanup_connection(module_ctx.connection_id)
12923            .unwrap();
12924    }
12925
12926    #[tokio::test]
12927    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12928        let registry = Arc::new(Registry::default());
12929        let supervisor_handle = SupervisorHandle::new();
12930        let supervisor =
12931            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12932                .with_handle(supervisor_handle.clone())
12933                .with_connection_file_path(
12934                    std::env::temp_dir()
12935                        .join(format!("subc-route-open-warming-{}", std::process::id())),
12936                );
12937        let module = supervisor
12938            .supervise_configured(
12939                ModuleSpec {
12940                    module_id: "warming".to_string(),
12941                    program: fake_aft_stub_path(),
12942                    args: Vec::new(),
12943                    env: Vec::new(),
12944                    reserved: false,
12945                    reserved_prefixes: Vec::new(),
12946                    protocol: ModuleProtocol::Subc,
12947                    overlap: Default::default(),
12948                },
12949                true,
12950            )
12951            .unwrap();
12952        assert_eq!(module.state().unwrap(), ModuleState::Running);
12953
12954        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12955        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12956        let response = handler
12957            .handle_control_frame(
12958                &ctx,
12959                route_open_frame(304, "warming", unique_project_root("warming")),
12960            )
12961            .await
12962            .unwrap();
12963        module.stop().await.unwrap();
12964
12965        assert_eq!(response[0].header.ty, FrameType::Error);
12966        let error = parse_error(&response[0]);
12967        assert_eq!(error["code"], "module_warming");
12968        assert!(error["message"]
12969            .as_str()
12970            .unwrap()
12971            .contains("state=running, enabled=true, live=false"));
12972    }
12973
12974    #[test]
12975    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12976        let handler = ControlHandler::new(Arc::new(Registry::default()));
12977        let capture = EventCapture::default();
12978        let _subscriber =
12979            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12980        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12981        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12982        let pending = (0..limit).collect::<Vec<_>>();
12983        let response = handler
12984            .route_open_capacity_refusal(
12985                &ctx,
12986                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12987                "busy",
12988                pending.len(),
12989                limit,
12990            )
12991            .unwrap();
12992        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12993        let event = capture
12994            .events()
12995            .into_iter()
12996            .find(|event| {
12997                event.target == "control"
12998                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12999            })
13000            .expect("connection admission refusal event");
13001        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13002        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13003        // Refused before `handle_route_open` parses the request, so the caller
13004        // fields are read from the frame itself.
13005        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13006        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13007    }
13008
13009    #[test]
13010    fn route_open_target_cap_logs_admission_reason_and_capacity() {
13011        let handler = ControlHandler::new(Arc::new(Registry::default()));
13012        let capture = EventCapture::default();
13013        let _subscriber =
13014            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13015        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13016        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13017        let guards = (0..limit)
13018            .map(|_| {
13019                handler
13020                    .route_bind_concurrency
13021                    .try_admit("busy", limit)
13022                    .unwrap()
13023            })
13024            .collect::<Vec<_>>();
13025        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13026            Err(in_flight) => in_flight,
13027            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13028        };
13029        let response = handler
13030            .route_open_target_capacity_refusal(
13031                &ctx,
13032                &RouteOpenCaller::unknown(),
13033                &route_open_frame(397, "busy", unique_project_root("target-cap")),
13034                "busy",
13035                in_flight,
13036            )
13037            .unwrap();
13038        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13039        let event = capture
13040            .events()
13041            .into_iter()
13042            .find(|event| {
13043                event.target == "control"
13044                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13045            })
13046            .expect("target admission refusal event");
13047        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13048        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13049        drop(guards);
13050    }
13051
13052    #[cfg(unix)]
13053    #[tokio::test(start_paused = true)]
13054    async fn supervisor_list_shows_a_failed_modules_next_retry() {
13055        let registry = Arc::new(Registry::default());
13056        let handle = SupervisorHandle::new();
13057        let supervisor =
13058            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13059                .with_handle(handle.clone());
13060        let module = supervisor
13061            .spawn(crate::ModuleSpec {
13062                module_id: "retry-list".into(),
13063                program: PathBuf::from("/bin/sh"),
13064                args: vec!["-c".into(), "exit 17".into()],
13065                env: vec![],
13066                reserved: false,
13067                reserved_prefixes: vec![],
13068                protocol: ModuleProtocol::None,
13069                overlap: Default::default(),
13070            })
13071            .unwrap();
13072        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13073        while module.state().unwrap() != crate::ModuleState::Failed {
13074            assert!(
13075                std::time::Instant::now() < deadline,
13076                "waiting for budget failure"
13077            );
13078            tokio::task::yield_now().await;
13079        }
13080        let handler = ControlHandler::new(registry).with_supervisor(handle);
13081        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13082        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13083        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13084        assert_eq!(body["modules"][0]["state"], "failed");
13085        let retry = body["modules"][0]["next_retry_at_ms"]
13086            .as_u64()
13087            .expect("scheduled retry on the wire");
13088        let now = std::time::SystemTime::now()
13089            .duration_since(std::time::UNIX_EPOCH)
13090            .unwrap()
13091            .as_millis() as u64;
13092        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13093        module.set_enabled(false).await.unwrap();
13094        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13095        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13096        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13097        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13098    }
13099
13100    #[tokio::test]
13101    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13102        let registry = Arc::new(Registry::default());
13103        let handle = SupervisorHandle::new();
13104        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13105            .with_handle(handle.clone());
13106        supervisor
13107            .supervise_configured(
13108                crate::ModuleSpec {
13109                    module_id: "drift".into(),
13110                    program: PathBuf::from("disabled-module"),
13111                    args: Vec::new(),
13112                    env: Vec::new(),
13113                    reserved: false,
13114                    reserved_prefixes: Vec::new(),
13115                    protocol: ModuleProtocol::Subc,
13116                    overlap: Default::default(),
13117                },
13118                false,
13119            )
13120            .unwrap();
13121        let handler = ControlHandler::new(registry).with_supervisor(handle);
13122        for configured in [None, Some(false), Some(true)] {
13123            handler.capability_evaluator.configure(
13124                configured.map(|enabled| ("drift".to_string(), enabled)),
13125                BTreeMap::new(),
13126            );
13127            let frame =
13128                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13129            let replies = handler
13130                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13131                .await
13132                .unwrap();
13133            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13134            assert!(body.get("configured_enabled").is_none());
13135            assert_eq!(body["op"], "supervisor.ack");
13136            let frame =
13137                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13138            let replies = handler.handle_supervisor_list(frame).await.unwrap();
13139            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13140            assert_eq!(
13141                body["modules"][0].get("configured_enabled"),
13142                configured.map(Value::Bool).as_ref()
13143            );
13144        }
13145    }
13146
13147    #[tokio::test(flavor = "current_thread")]
13148    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13149        let handler = ControlHandler::new(Arc::new(Registry::default()));
13150        let capture = EventCapture::default();
13151        let _subscriber =
13152            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13153        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13154        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13155
13156        let _ = handler
13157            .handle_client_control_request(
13158                &ctx,
13159                frame,
13160                ClientControlRequest::SupervisorSetEnabled {
13161                    module_id: "broca".to_string(),
13162                    enabled: false,
13163                },
13164                &mut None,
13165            )
13166            .await
13167            .unwrap();
13168
13169        let events = capture
13170            .events()
13171            .into_iter()
13172            .filter(|event| {
13173                event.target == "control"
13174                    && event.fields.get("message").map(String::as_str)
13175                        == Some("supervisor request received")
13176            })
13177            .collect::<Vec<_>>();
13178        assert_eq!(events.len(), 1, "one supervisor request log line");
13179        let event = &events[0];
13180        assert_eq!(event.level, tracing::Level::INFO);
13181        assert_eq!(
13182            event.fields.get("op"),
13183            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13184        );
13185        assert_eq!(
13186            event.fields.get("module_id"),
13187            Some(&"\"broca\"".to_string())
13188        );
13189        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13190        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13191        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13192    }
13193
13194    #[tokio::test(flavor = "current_thread")]
13195    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13196        let registry = Arc::new(Registry::default());
13197        let module_connection = ConnectionId::new(303);
13198        registry
13199            .register_with_control_ops(
13200                manifest("aft", PROTOCOL_VERSION),
13201                PROTOCOL_VERSION,
13202                module_connection,
13203                module_baseline_control_ops(),
13204            )
13205            .unwrap();
13206        let handler = ControlHandler::new(registry);
13207        let capture = EventCapture::default();
13208        let _subscriber =
13209            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13210        let (ctx, _rx) = route_ctx(module_connection);
13211        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13212
13213        let _ = handler
13214            .handle_client_control_request(
13215                &ctx,
13216                frame,
13217                ClientControlRequest::SupervisorSetEnabled {
13218                    module_id: "broca".to_string(),
13219                    enabled: false,
13220                },
13221                &mut None,
13222            )
13223            .await
13224            .unwrap();
13225
13226        let callers = capture
13227            .events()
13228            .into_iter()
13229            .filter(|event| {
13230                event.fields.get("message").map(String::as_str)
13231                    == Some("supervisor request received")
13232            })
13233            .map(|event| event.fields.get("caller").cloned())
13234            .collect::<Vec<_>>();
13235        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13236    }
13237
13238    #[tokio::test(flavor = "current_thread")]
13239    async fn supervisor_list_does_not_log_request_received() {
13240        let handler = ControlHandler::new(Arc::new(Registry::default()));
13241        let capture = EventCapture::default();
13242        let _subscriber =
13243            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13244        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13245        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13246
13247        handler
13248            .handle_client_control_request(
13249                &ctx,
13250                frame,
13251                ClientControlRequest::SupervisorList {},
13252                &mut None,
13253            )
13254            .await
13255            .unwrap();
13256
13257        assert!(capture.events().into_iter().all(|event| {
13258            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13259        }));
13260    }
13261
13262    #[tokio::test(flavor = "current_thread")]
13263    async fn supervisor_rescan_preview_does_not_log_request_received() {
13264        let handler = ControlHandler::new(Arc::new(Registry::default()));
13265        let capture = EventCapture::default();
13266        let _subscriber =
13267            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13268        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13269        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13270
13271        handler
13272            .handle_client_control_request(
13273                &ctx,
13274                frame,
13275                ClientControlRequest::SupervisorRescan { preview: true },
13276                &mut None,
13277            )
13278            .await
13279            .unwrap();
13280
13281        assert!(capture.events().into_iter().all(|event| {
13282            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13283        }));
13284    }
13285
13286    struct ApproveOperator;
13287    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13288        fn prompt(
13289            &self,
13290            _: &str,
13291            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13292        ) -> crate::operator_confirm::ProviderResult {
13293            crate::operator_confirm::ProviderResult::Approved
13294        }
13295    }
13296
13297    #[tokio::test]
13298    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13299        assert_eq!(
13300            ControlHandler::default().privileged_operator_prompt,
13301            PrivilegedOperatorPrompt::Log
13302        );
13303        for (initial, changed, running) in [
13304            ("log", "enforce", PrivilegedOperatorPrompt::Log),
13305            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13306        ] {
13307            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13308            let path = dir.join("subc.jsonc");
13309            std::fs::write(
13310                &path,
13311                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13312            )
13313            .unwrap();
13314            let config = crate::daemon_config::load(&path).unwrap().unwrap();
13315            let registry = Arc::new(Registry::default());
13316            let handle = SupervisorHandle::new();
13317            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13318                .with_handle(handle.clone());
13319            let handler = ControlHandler::new(registry)
13320                .with_supervisor(handle)
13321                .with_privileged_operator_prompt(config.privileged_operator_prompt)
13322                .with_supervisor_rescan(supervisor, &path, config.port);
13323            handler.forwarding.operator_confirms().configure(
13324                Arc::new(ApproveOperator),
13325                crate::operator_confirm::OperatorLimits::default(),
13326            );
13327            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13328
13329            for (disk_mode, needs_restart) in [
13330                (Some(initial), false),
13331                (Some(changed), true),
13332                (Some(changed), true),
13333                (None, running == PrivilegedOperatorPrompt::Enforce),
13334                (Some(initial), false),
13335            ] {
13336                let mut doc = json!({"version":1});
13337                if let Some(mode) = disk_mode {
13338                    doc["privileged_operator_prompt"] = json!(mode);
13339                }
13340                std::fs::write(&path, doc.to_string()).unwrap();
13341                for preview in [true, false] {
13342                    let request = ClientControlRequest::SupervisorRescan { preview };
13343                    let frame = Frame::build(
13344                        FrameType::Request,
13345                        control_flags(),
13346                        0,
13347                        0,
13348                        1,
13349                        serde_json::to_vec(&request).unwrap(),
13350                    )
13351                    .unwrap();
13352                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13353                    assert_eq!(frames.len(), 1);
13354                    let ClientControlResponse::SupervisorRescan { result } =
13355                        serde_json::from_slice(&frames[0].body).unwrap()
13356                    else {
13357                        panic!("expected a rescan response");
13358                    };
13359                    let expected = if needs_restart {
13360                        vec!["privileged_operator_prompt"]
13361                    } else {
13362                        Vec::new()
13363                    };
13364                    assert_eq!(result.restart_required, expected);
13365                    assert_eq!(handler.privileged_operator_prompt, running);
13366                    assert_eq!(
13367                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13368                        running
13369                    );
13370                }
13371            }
13372        }
13373    }
13374
13375    #[tokio::test]
13376    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13377        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13378        let path = dir.join("subc.jsonc");
13379        let registry = Arc::new(Registry::default());
13380        let handle = SupervisorHandle::new();
13381        let supervisor =
13382            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13383        let handler = ControlHandler::new(registry)
13384            .with_supervisor(handle)
13385            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13386            .with_supervisor_rescan(supervisor, &path, None);
13387        handler.forwarding.operator_confirms().configure(
13388            Arc::new(ApproveOperator),
13389            crate::operator_confirm::OperatorLimits::default(),
13390        );
13391        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13392        for preview in [true, false] {
13393            std::fs::write(
13394                &path,
13395                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13396            )
13397            .unwrap();
13398            let request = ClientControlRequest::SupervisorRescan { preview };
13399            let frame = Frame::build(
13400                FrameType::Request,
13401                control_flags(),
13402                0,
13403                0,
13404                1,
13405                serde_json::to_vec(&request).unwrap(),
13406            )
13407            .unwrap();
13408            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13409            assert_eq!(frames.len(), 1);
13410            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13411            assert_eq!(
13412                handler.privileged_operator_prompt,
13413                PrivilegedOperatorPrompt::Enforce
13414            );
13415        }
13416    }
13417
13418    /// One wire code has several senders, so the refusal line names the check
13419    /// that refused. This drives the shared refusal path for ordinary refusals
13420    /// with an unregistered
13421    /// target and requires the branch label on the event.
13422    #[tokio::test]
13423    async fn route_open_refusal_names_the_check_that_refused() {
13424        let handler = ControlHandler::new(Arc::new(Registry::default()));
13425        let capture = EventCapture::default();
13426        let _subscriber =
13427            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13428        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13429        let response = handler
13430            .handle_control_frame(
13431                &ctx,
13432                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13433            )
13434            .await
13435            .unwrap();
13436
13437        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13438        let event = capture
13439            .events()
13440            .into_iter()
13441            .find(|event| {
13442                event.target == "control"
13443                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13444            })
13445            .expect("route.open refusal event");
13446        assert_eq!(
13447            event.fields.get("reason"),
13448            Some(&"\"not_registered\"".to_string())
13449        );
13450    }
13451
13452    #[tokio::test]
13453    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13454        let registry = Arc::new(Registry::default());
13455        let supervisor_handle = SupervisorHandle::new();
13456        let supervisor =
13457            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13458                .with_handle(supervisor_handle.clone())
13459                .with_connection_file_path(std::env::temp_dir().join(format!(
13460                    "subc-route-open-refusal-info-{}",
13461                    std::process::id()
13462                )));
13463        let module = supervisor
13464            .supervise_configured(
13465                ModuleSpec {
13466                    module_id: "warming".to_string(),
13467                    program: fake_aft_stub_path(),
13468                    args: Vec::new(),
13469                    env: Vec::new(),
13470                    reserved: false,
13471                    reserved_prefixes: Vec::new(),
13472                    protocol: ModuleProtocol::Subc,
13473                    overlap: Default::default(),
13474                },
13475                true,
13476            )
13477            .unwrap();
13478        assert_eq!(module.state().unwrap(), ModuleState::Running);
13479
13480        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13481        assert!(handler
13482            .counters()
13483            .snapshot()
13484            .get("route_open_refused_by_code")
13485            .is_none());
13486        let capture = EventCapture::default();
13487        let _subscriber =
13488            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13489        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13490        let response = handler
13491            .handle_control_frame(
13492                &ctx,
13493                route_open_frame(394, "warming", unique_project_root("refusal-info")),
13494            )
13495            .await
13496            .unwrap();
13497        module.stop().await.unwrap();
13498
13499        assert_eq!(parse_error(&response[0])["code"], "module_warming");
13500        let event = capture
13501            .events()
13502            .into_iter()
13503            .find(|event| {
13504                event.target == "control"
13505                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13506            })
13507            .expect("route.open refusal event");
13508        assert_eq!(
13509            event.fields.get("module_id"),
13510            Some(&"\"warming\"".to_string())
13511        );
13512        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13513        assert_eq!(
13514            event.fields.get("reason"),
13515            Some(&"\"supervised_not_registered\"".to_string())
13516        );
13517        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13518        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13519        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13520        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13521        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13522        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13523        assert_eq!(
13524            handler.counters().snapshot()["route_open_refused_by_code"],
13525            json!({ "module_warming": 1 })
13526        );
13527    }
13528
13529    const OUTAGE_START: &str = "route.open refusing module: not serving";
13530    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13531
13532    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13533        capture
13534            .events()
13535            .into_iter()
13536            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13537            .collect()
13538    }
13539
13540    fn supervise_stub(
13541        registry: &Arc<Registry>,
13542        module_id: &str,
13543        enabled: bool,
13544    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13545        let supervisor_handle = SupervisorHandle::new();
13546        let supervisor =
13547            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13548                .with_handle(supervisor_handle.clone())
13549                .with_connection_file_path(std::env::temp_dir().join(format!(
13550                    "subc-route-outage-{module_id}-{}",
13551                    std::process::id()
13552                )));
13553        let module = supervisor
13554            .supervise_configured(
13555                ModuleSpec {
13556                    module_id: module_id.to_string(),
13557                    program: fake_aft_stub_path(),
13558                    args: Vec::new(),
13559                    env: Vec::new(),
13560                    reserved: false,
13561                    reserved_prefixes: Vec::new(),
13562                    protocol: ModuleProtocol::Subc,
13563                    overlap: Default::default(),
13564                },
13565                enabled,
13566            )
13567            .unwrap();
13568        (supervisor_handle, module)
13569    }
13570
13571    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13572        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13573            module_id: module_id.to_string(),
13574            drain_timeout_ms: Some(50),
13575        })
13576        .unwrap();
13577        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13578    }
13579
13580    /// Two handlers built over one forwarding table must share one outage
13581    /// tracker; separate trackers would each log their own opening line for
13582    /// the same outage.
13583    #[test]
13584    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13585        let registry = Arc::new(Registry::default());
13586        let forwarding = Arc::new(ForwardingTable::default());
13587        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13588        let second = ControlHandler::with_forwarding(registry, forwarding);
13589        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13590    }
13591
13592    /// A client can name any module id it likes. Refusing an unknown one,
13593    /// however often, must not create outage state or outage lines, or the
13594    /// tracker would be a memory sink any client could fill.
13595    #[tokio::test(flavor = "current_thread")]
13596    async fn route_open_unknown_module_refusals_add_no_outage_state() {
13597        let handler = ControlHandler::new(Arc::new(Registry::default()));
13598        let capture = EventCapture::default();
13599        let _subscriber =
13600            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13601        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13602        for corr in 0..8 {
13603            let response = handler
13604                .handle_control_frame(
13605                    &ctx,
13606                    route_open_frame(
13607                        380 + corr,
13608                        &format!("nobody-{corr}"),
13609                        unique_project_root("outage-unknown"),
13610                    ),
13611                )
13612                .await
13613                .unwrap();
13614            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13615        }
13616
13617        assert_eq!(handler.route_outages.tracked_module_count(), 0);
13618        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13619        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13620    }
13621
13622    /// Drives the refusal path end to end: a supervised module that served
13623    /// before and stopped being registered with no instruction to stop is a
13624    /// WARN, and the same module refused after an operator `supervisor.restart`
13625    /// is an INFO.
13626    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13627    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13628        let registry = Arc::new(Registry::default());
13629        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
13630        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13631        let capture = EventCapture::default();
13632        let _subscriber =
13633            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13634        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13635        // The stub never registers, so pretend it served once: otherwise every
13636        // refusal would fall in its startup window.
13637        handler.route_outages.record_accepted("outage-restart");
13638
13639        let response = handler
13640            .handle_control_frame(
13641                &ctx,
13642                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13643            )
13644            .await
13645            .unwrap();
13646        assert_eq!(response[0].header.ty, FrameType::Error);
13647        let starts = outage_lines(&capture, OUTAGE_START);
13648        assert_eq!(starts.len(), 1, "{starts:?}");
13649        assert_eq!(starts[0].level, tracing::Level::WARN);
13650        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13651        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13652        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13653        handler.route_outages.record_accepted("outage-restart");
13654        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13655
13656        let restart = handler
13657            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13658            .await
13659            .unwrap();
13660        assert_eq!(
13661            restart[0].header.ty,
13662            FrameType::Response,
13663            "{:?}",
13664            parse_error(&restart[0])
13665        );
13666        handler
13667            .handle_control_frame(
13668                &ctx,
13669                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13670            )
13671            .await
13672            .unwrap();
13673        module.stop().await.unwrap();
13674
13675        let starts = outage_lines(&capture, OUTAGE_START);
13676        assert_eq!(starts.len(), 2, "{starts:?}");
13677        assert_eq!(starts[1].level, tracing::Level::INFO);
13678        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13679    }
13680
13681    /// A restart refused before it touched the module (here: the module is
13682    /// disabled) must clear its operator mark, so the next real outage is
13683    /// still reported as a warning.
13684    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13685    async fn failed_operator_restart_leaves_no_operator_mark() {
13686        let registry = Arc::new(Registry::default());
13687        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
13688        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13689        let capture = EventCapture::default();
13690        let _subscriber =
13691            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13692        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13693        handler.route_outages.record_accepted("outage-disabled");
13694
13695        let restart = handler
13696            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13697            .await
13698            .unwrap();
13699        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13700        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13701
13702        handler
13703            .handle_control_frame(
13704                &ctx,
13705                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13706            )
13707            .await
13708            .unwrap();
13709        let starts = outage_lines(&capture, OUTAGE_START);
13710        assert_eq!(starts.len(), 1, "{starts:?}");
13711        assert_eq!(starts[0].level, tracing::Level::WARN);
13712    }
13713
13714    #[tokio::test(flavor = "current_thread")]
13715    async fn route_open_unknown_module_escapes_target_module_id() {
13716        let handler = ControlHandler::new(Arc::new(Registry::default()));
13717        let capture = EventCapture::default();
13718        let _subscriber =
13719            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13720        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13721        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13722        let response = handler
13723            .handle_control_frame(
13724                &ctx,
13725                route_open_frame(
13726                    395,
13727                    hostile_module_id,
13728                    unique_project_root("hostile-target-module-id"),
13729                ),
13730            )
13731            .await
13732            .unwrap();
13733
13734        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13735        let event = capture
13736            .events()
13737            .into_iter()
13738            .find(|event| {
13739                event.target == "control"
13740                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13741            })
13742            .expect("route.open unknown-module refusal event");
13743        let logged = event.fields.get("module_id").expect("module_id field");
13744        assert!(!logged.bytes().any(|byte| byte < 0x20));
13745        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13746    }
13747
13748    #[tokio::test(flavor = "current_thread")]
13749    async fn route_open_module_rejection_uses_daemon_counter_key() {
13750        let registry = Arc::new(Registry::default());
13751        let forwarding = Arc::new(ForwardingTable::default());
13752        let handler =
13753            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13754        let module_connection = ConnectionId::new(95);
13755        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13756        hello_via_sink(
13757            &handler,
13758            &module_ctx,
13759            &mut module_rx,
13760            hello_frame("aft", PROTOCOL_VERSION, 395),
13761        )
13762        .await;
13763
13764        let client_connection = ConnectionId::new(96);
13765        let (client_ctx, _client_rx) = route_ctx(client_connection);
13766        let capture = EventCapture::default();
13767        let _subscriber =
13768            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13769        let (route_task, bind) = relay_route_open(
13770            &handler,
13771            client_connection,
13772            &client_ctx.egress,
13773            &mut module_rx,
13774            396,
13775            "aft",
13776            "hostile-module-code",
13777        )
13778        .await;
13779        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13780        let rejection = Frame::build(
13781            FrameType::Error,
13782            control_flags(),
13783            0,
13784            0,
13785            bind.header.corr,
13786            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13787        )
13788        .unwrap();
13789        handler
13790            .handle_control_frame(&module_ctx, rejection)
13791            .await
13792            .unwrap();
13793
13794        let response = route_task.await.unwrap();
13795        assert_eq!(parse_error(&response[0])["code"], hostile_code);
13796        let counters = handler.counters().snapshot();
13797        assert_eq!(
13798            counters["route_open_refused_by_code"],
13799            json!({ "module_rejected": 1 })
13800        );
13801        assert!(counters["route_open_refused_by_code"]
13802            .get(hostile_code)
13803            .is_none());
13804
13805        let event = capture
13806            .events()
13807            .into_iter()
13808            .find(|event| {
13809                event.target == "control"
13810                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13811            })
13812            .expect("route.open module-rejection refusal event");
13813        let logged = event.fields.get("module_code").expect("module_code field");
13814        assert!(!logged.bytes().any(|byte| byte < 0x20));
13815        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13816    }
13817
13818    #[tokio::test]
13819    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13820        let registry = Arc::new(Registry::default());
13821        let supervisor_handle = SupervisorHandle::new();
13822        let missing_program = std::env::temp_dir().join(format!(
13823            "subc-route-open-missing-program-{}",
13824            std::process::id()
13825        ));
13826        let supervisor =
13827            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13828                .with_handle(supervisor_handle.clone());
13829        let module = supervisor
13830            .supervise_configured(
13831                ModuleSpec {
13832                    module_id: "failed".to_string(),
13833                    program: missing_program,
13834                    args: Vec::new(),
13835                    env: Vec::new(),
13836                    reserved: false,
13837                    reserved_prefixes: Vec::new(),
13838                    protocol: ModuleProtocol::Subc,
13839                    overlap: Default::default(),
13840                },
13841                true,
13842            )
13843            .unwrap();
13844        assert_eq!(module.state().unwrap(), ModuleState::Failed);
13845
13846        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13847        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13848        let response = handler
13849            .handle_control_frame(
13850                &ctx,
13851                route_open_frame(305, "failed", unique_project_root("failed")),
13852            )
13853            .await
13854            .unwrap();
13855
13856        assert_eq!(response[0].header.ty, FrameType::Error);
13857        let error = parse_error(&response[0]);
13858        assert_eq!(error["code"], "target_unavailable");
13859        assert!(error["message"]
13860            .as_str()
13861            .unwrap()
13862            .contains("state=failed, enabled=true, live=false"));
13863    }
13864
13865    #[tokio::test]
13866    async fn route_open_role_mismatch_remains_target_unavailable() {
13867        let registry = Arc::new(Registry::default());
13868        let handler = ControlHandler::new(Arc::clone(&registry));
13869        handler
13870            .handle_control(
13871                ConnectionId::new(41),
13872                non_routable_hello_frame_with_control_ops("health-only", 306, None),
13873            )
13874            .unwrap();
13875
13876        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13877        let response = handler
13878            .handle_control_frame(
13879                &ctx,
13880                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13881            )
13882            .await
13883            .unwrap();
13884
13885        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13886        assert!(parse_error(&response[0])["message"]
13887            .as_str()
13888            .unwrap()
13889            .contains("does not provide the requested target"));
13890    }
13891
13892    #[tokio::test]
13893    async fn route_open_inactive_registration_remains_target_unavailable() {
13894        let registry = Arc::new(Registry::default());
13895        let handler = ControlHandler::new(Arc::clone(&registry));
13896        handler
13897            .handle_control(
13898                ConnectionId::new(43),
13899                hello_frame("inactive", PROTOCOL_VERSION, 308),
13900            )
13901            .unwrap();
13902        assert!(registry
13903            .set_module_state_for_test("inactive", ChannelState::Closed)
13904            .unwrap());
13905
13906        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13907        let response = handler
13908            .handle_control_frame(
13909                &ctx,
13910                route_open_frame(309, "inactive", unique_project_root("inactive")),
13911            )
13912            .await
13913            .unwrap();
13914
13915        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13916        assert!(parse_error(&response[0])["message"]
13917            .as_str()
13918            .unwrap()
13919            .contains("is not active"));
13920    }
13921
13922    #[tokio::test]
13923    async fn late_health_reply_is_recorded_through_the_module_response_path() {
13924        let registry = Arc::new(Registry::default());
13925        let forwarding = Arc::new(ForwardingTable::default());
13926        let supervisor_handle = SupervisorHandle::new();
13927        let supervisor =
13928            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
13929                .with_forwarding(Arc::clone(&forwarding))
13930                .with_handle(supervisor_handle.clone());
13931        let module = supervisor
13932            .supervise_configured(
13933                crate::ModuleSpec {
13934                    module_id: "late-health-response".to_string(),
13935                    program: PathBuf::from("disabled-module"),
13936                    args: Vec::new(),
13937                    env: Vec::new(),
13938                    reserved: false,
13939                    reserved_prefixes: Vec::new(),
13940                    protocol: ModuleProtocol::Subc,
13941                    overlap: Default::default(),
13942                },
13943                false,
13944            )
13945            .unwrap();
13946        let handler =
13947            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13948                .with_supervisor(supervisor_handle);
13949        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13950        handler
13951            .handle_control_frame(
13952                &module_ctx,
13953                hello_frame_with_control_ops(
13954                    "late-health-response",
13955                    PROTOCOL_VERSION,
13956                    7,
13957                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13958                ),
13959            )
13960            .await
13961            .unwrap();
13962        let probe_started_at = Instant::now() - Duration::from_millis(80);
13963        let pending = forwarding
13964            .begin_health_probe_rpc_for(
13965                "late-health-response",
13966                MODULE_CONTROL_OP_HEALTH_CHECK,
13967                probe_started_at,
13968                Instant::now() - Duration::from_millis(1),
13969            )
13970            .unwrap();
13971        assert!(forwarding
13972            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13973            .unwrap());
13974
13975        let responses = handler
13976            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13977            .await
13978            .unwrap();
13979
13980        assert!(responses.is_empty());
13981        let health = module.status().unwrap().health;
13982        assert_eq!(health.late_answer_count, 1);
13983        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13984    }
13985
13986    #[tokio::test]
13987    async fn health_probe_timeout_and_module_death_are_typed() {
13988        let registry = Arc::new(Registry::default());
13989        let forwarding = Arc::new(ForwardingTable::default());
13990        let handler =
13991            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13992                .with_health_probe_timeout(Duration::from_millis(50));
13993        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13994        hello_via_sink(
13995            &handler,
13996            &module_ctx,
13997            &mut module_rx,
13998            hello_frame_with_control_ops(
13999                "aft",
14000                PROTOCOL_VERSION,
14001                7,
14002                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14003            ),
14004        )
14005        .await;
14006
14007        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14008        let responses = handler
14009            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14010            .await
14011            .unwrap();
14012        assert_eq!(responses[0].header.ty, FrameType::Error);
14013        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14014        let _ = module_rx.try_recv();
14015
14016        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14017        let health_handler = handler.clone();
14018        let death_task = tokio::spawn(async move {
14019            health_handler
14020                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14021                .await
14022                .unwrap()
14023        });
14024        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14025            .await
14026            .unwrap()
14027            .unwrap();
14028        handler
14029            .cleanup_connection(module_ctx.connection_id)
14030            .unwrap();
14031        let responses = death_task.await.unwrap();
14032        assert_eq!(responses[0].header.ty, FrameType::Error);
14033        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14034    }
14035
14036    #[test]
14037    fn hello_requires_exact_protocol_version() {
14038        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14039            let registry = Arc::new(Registry::default());
14040            let handler = ControlHandler::new(Arc::clone(&registry));
14041            let responses = handler
14042                .handle_control(
14043                    ConnectionId::new(connection),
14044                    hello_frame("aft", offered, 9),
14045                )
14046                .unwrap();
14047
14048            assert_eq!(responses.len(), 1);
14049            assert_eq!(responses[0].header.ty, FrameType::Error);
14050            let error = parse_error(&responses[0]);
14051            assert_eq!(error["code"], "version_unsupported");
14052            assert!(registry.get_module("aft").unwrap().is_none());
14053            assert_eq!(registry.active_registration_count().unwrap(), 0);
14054        }
14055    }
14056
14057    #[test]
14058    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14059        let registry = Arc::new(Registry::default());
14060        let forwarding = Arc::new(ForwardingTable::default());
14061        let handler =
14062            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14063        let module_connection = ConnectionId::new(301);
14064        let registration = registry
14065            .register_with_control_ops(
14066                manifest("aft-push", PROTOCOL_VERSION),
14067                PROTOCOL_VERSION,
14068                module_connection,
14069                module_baseline_control_ops(),
14070            )
14071            .unwrap();
14072        let (module_tx, _module_rx) = mpsc::channel(8);
14073        let endpoint = forwarding
14074            .register_module_connection(
14075                module_connection,
14076                "aft-push".to_string(),
14077                PROTOCOL_VERSION,
14078                manifest_concurrency(&registration.manifest),
14079                FrameSink::new(module_tx),
14080            )
14081            .unwrap();
14082
14083        // A push op this version does not know is ignored (forward-compat), not errored.
14084        let unknown = Frame::build(
14085            FrameType::Push,
14086            control_flags(),
14087            0,
14088            0,
14089            5,
14090            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14091        )
14092        .unwrap();
14093        let out = handler.handle_status_update(endpoint, unknown).unwrap();
14094        assert!(
14095            out.is_empty(),
14096            "unknown push op must be ignored, got {out:?}"
14097        );
14098
14099        // A malformed body for a KNOWN op is a real error worth surfacing.
14100        let malformed = Frame::build(
14101            FrameType::Push,
14102            control_flags(),
14103            0,
14104            0,
14105            6,
14106            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14107        )
14108        .unwrap();
14109        let out = handler.handle_status_update(endpoint, malformed).unwrap();
14110        assert_eq!(out.len(), 1);
14111        assert_eq!(out[0].header.ty, FrameType::Error);
14112        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14113    }
14114
14115    #[test]
14116    fn hello_rejected_when_connection_already_owns_client_routes() {
14117        let registry = Arc::new(Registry::default());
14118        let forwarding = Arc::new(ForwardingTable::default());
14119        let handler =
14120            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14121        // Commits a client route on connection 202 (bound to a module on conn 101).
14122        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
14123        let client_connection = ConnectionId::new(202);
14124
14125        // That same connection now tries to register as a module: rejected, so one
14126        // connection never holds both client-route and module-endpoint state.
14127        let responses = handler
14128            .handle_control(
14129                client_connection,
14130                hello_frame("aft-second", PROTOCOL_VERSION, 9),
14131            )
14132            .unwrap();
14133        assert_eq!(responses[0].header.ty, FrameType::Error);
14134        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14135        assert!(registry.get_module("aft-second").unwrap().is_none());
14136    }
14137
14138    #[tokio::test]
14139    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14140        let registry = Arc::new(Registry::default());
14141        let forwarding = Arc::new(ForwardingTable::default());
14142        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14143        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14144        hello_via_sink(
14145            &handler,
14146            &module_ctx,
14147            &mut module_rx,
14148            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14149        )
14150        .await;
14151        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14152        let pending = forwarding
14153            .begin_route_bind_relay_for_test(
14154                client_ctx.connection_id,
14155                client_ctx.egress.clone(),
14156                2,
14157                "alpha",
14158            )
14159            .unwrap();
14160        forwarding
14161            .complete_pending_relay(
14162                module_ctx.connection_id,
14163                pending.corr,
14164                RouteBindRelayOutcome::Accepted,
14165            )
14166            .unwrap();
14167        client_rx.try_recv().unwrap();
14168        for module_id in ["beta", "alpha"] {
14169            let replies = handler
14170                .handle_control_frame(
14171                    &module_ctx,
14172                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14173                )
14174                .await
14175                .unwrap();
14176            assert_eq!(replies.len(), 1, "second HELLO must be refused");
14177            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14178        }
14179        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14180        assert!(registry.get_module("beta").unwrap().is_none());
14181        assert!(matches!(
14182            forwarding
14183                .lookup_data_route(
14184                    client_ctx.connection_id,
14185                    pending.client_channel,
14186                    pending.client_epoch,
14187                )
14188                .unwrap(),
14189            DataRoute::Client(DataRouteState::Bound(_))
14190        ));
14191        assert!(handler
14192            .hello_launch_nonces
14193            .lock()
14194            .unwrap()
14195            .presented(module_ctx.connection_id, Some("alpha-nonce")));
14196        assert!(module_rx.try_recv().is_err());
14197    }
14198
14199    #[test]
14200    fn reserved_module_hello_requires_matching_launch_nonce() {
14201        let registry = Arc::new(Registry::default());
14202        let supervisor = SupervisorHandle::new();
14203        // The supervisor recorded the nonce it injected when it spawned the reserved
14204        // module; the HELLO verifier checks against the same shared handle.
14205        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14206        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14207
14208        // A HELLO with NO nonce is rejected.
14209        let no_nonce = handler
14210            .handle_control(
14211                ConnectionId::new(1),
14212                hello_frame("vault", PROTOCOL_VERSION, 1),
14213            )
14214            .unwrap();
14215        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14216        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14217        assert!(registry.get_module("vault").unwrap().is_none());
14218
14219        // A HELLO with the WRONG nonce is rejected.
14220        let wrong = handler
14221            .handle_control(
14222                ConnectionId::new(2),
14223                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14224            )
14225            .unwrap();
14226        assert_eq!(wrong[0].header.ty, FrameType::Error);
14227        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14228        assert!(registry.get_module("vault").unwrap().is_none());
14229
14230        // A HELLO with the CORRECT nonce registers.
14231        let ok = handler
14232            .handle_control(
14233                ConnectionId::new(3),
14234                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14235            )
14236            .unwrap();
14237        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14238        assert!(registry.get_module("vault").unwrap().is_some());
14239    }
14240
14241    #[test]
14242    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14243        let registry = Arc::new(Registry::default());
14244        let supervisor = SupervisorHandle::new();
14245        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14246        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14247        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14248
14249        let squat = handler
14250            .handle_control(
14251                ConnectionId::new(1),
14252                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14253            )
14254            .unwrap();
14255        assert_eq!(squat[0].header.ty, FrameType::Error);
14256        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14257        assert!(parse_error(&squat[0])["message"]
14258            .as_str()
14259            .unwrap()
14260            .contains("fed:"));
14261
14262        let accepted_peer = handler
14263            .handle_control(
14264                ConnectionId::new(2),
14265                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14266            )
14267            .unwrap();
14268        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14269
14270        let accepted_short = handler
14271            .handle_control(
14272                ConnectionId::new(3),
14273                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14274            )
14275            .unwrap();
14276        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14277
14278        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14279            let response = handler
14280                .handle_control(
14281                    ConnectionId::new(conn),
14282                    hello_frame(module_id, PROTOCOL_VERSION, conn),
14283                )
14284                .unwrap();
14285            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14286        }
14287    }
14288
14289    #[test]
14290    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14291        let registry = Arc::new(Registry::default());
14292        let supervisor = SupervisorHandle::new();
14293        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14294        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14295        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14296        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14297
14298        let owner_nonce = handler
14299            .handle_control(
14300                ConnectionId::new(1),
14301                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14302            )
14303            .unwrap();
14304        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14305        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14306        assert!(registry.get_module("fed:special").unwrap().is_none());
14307
14308        let exact_nonce = handler
14309            .handle_control(
14310                ConnectionId::new(2),
14311                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14312            )
14313            .unwrap();
14314        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14315        assert!(registry.get_module("fed:special").unwrap().is_some());
14316    }
14317
14318    #[test]
14319    fn non_reserved_module_ignores_launch_nonce() {
14320        let registry = Arc::new(Registry::default());
14321        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
14322        // registration succeeds whether a spawned process echoes a nonce or not.
14323        let handler = ControlHandler::new(Arc::clone(&registry));
14324        let no_nonce = handler
14325            .handle_control(
14326                ConnectionId::new(1),
14327                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14328            )
14329            .unwrap();
14330        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14331        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14332
14333        let echoed_nonce = handler
14334            .handle_control(
14335                ConnectionId::new(2),
14336                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14337            )
14338            .unwrap();
14339        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14340        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14341    }
14342
14343    #[test]
14344    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14345        let handler = ControlHandler::default();
14346        let conn = ConnectionId::new(1);
14347        let malformed = Frame::build(
14348            FrameType::Hello,
14349            control_flags(),
14350            0,
14351            0,
14352            3,
14353            b"{not json".to_vec(),
14354        )
14355        .unwrap();
14356
14357        let error = handler.handle_control(conn, malformed).unwrap();
14358        assert_eq!(error[0].header.ty, FrameType::Error);
14359        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14360
14361        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14362        let pong = handler.handle_control(conn, ping).unwrap();
14363        assert_eq!(pong[0].header.ty, FrameType::Pong);
14364        assert_eq!(pong[0].header.corr, 4);
14365    }
14366
14367    #[test]
14368    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14369        let registry = Arc::new(Registry::default());
14370        let handler = ControlHandler::new(Arc::clone(&registry));
14371
14372        handler
14373            .handle_control(
14374                ConnectionId::new(1),
14375                hello_frame("aft", PROTOCOL_VERSION, 1),
14376            )
14377            .unwrap();
14378        let duplicate = handler
14379            .handle_control(
14380                ConnectionId::new(2),
14381                hello_frame("aft", PROTOCOL_VERSION, 2),
14382            )
14383            .unwrap();
14384
14385        assert_eq!(duplicate[0].header.ty, FrameType::Error);
14386        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14387        let registration = registry.get_module("aft").unwrap().unwrap();
14388        assert_eq!(registration.connection_id, ConnectionId::new(1));
14389    }
14390
14391    #[test]
14392    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14393        let registry = Arc::new(Registry::default());
14394        let forwarding = Arc::new(ForwardingTable::default());
14395        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14396        let handler =
14397            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14398                .with_process_liveness(process_liveness);
14399        let (ctx, route_channel, route_epoch) =
14400            bind_liveness_route(&registry, &forwarding, "aft-dead");
14401        let responses = handler
14402            .handle_route_poll(
14403                &ctx,
14404                route_poll_frame(41, PollKind::Liveness, route_channel),
14405                route_channel,
14406                route_epoch,
14407                PollKind::Liveness,
14408            )
14409            .unwrap();
14410
14411        assert_eq!(responses.len(), 1);
14412        assert_eq!(responses[0].header.ty, FrameType::Response);
14413        assert_route_poll_liveness(&responses[0], false);
14414    }
14415
14416    #[test]
14417    fn liveness_poll_without_process_source_uses_bound_route() {
14418        let registry = Arc::new(Registry::default());
14419        let forwarding = Arc::new(ForwardingTable::default());
14420        let handler =
14421            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14422        let (ctx, route_channel, route_epoch) =
14423            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
14424        let responses = handler
14425            .handle_route_poll(
14426                &ctx,
14427                route_poll_frame(42, PollKind::Liveness, route_channel),
14428                route_channel,
14429                route_epoch,
14430                PollKind::Liveness,
14431            )
14432            .unwrap();
14433
14434        assert_route_poll_liveness(&responses[0], true);
14435    }
14436
14437    #[test]
14438    fn liveness_poll_untracked_process_source_uses_bound_route() {
14439        let registry = Arc::new(Registry::default());
14440        let forwarding = Arc::new(ForwardingTable::default());
14441        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14442        let handler =
14443            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14444                .with_process_liveness(process_liveness);
14445        let (ctx, route_channel, route_epoch) =
14446            bind_liveness_route(&registry, &forwarding, "aft-untracked");
14447        let responses = handler
14448            .handle_route_poll(
14449                &ctx,
14450                route_poll_frame(43, PollKind::Liveness, route_channel),
14451                route_channel,
14452                route_epoch,
14453                PollKind::Liveness,
14454            )
14455            .unwrap();
14456
14457        assert_route_poll_liveness(&responses[0], true);
14458    }
14459
14460    #[tokio::test]
14461    async fn unknown_op_returns_unknown_control_op() {
14462        let handler = ControlHandler::default();
14463        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14464        let request = Frame::build(
14465            FrameType::Request,
14466            control_flags(),
14467            0,
14468            0,
14469            55,
14470            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14471        )
14472        .unwrap();
14473
14474        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14475
14476        assert_eq!(response.len(), 1);
14477        assert_eq!(response[0].header.ty, FrameType::Error);
14478        assert_eq!(response[0].header.corr, 55);
14479        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14480    }
14481
14482    #[tokio::test]
14483    async fn supervisor_provenance_rejects_unknown_exact_module() {
14484        let handler = ControlHandler::default();
14485        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14486        let request = Frame::build(
14487            FrameType::Request,
14488            control_flags(),
14489            0,
14490            0,
14491            57,
14492            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14493        )
14494        .unwrap();
14495
14496        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14497
14498        assert_eq!(response.len(), 1);
14499        assert_eq!(response[0].header.ty, FrameType::Error);
14500        assert_eq!(response[0].header.corr, 57);
14501        let error = parse_error(&response[0]);
14502        assert_eq!(error["code"], "unknown_module");
14503        assert_eq!(error["message"], "module_id 'missing' is not supervised");
14504    }
14505
14506    #[test]
14507    fn provenance_probe_override_keeps_handler_tests_deterministic() {
14508        let expected = subc_control::RunningImageAgreement::Unavailable {
14509            reason: subc_control::RunningImageUnavailableReason::HashFailed,
14510        };
14511        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14512        assert_eq!(handler.provenance_probe_override, Some(expected));
14513    }
14514
14515    #[test]
14516    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14517        let verdict = reload_verdict(
14518            std::path::Path::new("/bin/new"),
14519            Some(std::path::Path::new("/bin/old")),
14520            subc_control::RunningImageAgreement::Unavailable {
14521                reason: subc_control::RunningImageUnavailableReason::HashFailed,
14522            },
14523        );
14524        assert!(matches!(
14525            verdict.path,
14526            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14527                if configured == std::path::Path::new("/bin/new")
14528                    && spawned_from == std::path::Path::new("/bin/old")
14529        ));
14530    }
14531
14532    #[test]
14533    fn launch_spec_without_running_digest_is_unavailable_not_match() {
14534        use subc_control::{
14535            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14536        };
14537        let configured = crate::supervise::ModuleSpec {
14538            module_id: "legacy".into(),
14539            program: "/module".into(),
14540            args: Vec::new(),
14541            env: Vec::new(),
14542            reserved: false,
14543            reserved_prefixes: Vec::new(),
14544            protocol: subc_control::ModuleProtocol::Subc,
14545            overlap: Default::default(),
14546        };
14547        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14548        assert_eq!(
14549            launch_spec_agreement(&configured, false, Some(&digest)),
14550            Agreement::Unavailable {
14551                reason: Reason::NotRunning
14552            }
14553        );
14554        assert_eq!(
14555            launch_spec_agreement(&configured, true, None),
14556            Agreement::Unavailable {
14557                reason: Reason::SpawnedSpecUnavailable
14558            }
14559        );
14560        assert_eq!(
14561            launch_spec_agreement(&configured, true, Some(&digest)),
14562            Agreement::Match
14563        );
14564    }
14565
14566    #[cfg(windows)]
14567    #[test]
14568    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14569        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14570        let configured = dir.join("module.exe");
14571        std::fs::write(&configured, b"executable path fixture").unwrap();
14572        let spawned = std::fs::canonicalize(&configured).unwrap();
14573        assert_ne!(
14574            configured, spawned,
14575            "fixture must exercise different path spellings"
14576        );
14577        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14578        let lower_case = configured.to_string_lossy().to_lowercase();
14579        for spelling in [
14580            &configured,
14581            Path::new(&forward_slashes),
14582            Path::new(&lower_case),
14583        ] {
14584            let verdict = reload_verdict(
14585                spelling,
14586                Some(&spawned),
14587                subc_control::RunningImageAgreement::Unavailable {
14588                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14589                },
14590            );
14591            assert_eq!(
14592                verdict.path,
14593                ReloadPathAgreement::Match,
14594                "configured {spelling:?}, spawned {spawned:?}"
14595            );
14596        }
14597    }
14598
14599    #[cfg(windows)]
14600    #[test]
14601    fn windows_reload_path_keeps_different_existing_program_pending() {
14602        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14603        let old = dir.join("old.exe");
14604        let new = dir.join("new.exe");
14605        std::fs::write(&old, b"same executable content").unwrap();
14606        std::fs::copy(&old, &new).unwrap();
14607        let spawned = std::fs::canonicalize(&old).unwrap();
14608        let image = subc_control::RunningImageAgreement::Unavailable {
14609            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14610        };
14611        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14612        assert!(
14613            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14614            if configured == new && spawned_from == spawned)
14615        );
14616        let missing = dir.join("missing.exe");
14617        assert!(matches!(
14618            reload_verdict(&missing, Some(&spawned), image).path,
14619            ReloadPathAgreement::Mismatch { .. }
14620        ));
14621    }
14622
14623    #[test]
14624    fn reload_verdict_detects_replaced_image_at_same_path() {
14625        let image = subc_control::RunningImageAgreement::Mismatch {
14626            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14627                digest: "old".into(),
14628            },
14629            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14630                digest: "new".into(),
14631            },
14632        };
14633        let verdict = reload_verdict(
14634            std::path::Path::new("/bin/same"),
14635            Some(std::path::Path::new("/bin/same")),
14636            image.clone(),
14637        );
14638        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14639        assert_eq!(verdict.image, image);
14640    }
14641
14642    #[test]
14643    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14644        let image = subc_control::RunningImageAgreement::Unavailable {
14645            reason: subc_control::RunningImageUnavailableReason::NotRunning,
14646        };
14647        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14648        assert_eq!(
14649            verdict.path,
14650            subc_control::ReloadPathAgreement::Unavailable {
14651                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14652            }
14653        );
14654        assert_eq!(verdict.image, image);
14655
14656        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14657            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14658        };
14659        let verdict = reload_verdict(
14660            std::path::Path::new("/bin/same"),
14661            Some(std::path::Path::new("/bin/same")),
14662            unconfirmed.clone(),
14663        );
14664        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14665        assert_eq!(verdict.image, unconfirmed);
14666    }
14667
14668    #[test]
14669    fn reload_verdict_preserves_each_image_unavailability_reason() {
14670        use subc_control::RunningImageUnavailableReason as Reason;
14671
14672        for reason in [
14673            Reason::NotRunning,
14674            Reason::UnsupportedPlatform,
14675            Reason::RunningExecutableUnreadable,
14676            Reason::SpawnedPathUnreadable,
14677            Reason::HashFailed,
14678            Reason::ProcessIdentityUnconfirmed,
14679            Reason::Unknown("future_probe_reason".to_string()),
14680        ] {
14681            let image = subc_control::RunningImageAgreement::Unavailable {
14682                reason: reason.clone(),
14683            };
14684            let verdict = reload_verdict(
14685                std::path::Path::new("/bin/same"),
14686                Some(std::path::Path::new("/bin/same")),
14687                image.clone(),
14688            );
14689            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14690            assert_eq!(verdict.image, image, "{reason:?}");
14691        }
14692    }
14693
14694    #[tokio::test]
14695    async fn malformed_control_bodies_return_invalid_control_body() {
14696        let handler = ControlHandler::default();
14697        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14698
14699        for (corr, body) in [
14700            (56, br#"{"route_channel":1}"#.as_slice()),
14701            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14702            (
14703                58,
14704                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14705            ),
14706        ] {
14707            let request = Frame::build(
14708                FrameType::Request,
14709                control_flags(),
14710                0,
14711                0,
14712                corr,
14713                body.to_vec(),
14714            )
14715            .unwrap();
14716            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14717
14718            assert_eq!(response.len(), 1);
14719            assert_eq!(response[0].header.ty, FrameType::Error);
14720            assert_eq!(response[0].header.corr, corr);
14721            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14722        }
14723    }
14724
14725    #[tokio::test]
14726    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14727        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14728        let registry = Arc::new(Registry::default());
14729        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14730        let router = Router::with_control_handler(Arc::clone(&control));
14731        let connection = router.begin_connection();
14732        let (ctx, mut rx) = route_ctx(connection.id());
14733
14734        router
14735            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14736            .await
14737            .unwrap();
14738        let response = rx.recv().await.unwrap();
14739        let ack = parse_ack(&response);
14740        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14741        let channel = 1;
14742
14743        let goodbye =
14744            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14745        router.route_for_connection(&ctx, goodbye).await.unwrap();
14746        assert!(rx.try_recv().is_err());
14747        assert!(registry.get_module("aft").unwrap().is_none());
14748
14749        router
14750            .route_for_connection(&ctx, channel_request(channel, 13))
14751            .await
14752            .unwrap();
14753        let error_frame = rx.recv().await.unwrap();
14754        assert_eq!(error_frame.header.ty, FrameType::Error);
14755        assert_eq!(error_frame.header.channel, channel);
14756        let captured = crate::router::test_log::captured_logs(&logs);
14757        assert_eq!(
14758            captured
14759                .lines()
14760                .filter(|line| {
14761                    line.contains("module_id=aft")
14762                        && line.contains("reason=explicit_goodbye")
14763                        && line.contains("module registration ended")
14764                })
14765                .count(),
14766            1,
14767            "unexpected GOODBYE registry log: {captured}"
14768        );
14769    }
14770
14771    #[tokio::test]
14772    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14773        let registry = Arc::new(Registry::default());
14774        let handler = ControlHandler::new(registry).with_capability_config(
14775            [("prov".to_string(), true), ("cons".to_string(), true)],
14776            BTreeMap::new(),
14777        );
14778        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14779        register_capability_manifest(
14780            &handler,
14781            &provider_ctx,
14782            &mut provider_rx,
14783            capability_manifest("prov", &["thing/v1"], &[]),
14784            1,
14785        )
14786        .await;
14787        let mut consumer = capability_manifest("cons", &[], &[]);
14788        consumer.capabilities.as_mut().unwrap().requires.push(
14789            subc_protocol::manifest::CapabilityRequirement {
14790                capability: "thing/v1".to_string(),
14791                need: subc_protocol::manifest::CapabilityNeed::Required,
14792            },
14793        );
14794        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14795        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14796        assert_eq!(
14797            handler.capability_evaluator.verdict("cons", "thing/v1"),
14798            Some(CapabilityVerdict::Provided)
14799        );
14800        let (mut client_rx, _) = open_route_for_capability_test(
14801            &handler,
14802            &provider_ctx,
14803            &mut provider_rx,
14804            703,
14805            3,
14806            "prov",
14807            None,
14808        )
14809        .await;
14810        let goodbye =
14811            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14812        handler
14813            .handle_control_frame(&provider_ctx, goodbye)
14814            .await
14815            .unwrap();
14816        assert_eq!(
14817            handler.capability_evaluator.verdict("cons", "thing/v1"),
14818            Some(CapabilityVerdict::NeverProvided)
14819        );
14820        let closed = client_rx
14821            .try_recv()
14822            .expect("GOODBYE pushes route.closed before route GOODBYE");
14823        assert!(
14824            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14825            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14826        );
14827        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14828        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14829    }
14830
14831    struct RouteClosedFixture {
14832        handler: ControlHandler,
14833        #[cfg(unix)]
14834        supervisor: Supervisor,
14835        module_ctx: RouteCtx,
14836        #[cfg(unix)]
14837        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14838        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14839        route: Arc<crate::forwarding::RouteBinding>,
14840    }
14841
14842    async fn route_closed_fixture() -> RouteClosedFixture {
14843        let registry = Arc::new(Registry::default());
14844        let handler = ControlHandler::new(Arc::clone(&registry));
14845        #[cfg(unix)]
14846        let supervisor = Supervisor::new(registry, RestartPolicy::default())
14847            .with_forwarding(Arc::clone(&handler.forwarding));
14848        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14849        register_capability_manifest(
14850            &handler,
14851            &module_ctx,
14852            &mut module_rx,
14853            capability_manifest("prov", &[], &[]),
14854            1,
14855        )
14856        .await;
14857        let (client_rx, bind) = open_route_for_capability_test(
14858            &handler,
14859            &module_ctx,
14860            &mut module_rx,
14861            702,
14862            2,
14863            "prov",
14864            None,
14865        )
14866        .await;
14867        let ModuleControlRequest::RouteBind {
14868            route_channel: channel,
14869            epoch: route_epoch,
14870            ..
14871        } = bind
14872        else {
14873            panic!("expected route.bind");
14874        };
14875        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14876            handler
14877                .forwarding
14878                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14879                .unwrap()
14880        else {
14881            panic!("the acknowledged route must be bound");
14882        };
14883        RouteClosedFixture {
14884            handler,
14885            #[cfg(unix)]
14886            supervisor,
14887            module_ctx,
14888            #[cfg(unix)]
14889            module_rx,
14890            client_rx,
14891            route,
14892        }
14893    }
14894
14895    impl RouteClosedFixture {
14896        #[cfg(unix)]
14897        async fn cut(&mut self) {
14898            let notices = async {
14899                let mut draining = self.module_rx.recv().await.unwrap();
14900                assert!(matches!(
14901                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14902                        &draining.body
14903                    )
14904                    .unwrap(),
14905                    subc_protocol::session::ModuleControlCommand::Draining {
14906                        reason: RouteCloseReason::Restart,
14907                        ..
14908                    }
14909                ));
14910                draining.flushed.take().unwrap().send(()).unwrap();
14911                let mut closing = self.client_rx.recv().await.unwrap();
14912                assert!(matches!(
14913                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14914                    ClientControlPush::RouteClosing {
14915                        reason: RouteCloseReason::Restart,
14916                        ..
14917                    }
14918                ));
14919                closing.flushed.take().unwrap().send(()).unwrap();
14920            };
14921            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14922            result.unwrap();
14923        }
14924
14925        fn disconnect(&mut self) -> ClientControlPush {
14926            self.handler
14927                .cleanup_connection(self.module_ctx.connection_id)
14928                .unwrap();
14929            let closed = self.client_rx.try_recv().unwrap();
14930            assert_eq!(closed.header.ty, FrameType::Push);
14931            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14932            assert!(matches!(
14933                &push,
14934                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14935                    if channels == &[self.route.client_channel]
14936            ));
14937            assert_eq!(
14938                self.client_rx.try_recv().unwrap().header.ty,
14939                FrameType::Goodbye
14940            );
14941            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14942            push
14943        }
14944    }
14945
14946    #[cfg(unix)]
14947    #[tokio::test(start_paused = true)]
14948    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14949        let mut fixture = route_closed_fixture().await;
14950        fixture.cut().await;
14951        assert!(matches!(
14952            fixture.disconnect(),
14953            ClientControlPush::RouteClosed {
14954                reason: RouteCloseReason::Restart,
14955                drained: true,
14956                ..
14957            }
14958        ));
14959    }
14960
14961    #[cfg(unix)]
14962    #[tokio::test(start_paused = true)]
14963    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14964        let mut fixture = route_closed_fixture().await;
14965        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14966        let started = tokio::time::Instant::now();
14967        fixture.cut().await;
14968        assert!(started.elapsed() >= Duration::from_secs(2));
14969        assert_eq!(fixture.route.flow.in_flight(), 1);
14970        assert!(matches!(
14971            fixture.disconnect(),
14972            ClientControlPush::RouteClosed {
14973                reason: RouteCloseReason::Restart,
14974                drained: false,
14975                ..
14976            }
14977        ));
14978    }
14979
14980    #[tokio::test]
14981    async fn module_connection_crash_reports_crash_and_not_drained() {
14982        let mut fixture = route_closed_fixture().await;
14983        // The server calls cleanup_connection on module EOF. Without a planned
14984        // daemon shutdown, that disconnect must remain a crash with no prior
14985        // route.closing notice, even when the route has no in-flight calls.
14986        assert!(matches!(
14987            fixture.disconnect(),
14988            ClientControlPush::RouteClosed {
14989                reason: RouteCloseReason::Crash,
14990                drained: false,
14991                ..
14992            }
14993        ));
14994    }
14995
14996    #[tokio::test]
14997    async fn dropping_router_connection_releases_registration() {
14998        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14999        let registry = Arc::new(Registry::default());
15000        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
15001        let router = Router::with_control_handler(Arc::clone(&control));
15002        let connection = router.begin_connection();
15003        let connection_id = connection.id();
15004        let (ctx, mut rx) = route_ctx(connection_id);
15005
15006        router
15007            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15008            .await
15009            .unwrap();
15010        let response = rx.recv().await.unwrap();
15011        let ack = parse_ack(&response);
15012        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15013        assert!(registry.get_module("aft").unwrap().is_some());
15014
15015        drop(connection);
15016
15017        assert!(registry.get_module("aft").unwrap().is_none());
15018        assert_eq!(registry.active_registration_count().unwrap(), 0);
15019
15020        control
15021            .cleanup_connection(ConnectionId::new(u64::MAX))
15022            .unwrap();
15023        let captured = crate::router::test_log::captured_logs(&logs);
15024        println!("captured registry lifecycle logs:\n{captured}");
15025        let events: Vec<_> = captured
15026            .lines()
15027            .filter(|line| {
15028                line.contains("module registered module_id=aft ")
15029                    || line.contains("module registration ended")
15030            })
15031            .collect();
15032        assert_eq!(
15033            events.len(),
15034            2,
15035            "unexpected registry lifecycle logs: {captured}"
15036        );
15037        assert!(events[0].contains("module registered module_id=aft "));
15038        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15039        assert!(events[1].contains(&format!(
15040            "module_id=aft connection_id={}",
15041            connection_id.get()
15042        )));
15043        assert!(events[1].contains("reason=connection_closed"));
15044        assert!(events[1].contains("module registration ended"));
15045        assert_eq!(
15046            captured
15047                .lines()
15048                .filter(|line| line.contains("module registered module_id=aft "))
15049                .count(),
15050            1,
15051            "legacy registration admission line must appear once: {captured}"
15052        );
15053    }
15054
15055    #[tokio::test]
15056    async fn hello_registration_keeps_legacy_module_registered_line_once() {
15057        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15058        let registry = Arc::new(Registry::default());
15059        let control = ControlHandler::new(Arc::clone(&registry));
15060        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15061        hello_via_sink(
15062            &control,
15063            &ctx,
15064            &mut rx,
15065            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15066        )
15067        .await;
15068
15069        let captured = crate::router::test_log::captured_logs(&logs);
15070        let admissions: Vec<_> = captured
15071            .lines()
15072            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15073            .collect();
15074        assert_eq!(
15075            admissions.len(),
15076            1,
15077            "legacy admission line must remain exactly once: {captured}"
15078        );
15079        assert!(admissions[0].contains("routable_provider=true"));
15080        assert!(admissions[0].contains("connection_id=777"));
15081    }
15082
15083    fn capability_manifest(
15084        module_id: &str,
15085        provides: &[&str],
15086        must_never_reach: &[&str],
15087    ) -> ModuleManifest {
15088        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15089        manifest.capabilities = Some(CapabilityDeclarations {
15090            provides: provides
15091                .iter()
15092                .map(|capability| (*capability).to_string())
15093                .collect(),
15094            requires: Vec::new(),
15095            must_never_reach: must_never_reach
15096                .iter()
15097                .map(|capability| (*capability).to_string())
15098                .collect(),
15099        });
15100        manifest
15101    }
15102
15103    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15104        Frame::build(
15105            FrameType::Hello,
15106            control_flags(),
15107            0,
15108            0,
15109            corr,
15110            serde_json::to_vec(&ModuleHelloBody {
15111                protocol_ver: manifest.protocol_ver,
15112                manifest,
15113                control_ops: None,
15114                launch_nonce: None,
15115            })
15116            .expect("capability test HELLO serializes"),
15117        )
15118        .expect("capability test HELLO frame builds")
15119    }
15120
15121    fn catalog_update_with_capabilities_frame(
15122        corr: u64,
15123        capabilities: CapabilityDeclarations,
15124    ) -> Frame {
15125        Frame::build(
15126            FrameType::Request,
15127            control_flags(),
15128            0,
15129            0,
15130            corr,
15131            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15132                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15133                capabilities: Some(capabilities),
15134                ready: None,
15135            })
15136            .expect("capability catalog.update serializes"),
15137        )
15138        .expect("capability catalog.update frame builds")
15139    }
15140
15141    async fn register_capability_manifest(
15142        handler: &ControlHandler,
15143        ctx: &RouteCtx,
15144        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15145        manifest: ModuleManifest,
15146        corr: u64,
15147    ) {
15148        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15149    }
15150
15151    async fn open_route_for_capability_test(
15152        handler: &ControlHandler,
15153        target_ctx: &RouteCtx,
15154        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15155        client_connection_id: u64,
15156        corr: u64,
15157        target_module_id: &str,
15158        consumer_identity: Option<ConsumerIdentity>,
15159    ) -> (
15160        mpsc::Receiver<crate::router::OutboundFrame>,
15161        ModuleControlRequest,
15162    ) {
15163        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15164        let route_handler = handler.clone();
15165        let target_module_id = target_module_id.to_string();
15166        let route_task = tokio::spawn(async move {
15167            route_handler
15168                .handle_control_frame(
15169                    &client_ctx,
15170                    route_open_frame_with_admission_facts(
15171                        corr,
15172                        &target_module_id,
15173                        unique_project_root("admission-facts"),
15174                        consumer_identity,
15175                        None,
15176                    ),
15177                )
15178                .await
15179                .expect("capability test route.open succeeds")
15180        });
15181        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15182            .await
15183            .expect("capability test route.open must reach route.bind")
15184            .expect("target control receiver stays open");
15185        let bind_request: ModuleControlRequest =
15186            serde_json::from_slice(&bind.body).expect("route.bind decodes");
15187        handler
15188            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15189            .await
15190            .expect("capability test route.bind ACK succeeds");
15191        assert!(route_task.await.expect("route.open task joins").is_empty());
15192        let opened = client_rx
15193            .recv()
15194            .await
15195            .expect("successful route.open publishes a response");
15196        assert!(matches!(
15197            serde_json::from_slice::<ClientControlResponse>(&opened.body),
15198            Ok(ClientControlResponse::RouteOpen { .. })
15199        ));
15200        (client_rx, bind_request)
15201    }
15202
15203    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15204        assert_eq!(frame.header.ty, FrameType::Push);
15205        assert_eq!(frame.header.channel, 0);
15206        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15207            .expect("route.closed control push decodes");
15208        let ClientControlPush::RouteClosed { channels, .. } = &push else {
15209            panic!("expected route.closed");
15210        };
15211        assert_eq!(channels.len(), 1, "exactly one violating route closed");
15212        let closed_channel = channels[0];
15213        let channels = channels.clone();
15214        assert_eq!(
15215            push,
15216            ClientControlPush::RouteClosed {
15217                module_id: target_module_id.to_string(),
15218                channels,
15219                reason: RouteCloseReason::CapabilityDenied,
15220                drained: false,
15221                abandoned: 0,
15222                excluded_subscriptions: 0,
15223                terminal: Some(false),
15224            }
15225        );
15226        closed_channel
15227    }
15228
15229    #[tokio::test]
15230    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15231        let registry = Arc::new(Registry::default());
15232        let forwarding = Arc::new(ForwardingTable::default());
15233        let supervisor = SupervisorHandle::new();
15234        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15235        let handler =
15236            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15237                .with_supervisor(supervisor);
15238        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15239        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15240        register_capability_manifest(
15241            &handler,
15242            &target_ctx,
15243            &mut target_rx,
15244            capability_manifest("target", &["credentials-provider/v1"], &[]),
15245            1,
15246        )
15247        .await;
15248        register_capability_manifest(
15249            &handler,
15250            &opener_ctx,
15251            &mut opener_rx,
15252            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15253            2,
15254        )
15255        .await;
15256
15257        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15258        let replies = handler
15259            .handle_control_frame(
15260                &client_ctx,
15261                route_open_frame_with_admission_facts(
15262                    3,
15263                    "target",
15264                    unique_project_root("admission-facts"),
15265                    Some(ConsumerIdentity {
15266                        module_id: "opener".to_string(),
15267                        launch_nonce: "opener-nonce".to_string(),
15268                    }),
15269                    None,
15270                ),
15271            )
15272            .await
15273            .expect("denied route.open returns a typed frame");
15274        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15275        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15276        assert!(
15277            target_rx.try_recv().is_err(),
15278            "forbidden route.open must not relay route.bind"
15279        );
15280    }
15281
15282    #[tokio::test]
15283    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15284        let registry = Arc::new(Registry::default());
15285        let forwarding = Arc::new(ForwardingTable::default());
15286        let supervisor = SupervisorHandle::new();
15287        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15288        let handler =
15289            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15290                .with_supervisor(supervisor);
15291        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15292        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15293        register_capability_manifest(
15294            &handler,
15295            &target_ctx,
15296            &mut target_rx,
15297            capability_manifest("target", &["credentials-provider/v1"], &[]),
15298            1,
15299        )
15300        .await;
15301        register_capability_manifest(
15302            &handler,
15303            &old_opener_ctx,
15304            &mut old_opener_rx,
15305            capability_manifest("opener", &[], &[]),
15306            2,
15307        )
15308        .await;
15309        let (mut client_rx, _) = open_route_for_capability_test(
15310            &handler,
15311            &target_ctx,
15312            &mut target_rx,
15313            712,
15314            3,
15315            "target",
15316            Some(ConsumerIdentity {
15317                module_id: "opener".to_string(),
15318                launch_nonce: "opener-nonce".to_string(),
15319            }),
15320        )
15321        .await;
15322        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15323
15324        handler
15325            .cleanup_connection(old_opener_ctx.connection_id)
15326            .expect("old opener registration cleans up");
15327        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15328        register_capability_manifest(
15329            &handler,
15330            &new_opener_ctx,
15331            &mut new_opener_rx,
15332            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15333            4,
15334        )
15335        .await;
15336
15337        let closed_channel = assert_capability_denied_push(
15338            client_rx
15339                .try_recv()
15340                .expect("HELLO deny addition must emit route.closed")
15341                .frame,
15342            "target",
15343        );
15344        let goodbye = client_rx
15345            .try_recv()
15346            .expect("HELLO deny addition must send the client a route GOODBYE")
15347            .frame;
15348        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15349        assert_eq!(goodbye.header.channel, closed_channel);
15350        assert_ne!(goodbye.header.channel, 0);
15351        assert_ne!(goodbye.header.epoch, 0);
15352        assert!(client_rx.try_recv().is_err());
15353        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15354        assert!(matches!(
15355            target_rx.try_recv(),
15356            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15357        ));
15358    }
15359
15360    #[tokio::test]
15361    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15362        let registry = Arc::new(Registry::default());
15363        let forwarding = Arc::new(ForwardingTable::default());
15364        let supervisor = SupervisorHandle::new();
15365        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15366        let handler =
15367            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15368                .with_supervisor(supervisor);
15369        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15370        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15371        register_capability_manifest(
15372            &handler,
15373            &target_ctx,
15374            &mut target_rx,
15375            capability_manifest("target", &[], &[]),
15376            1,
15377        )
15378        .await;
15379        register_capability_manifest(
15380            &handler,
15381            &opener_ctx,
15382            &mut opener_rx,
15383            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15384            2,
15385        )
15386        .await;
15387        let (mut client_rx, _) = open_route_for_capability_test(
15388            &handler,
15389            &target_ctx,
15390            &mut target_rx,
15391            722,
15392            3,
15393            "target",
15394            Some(ConsumerIdentity {
15395                module_id: "opener".to_string(),
15396                launch_nonce: "opener-nonce".to_string(),
15397            }),
15398        )
15399        .await;
15400        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15401
15402        let replies = handler
15403            .handle_control_frame(
15404                &target_ctx,
15405                catalog_update_with_capabilities_frame(
15406                    4,
15407                    CapabilityDeclarations {
15408                        provides: vec!["credentials-provider/v1".to_string()],
15409                        requires: Vec::new(),
15410                        must_never_reach: Vec::new(),
15411                    },
15412                ),
15413            )
15414            .await
15415            .expect("claim catalog.update succeeds");
15416        assert!(matches!(
15417            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15418            Ok(ModuleControlResponseToModule::CatalogUpdate {})
15419        ));
15420        let closed_channel = assert_capability_denied_push(
15421            client_rx
15422                .try_recv()
15423                .expect("claim addition must emit route.closed")
15424                .frame,
15425            "target",
15426        );
15427        let goodbye = client_rx
15428            .try_recv()
15429            .expect("claim addition must send the client a route GOODBYE")
15430            .frame;
15431        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15432        assert_eq!(goodbye.header.channel, closed_channel);
15433        assert_ne!(goodbye.header.channel, 0);
15434        assert_ne!(goodbye.header.epoch, 0);
15435        assert!(client_rx.try_recv().is_err());
15436        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15437        assert!(matches!(
15438            target_rx.try_recv(),
15439            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15440        ));
15441    }
15442
15443    #[tokio::test]
15444    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15445        let registry = Arc::new(Registry::default());
15446        let forwarding = Arc::new(ForwardingTable::default());
15447        let supervisor = SupervisorHandle::new();
15448        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15449        let handler =
15450            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15451                .with_supervisor(supervisor);
15452        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15453        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15454        register_capability_manifest(
15455            &handler,
15456            &target_ctx,
15457            &mut target_rx,
15458            capability_manifest("target", &["credentials-provider/v1"], &[]),
15459            1,
15460        )
15461        .await;
15462        register_capability_manifest(
15463            &handler,
15464            &opener_ctx,
15465            &mut opener_rx,
15466            capability_manifest("opener", &[], &[]),
15467            2,
15468        )
15469        .await;
15470        let (mut client_rx, _) = open_route_for_capability_test(
15471            &handler,
15472            &target_ctx,
15473            &mut target_rx,
15474            732,
15475            3,
15476            "target",
15477            Some(ConsumerIdentity {
15478                module_id: "opener".to_string(),
15479                launch_nonce: "opener-nonce".to_string(),
15480            }),
15481        )
15482        .await;
15483        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15484
15485        handler
15486            .handle_control_frame(
15487                &target_ctx,
15488                catalog_update_with_capabilities_frame(
15489                    4,
15490                    CapabilityDeclarations {
15491                        provides: Vec::new(),
15492                        requires: Vec::new(),
15493                        must_never_reach: Vec::new(),
15494                    },
15495                ),
15496            )
15497            .await
15498            .expect("claim removal catalog.update succeeds");
15499        assert_eq!(
15500            forwarding.active_binding_count().unwrap(),
15501            1,
15502            "removing an attested target claim must leave the route census unchanged"
15503        );
15504        assert!(
15505            client_rx.try_recv().is_err(),
15506            "claim removal must not emit route.closed or a client route GOODBYE"
15507        );
15508        assert!(
15509            target_rx.try_recv().is_err(),
15510            "claim removal must not send the target a route GOODBYE"
15511        );
15512    }
15513
15514    /// A direct client may open a route to a denied capability provider; this
15515    /// policy applies only to attested supervised module origins, not to direct clients.
15516    #[tokio::test]
15517    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15518        let registry = Arc::new(Registry::default());
15519        let forwarding = Arc::new(ForwardingTable::default());
15520        let supervisor = SupervisorHandle::new();
15521        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15522        let handler =
15523            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15524                .with_supervisor(supervisor);
15525        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15526        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15527        register_capability_manifest(
15528            &handler,
15529            &target_ctx,
15530            &mut target_rx,
15531            capability_manifest("target", &["credentials-provider/v1"], &[]),
15532            1,
15533        )
15534        .await;
15535        register_capability_manifest(
15536            &handler,
15537            &opener_ctx,
15538            &mut opener_rx,
15539            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15540            2,
15541        )
15542        .await;
15543
15544        let (_client_rx, bind) = open_route_for_capability_test(
15545            &handler,
15546            &target_ctx,
15547            &mut target_rx,
15548            742,
15549            3,
15550            "target",
15551            None,
15552        )
15553        .await;
15554        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15555            panic!("direct scope-honesty route must bind");
15556        };
15557        assert_eq!(principal, Some(Principal::Direct));
15558        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15559    }
15560
15561    /// A module that denies a capability receives no self-route exemption when it
15562    /// also attestedly provides that capability.
15563    #[tokio::test]
15564    async fn must_never_reach_self_route_is_capability_forbidden() {
15565        let registry = Arc::new(Registry::default());
15566        let forwarding = Arc::new(ForwardingTable::default());
15567        let supervisor = SupervisorHandle::new();
15568        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15569        let handler =
15570            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15571                .with_supervisor(supervisor);
15572        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15573        register_capability_manifest(
15574            &handler,
15575            &self_ctx,
15576            &mut self_rx,
15577            capability_manifest(
15578                "self-provider",
15579                &["credentials-provider/v1"],
15580                &["credentials-provider/v1"],
15581            ),
15582            1,
15583        )
15584        .await;
15585
15586        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15587        let replies = handler
15588            .handle_control_frame(
15589                &client_ctx,
15590                route_open_frame_with_admission_facts(
15591                    2,
15592                    "self-provider",
15593                    unique_project_root("admission-facts"),
15594                    Some(ConsumerIdentity {
15595                        module_id: "self-provider".to_string(),
15596                        launch_nonce: "self-nonce".to_string(),
15597                    }),
15598                    None,
15599                ),
15600            )
15601            .await
15602            .expect("self-route refusal returns a typed frame");
15603        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15604        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15605        assert!(
15606            self_rx.try_recv().is_err(),
15607            "self denial must not relay route.bind"
15608        );
15609    }
15610
15611    #[test]
15612    fn unsupported_channel_zero_frame_returns_error() {
15613        let handler = ControlHandler::default();
15614        let request = Frame::build(
15615            FrameType::Request,
15616            control_flags(),
15617            0,
15618            0,
15619            21,
15620            b"opaque".to_vec(),
15621        )
15622        .unwrap();
15623
15624        let response = handler
15625            .handle_control(ConnectionId::new(1), request)
15626            .unwrap();
15627
15628        assert_eq!(response[0].header.ty, FrameType::Error);
15629        assert_eq!(
15630            parse_error(&response[0])["code"],
15631            "unsupported_control_frame"
15632        );
15633    }
15634
15635    /// Blue/green swap at the control-plane boundary. The supervisor that opens
15636    /// a swap is not wired yet, so the candidate is registered here directly
15637    /// into the registry and forwarding candidate slots, the way the swap's
15638    /// HELLO admission will.
15639    mod swap {
15640        use super::*;
15641
15642        const INCUMBENT: ConnectionId = ConnectionId::new(30);
15643        const CANDIDATE: ConnectionId = ConnectionId::new(40);
15644
15645        struct Swap {
15646            registry: Arc<Registry>,
15647            forwarding: Arc<ForwardingTable>,
15648            handler: ControlHandler,
15649            incumbent_ctx: RouteCtx,
15650            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15651            candidate_ctx: RouteCtx,
15652            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15653        }
15654
15655        async fn swap_with_incumbent() -> Swap {
15656            let registry = Arc::new(Registry::default());
15657            let forwarding = Arc::new(ForwardingTable::default());
15658            let handler =
15659                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15660            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15661            hello_via_sink(
15662                &handler,
15663                &incumbent_ctx,
15664                &mut incumbent_rx,
15665                hello_frame("aft", PROTOCOL_VERSION, 7),
15666            )
15667            .await;
15668            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15669            Swap {
15670                registry,
15671                forwarding,
15672                handler,
15673                incumbent_ctx,
15674                incumbent_rx,
15675                candidate_ctx,
15676                candidate_rx,
15677            }
15678        }
15679
15680        fn register_candidate(swap: &Swap, ready: Option<bool>) {
15681            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15682            candidate_manifest.ready = ready;
15683            let registration = swap
15684                .registry
15685                .register_candidate_with_control_ops(
15686                    candidate_manifest,
15687                    PROTOCOL_VERSION,
15688                    CANDIDATE,
15689                    module_baseline_control_ops(),
15690                )
15691                .unwrap();
15692            swap.forwarding
15693                .register_candidate_module_connection(
15694                    CANDIDATE,
15695                    "aft".to_string(),
15696                    PROTOCOL_VERSION,
15697                    manifest_concurrency(&registration.manifest),
15698                    swap.candidate_ctx.egress.clone(),
15699                )
15700                .unwrap();
15701        }
15702
15703        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15704            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15705            swap.registry.promote_candidate("aft").unwrap().unwrap();
15706            cutover.incumbent.unwrap()
15707        }
15708
15709        /// A swap candidate's HELLO reserves its candidate registry entry
15710        /// before installing its candidate forwarding endpoint. The swap
15711        /// supervisor promotes a ready candidate as soon as it reads it as
15712        /// registered, and promotion cuts forwarding over to the candidate
15713        /// endpoint, so the lookup the supervisor reads
15714        /// (`Registry::published_registration`) must not return the candidate
15715        /// until that endpoint exists. Otherwise the cutover finds no candidate
15716        /// endpoint and the swap fails as if the candidate had disconnected.
15717        #[tokio::test]
15718        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15719            let mut swap = swap_with_incumbent().await;
15720            let hold = Arc::new(std::sync::Barrier::new(2));
15721            let handler = swap
15722                .handler
15723                .clone()
15724                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15725            handler
15726                .supervisor
15727                .open_swap("aft", "candidate-nonce".to_string());
15728            let hello = {
15729                let handler = handler.clone();
15730                let candidate_ctx = swap.candidate_ctx.clone();
15731                std::thread::spawn(move || {
15732                    handler
15733                        .handle_hello(
15734                            candidate_ctx.connection_id,
15735                            Some(candidate_ctx.egress.clone()),
15736                            hello_frame_with_nonce(
15737                                "aft",
15738                                PROTOCOL_VERSION,
15739                                8,
15740                                Some("candidate-nonce"),
15741                            ),
15742                        )
15743                        .unwrap()
15744                })
15745            };
15746
15747            hold.wait();
15748            // Parked: the candidate slot is claimed but not published, and
15749            // forwarding has no candidate endpoint.
15750            assert!(swap
15751                .registry
15752                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15753                .unwrap()
15754                .is_some());
15755            assert!(swap
15756                .registry
15757                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15758                .unwrap()
15759                .is_none());
15760            assert!(swap
15761                .forwarding
15762                .module_endpoint_for_connection(CANDIDATE)
15763                .unwrap()
15764                .is_none());
15765
15766            hold.wait();
15767            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15768            assert_eq!(
15769                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15770                FrameType::HelloAck
15771            );
15772            let published = swap
15773                .registry
15774                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15775                .unwrap()
15776                .expect("candidate is published once its HELLO completes");
15777            assert_eq!(published.connection_id, CANDIDATE);
15778            assert!(swap
15779                .forwarding
15780                .module_endpoint_for_connection(CANDIDATE)
15781                .unwrap()
15782                .is_some());
15783        }
15784
15785        fn keyed_total(counters: &Value, key: &str) -> u64 {
15786            counters[key]
15787                .as_object()
15788                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15789                .unwrap_or(0)
15790        }
15791
15792        #[tokio::test]
15793        async fn replacement_logs_old_end_and_new_admission_once() {
15794            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15795            let mut swap = swap_with_incumbent().await;
15796            swap.handler
15797                .supervisor
15798                .open_swap("aft", "candidate-nonce".to_string());
15799            hello_via_sink(
15800                &swap.handler,
15801                &swap.candidate_ctx,
15802                &mut swap.candidate_rx,
15803                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15804            )
15805            .await;
15806            cutover(&swap);
15807            swap.handler.cleanup_connection(INCUMBENT).unwrap();
15808
15809            let captured = crate::router::test_log::captured_logs(&logs);
15810            println!("captured replacement registry logs:\n{captured}");
15811            let new_admissions: Vec<_> = captured
15812                .lines()
15813                .filter(|line| {
15814                    line.contains("connection_id=40")
15815                        && line.contains("swap candidate registered; not routable until cutover")
15816                })
15817                .collect();
15818            assert_eq!(
15819                new_admissions.len(),
15820                1,
15821                "unexpected admission logs: {captured}"
15822            );
15823            assert!(new_admissions[0].contains("module_id=aft"));
15824            assert!(new_admissions[0].contains("connection_id=40"));
15825            assert!(
15826                new_admissions[0].contains("swap candidate registered; not routable until cutover")
15827            );
15828            assert!(new_admissions[0].contains("ready=true"));
15829
15830            let old_admissions: Vec<_> = captured
15831                .lines()
15832                .filter(|line| {
15833                    line.contains("module registered module_id=aft ")
15834                        && line.contains("connection_id=30")
15835                })
15836                .collect();
15837            assert_eq!(
15838                old_admissions.len(),
15839                1,
15840                "unexpected incumbent admission: {captured}"
15841            );
15842
15843            let old_ends: Vec<_> = captured
15844                .lines()
15845                .filter(|line| {
15846                    line.contains("connection_id=30") && line.contains("module registration ended")
15847                })
15848                .collect();
15849            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15850            assert!(old_ends[0].contains("module_id=aft"));
15851            assert!(old_ends[0].contains("connection_id=30"));
15852            assert!(old_ends[0].contains("reason=replaced"));
15853            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15854            assert_eq!(
15855                captured
15856                    .lines()
15857                    .filter(|line| line.contains("module registration promoted"))
15858                    .count(),
15859                1,
15860                "unexpected promotion logs: {captured}"
15861            );
15862        }
15863
15864        /// An ack from the incumbent for a bind it was sent before cutover,
15865        /// arriving before the incumbent is drained. The incumbent is the live
15866        /// connection carrying every other client's routes, so the ack must
15867        /// not end it: the waiting client is told to retry, the reservation is
15868        /// given back, and the incumbent is told to drop just that binding.
15869        #[tokio::test]
15870        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15871            let mut swap = swap_with_incumbent().await;
15872            let handler = swap.handler.clone();
15873
15874            // A co-tenant route, bound on the incumbent before the swap.
15875            let cotenant = ConnectionId::new(31);
15876            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15877            let (cotenant_task, cotenant_bind) = relay_route_open(
15878                &handler,
15879                cotenant,
15880                &cotenant_ctx.egress,
15881                &mut swap.incumbent_rx,
15882                100,
15883                "aft",
15884                "swap-cotenant",
15885            )
15886            .await;
15887            handler
15888                .handle_control_frame(
15889                    &swap.incumbent_ctx,
15890                    route_bind_ack(cotenant_bind.header.corr),
15891                )
15892                .await
15893                .unwrap();
15894            assert!(cotenant_task.await.unwrap().is_empty());
15895            let (cotenant_channel, cotenant_epoch) =
15896                published_route(&cotenant_rx.recv().await.unwrap());
15897
15898            // A second route.open, relayed to the incumbent and not yet acked.
15899            let caller = ConnectionId::new(32);
15900            let (caller_ctx, mut caller_rx) = route_ctx(caller);
15901            let (caller_task, caller_bind) = relay_route_open(
15902                &handler,
15903                caller,
15904                &caller_ctx.egress,
15905                &mut swap.incumbent_rx,
15906                101,
15907                "aft",
15908                "swap-caller",
15909            )
15910            .await;
15911            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15912
15913            register_candidate(&swap, None);
15914            cutover(&swap);
15915
15916            // The incumbent acks after promotion and before any drain.
15917            let ack = handler
15918                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15919                .await;
15920            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15921            if module_loop_error.is_some() {
15922                // What the connection loop does with an untranslated router
15923                // error: end the connection, releasing every route on it.
15924                handler.cleanup_connection(INCUMBENT).unwrap();
15925            }
15926
15927            // 1. The incumbent's other routes survive.
15928            assert!(
15929                cotenant_rx.try_recv().is_err(),
15930                "the co-tenant route on the incumbent was torn down by one late ack: \
15931                 {module_loop_error:?}"
15932            );
15933            assert!(matches!(
15934                swap.forwarding
15935                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15936                    .unwrap(),
15937                DataRoute::Client(DataRouteState::Bound(_))
15938            ));
15939            assert_eq!(module_loop_error, None);
15940            assert!(swap
15941                .registry
15942                .get_module_by_connection(INCUMBENT)
15943                .unwrap()
15944                .is_some());
15945
15946            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
15947            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15948                .await
15949                .expect("the incumbent is told to drop the abandoned binding")
15950                .unwrap()
15951                .frame;
15952            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15953            assert_eq!(goodbye.header.channel, abandoned_channel);
15954            assert_eq!(goodbye.header.epoch, abandoned_epoch);
15955            assert!(swap.incumbent_rx.try_recv().is_err());
15956
15957            // 3. The waiting client gets a retryable refusal and no route.
15958            let response = caller_task.await.unwrap();
15959            assert_eq!(response.len(), 1);
15960            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15961            assert!(caller_rx.try_recv().is_err());
15962
15963            // 4. The reservation pair is given back, and the pending bind
15964            //    settled exactly once: one accepted open (the co-tenant) and one
15965            //    refused open (the caller), nothing counted twice.
15966            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15967            let counters = handler.counters().snapshot();
15968            assert_eq!(
15969                keyed_total(&counters, "route_open_accepted_by_principal"),
15970                1
15971            );
15972            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15973            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15974        }
15975
15976        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
15977        /// id would resolve to the promoted candidate and every new route.open
15978        /// would be refused as reloading, leaving neither process routable.
15979        #[tokio::test]
15980        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15981            let mut swap = swap_with_incumbent().await;
15982            register_candidate(&swap, None);
15983            let incumbent = cutover(&swap);
15984            swap.forwarding
15985                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15986                .unwrap()
15987                .expect("the incumbent is still registered");
15988
15989            let client = ConnectionId::new(33);
15990            let (client_ctx, mut client_rx) = route_ctx(client);
15991            let route_handler = swap.handler.clone();
15992            let open_ctx = RouteCtx {
15993                connection_id: client,
15994                egress: client_ctx.egress.clone(),
15995            };
15996            let mut route_task = tokio::spawn(async move {
15997                route_handler
15998                    .handle_control_frame(
15999                        &open_ctx,
16000                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16001                    )
16002                    .await
16003                    .unwrap()
16004            });
16005            let bind = tokio::select! {
16006                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16007                response = &mut route_task => {
16008                    let response = response.unwrap();
16009                    panic!(
16010                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
16011                        parse_error(&response[0])["code"]
16012                    );
16013                }
16014            };
16015            swap.handler
16016                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16017                .await
16018                .unwrap();
16019            assert!(route_task.await.unwrap().is_empty());
16020            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16021            match swap
16022                .forwarding
16023                .lookup_data_route(client, channel, epoch)
16024                .unwrap()
16025            {
16026                DataRoute::Client(DataRouteState::Bound(route)) => {
16027                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16028                }
16029                other => panic!("expected a bound route on the candidate, got {other:?}"),
16030            }
16031            assert!(swap.incumbent_rx.try_recv().is_err());
16032        }
16033
16034        /// A candidate declares itself ready with `catalog.update` on its own
16035        /// connection. If the connection-keyed registry lookups searched only the
16036        /// active slot, this would answer `not_registered` and the candidate
16037        /// would never become ready.
16038        #[tokio::test]
16039        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16040            let swap = swap_with_incumbent().await;
16041            register_candidate(&swap, Some(false));
16042            let update = Frame::build(
16043                FrameType::Request,
16044                control_flags(),
16045                0,
16046                0,
16047                55,
16048                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16049                    provides: manifest("aft", PROTOCOL_VERSION).provides,
16050                    capabilities: None,
16051                    ready: Some(true),
16052                })
16053                .unwrap(),
16054            )
16055            .unwrap();
16056
16057            let replies = swap
16058                .handler
16059                .handle_control_frame(&swap.candidate_ctx, update)
16060                .await
16061                .unwrap();
16062
16063            assert_eq!(replies.len(), 1);
16064            assert_eq!(
16065                replies[0].header.ty,
16066                FrameType::Response,
16067                "candidate catalog.update was refused: {:?}",
16068                serde_json::from_slice::<Value>(&replies[0].body).ok()
16069            );
16070            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16071            assert_eq!(
16072                swap.registry
16073                    .get_module("aft")
16074                    .unwrap()
16075                    .unwrap()
16076                    .connection_id,
16077                INCUMBENT
16078            );
16079        }
16080    }
16081
16082    /// The HELLO gate while the supervisor has a swap open: only the nonce it
16083    /// minted for the candidate admits a second process, into the candidate
16084    /// slot, and that check runs ahead of the reserved-module gate.
16085    mod swap_admission {
16086        use super::*;
16087
16088        const INCUMBENT_NONCE: &str = "incumbent-nonce";
16089        const CANDIDATE_NONCE: &str = "candidate-nonce";
16090
16091        fn handler_with_incumbent(
16092            module_id: &str,
16093            reserved: bool,
16094        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16095            let registry = Arc::new(Registry::default());
16096            let supervisor = SupervisorHandle::new();
16097            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16098            if reserved {
16099                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16100            }
16101            let handler =
16102                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
16103            let incumbent = handler
16104                .handle_control(
16105                    ConnectionId::new(1),
16106                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16107                )
16108                .unwrap();
16109            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16110            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16111            (registry, supervisor, handler)
16112        }
16113
16114        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
16115        /// admits every nonce, so while a swap is open the swap gate is the only
16116        /// thing between a key-holder and the candidate slot. A nonce the
16117        /// supervisor did not mint, or none at all, is refused, and neither the
16118        /// incumbent's registration nor the candidate slot moves.
16119        #[test]
16120        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16121            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16122
16123            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16124                let replies = handler
16125                    .handle_control(
16126                        ConnectionId::new(connection),
16127                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16128                    )
16129                    .unwrap();
16130                assert_eq!(replies[0].header.ty, FrameType::Error);
16131                assert_eq!(
16132                    parse_error(&replies[0])["code"],
16133                    "swap_token_invalid",
16134                    "nonce {nonce:?}"
16135                );
16136            }
16137            assert!(registry.get_candidate("aft").unwrap().is_none());
16138            assert_eq!(
16139                registry.get_module("aft").unwrap().unwrap().connection_id,
16140                ConnectionId::new(1)
16141            );
16142
16143            // Control: the minted token is admitted, into the candidate slot,
16144            // and only once.
16145            let admitted = handler
16146                .handle_control(
16147                    ConnectionId::new(4),
16148                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16149                )
16150                .unwrap();
16151            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16152            assert_eq!(
16153                registry
16154                    .get_candidate("aft")
16155                    .unwrap()
16156                    .unwrap()
16157                    .connection_id,
16158                ConnectionId::new(4)
16159            );
16160            assert_eq!(
16161                registry.get_module("aft").unwrap().unwrap().connection_id,
16162                ConnectionId::new(1),
16163                "the candidate must not take the active slot"
16164            );
16165            let replayed = handler
16166                .handle_control(
16167                    ConnectionId::new(5),
16168                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16169                )
16170                .unwrap();
16171            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16172
16173            // The case only this gate covers: the incumbent has died mid-swap,
16174            // so its duplicate refusal is gone too, and without the gate a
16175            // key-holder would take the id's ACTIVE slot.
16176            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16177            let squatter = handler
16178                .handle_control(
16179                    ConnectionId::new(6),
16180                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16181                )
16182                .unwrap();
16183            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16184            assert!(
16185                registry.get_module("aft").unwrap().is_none(),
16186                "a squatter took the active slot of an id being swapped"
16187            );
16188        }
16189
16190        /// Design mutation arm (iii). A reserved module's candidate presents a
16191        /// nonce the reserved gate has never seen (that gate holds the
16192        /// incumbent's), so the swap gate must run first or the candidate is
16193        /// refused `reserved_module` and a reserved module can never be swapped.
16194        #[test]
16195        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16196            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16197
16198            let replies = handler
16199                .handle_control(
16200                    ConnectionId::new(2),
16201                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16202                )
16203                .unwrap();
16204
16205            assert_eq!(
16206                replies[0].header.ty,
16207                FrameType::HelloAck,
16208                "reserved candidate refused: {:?}",
16209                serde_json::from_slice::<Value>(&replies[0].body).ok()
16210            );
16211            assert_eq!(
16212                registry
16213                    .get_candidate("vault")
16214                    .unwrap()
16215                    .unwrap()
16216                    .connection_id,
16217                ConnectionId::new(2)
16218            );
16219        }
16220
16221        /// With no swap open the gate is inert: the incumbent's reserved gate
16222        /// and duplicate refusal behave exactly as before.
16223        #[test]
16224        fn without_an_open_swap_the_ordinary_gates_decide() {
16225            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16226            supervisor.close_swap("vault");
16227
16228            let candidate = handler
16229                .handle_control(
16230                    ConnectionId::new(2),
16231                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16232                )
16233                .unwrap();
16234            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16235            let duplicate = handler
16236                .handle_control(
16237                    ConnectionId::new(3),
16238                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16239                )
16240                .unwrap();
16241            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16242            assert!(registry.get_candidate("vault").unwrap().is_none());
16243        }
16244    }
16245
16246    /// `scope.sync` and `scope.describe` through the real control handler: who
16247    /// may sync is decided by the registration and launch nonce of the module
16248    /// connection, never by the request body.
16249    mod scopes {
16250        mod idle_tests {
16251            use super::*;
16252            include!("scope_idle_tests.rs");
16253        }
16254        mod route_open_timings {
16255            use super::*;
16256            include!("route_open_timing_tests.rs");
16257        }
16258        mod route_open_refusal_log {
16259            use super::*;
16260            include!("route_open_refusal_log_tests.rs");
16261        }
16262        mod activity {
16263            include!("activity_scope_tests.rs");
16264        }
16265        mod delta {
16266            include!("scope_handler_delta_tests.rs");
16267        }
16268        mod call_modes {
16269            include!("scope_call_mode_tests.rs");
16270        }
16271        #[tokio::test]
16272        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16273            let rig = rig().await;
16274            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16275                "s".into(),
16276                1,
16277                "tool".into(),
16278            ))
16279            .unwrap();
16280            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16281            let (direct, _rx) = wide_ctx(999);
16282            for (ctx, expected) in [
16283                (&rig.owner, FrameType::Response),
16284                (&direct, FrameType::Error),
16285            ] {
16286                let frame =
16287                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16288                        .unwrap();
16289                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16290                assert_eq!(replies[0].header.ty, expected);
16291                if expected == FrameType::Response {
16292                    let reply: subc_control::ScopeServedByReply =
16293                        serde_json::from_slice(&replies[0].body).unwrap();
16294                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16295                } else {
16296                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16297                }
16298            }
16299            let reply = call(
16300                &rig.handler,
16301                &rig.owner,
16302                &ModuleControlRequestFromModule::ScopeDescribe {
16303                    owner: Principal::Reserved {
16304                        module_id: OWNER.into(),
16305                    },
16306                    scope_ref: "s".into(),
16307                },
16308            )
16309            .await;
16310            assert!(matches!(
16311                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16312                ModuleControlResponseToModule::ScopeDescribe { .. }
16313            ));
16314        }
16315
16316        use subc_protocol::scope::{
16317            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16318            ScopeStatus,
16319        };
16320
16321        use super::*;
16322
16323        const OWNER: &str = "prefrontal-core";
16324
16325        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16326            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16327        }
16328
16329        async fn call(
16330            handler: &ControlHandler,
16331            ctx: &RouteCtx,
16332            request: &ModuleControlRequestFromModule,
16333        ) -> Frame {
16334            let body = serde_json::to_vec(request).unwrap();
16335            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16336            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16337            assert_eq!(replies.len(), 1, "{replies:?}");
16338            replies.pop().unwrap()
16339        }
16340
16341        async fn sync(
16342            handler: &ControlHandler,
16343            ctx: &RouteCtx,
16344            generation: u64,
16345            scopes: Vec<ScopeRecord>,
16346        ) -> Result<ModuleControlResponseToModule, String> {
16347            let reply = call(
16348                handler,
16349                ctx,
16350                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16351            )
16352            .await;
16353            match reply.header.ty {
16354                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16355                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16356            }
16357        }
16358
16359        async fn describe(
16360            handler: &ControlHandler,
16361            ctx: &RouteCtx,
16362            owner: &str,
16363            scope_ref: &str,
16364        ) -> ModuleControlResponseToModule {
16365            let reply = call(
16366                handler,
16367                ctx,
16368                &ModuleControlRequestFromModule::ScopeDescribe {
16369                    owner: Principal::Reserved {
16370                        module_id: owner.to_string(),
16371                    },
16372                    scope_ref: scope_ref.to_string(),
16373                },
16374            )
16375            .await;
16376            assert_eq!(
16377                reply.header.ty,
16378                FrameType::Response,
16379                "{:?}",
16380                parse_error(&reply)
16381            );
16382            serde_json::from_slice(&reply.body).unwrap()
16383        }
16384
16385        #[tokio::test]
16386        async fn scope_describe_never_echoes_bind_proof_or_hash() {
16387            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16388            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16389            let mut rig = rig().await;
16390            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16391                .await;
16392            let _bound = rig
16393                .bound(
16394                    Some(AFT),
16395                    PLEXUS,
16396                    Some(
16397                        rig.selector("s", Some(1))
16398                            .with_bind_proof(Some(PROOF.into())),
16399                    ),
16400                )
16401                .await;
16402            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16403            let output = serde_json::to_string(&reply).unwrap();
16404            assert!(output.contains("\"status\":\"live\""));
16405            assert!(!output.contains(PROOF));
16406            assert!(!output.contains(HASH));
16407        }
16408
16409        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
16410        async fn module(
16411            handler: &ControlHandler,
16412            connection: u64,
16413            module_id: &str,
16414            nonce: Option<&str>,
16415        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16416            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16417            hello_via_sink(
16418                handler,
16419                &ctx,
16420                &mut rx,
16421                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16422            )
16423            .await;
16424            (ctx, rx)
16425        }
16426
16427        /// `direct` and every other client connection has no registration, so
16428        /// it can neither sync nor own a scope.
16429        #[tokio::test]
16430        async fn a_client_connection_cannot_sync_or_describe() {
16431            let handler = ControlHandler::new(Arc::new(Registry::default()));
16432            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16433            for request in [
16434                ModuleControlRequestFromModule::ScopeSync {
16435                    generation: 1,
16436                    scopes: vec![head("s", 1)],
16437                },
16438                ModuleControlRequestFromModule::ScopeDescribe {
16439                    owner: Principal::Direct,
16440                    scope_ref: "s".to_string(),
16441                },
16442            ] {
16443                let reply = call(&handler, &ctx, &request).await;
16444                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16445            }
16446            assert!(
16447                !handler
16448                    .scopes
16449                    .read()
16450                    .unwrap()
16451                    .describe(
16452                        &Principal::Reserved {
16453                            module_id: OWNER.to_string()
16454                        },
16455                        "s"
16456                    )
16457                    .owner_synced
16458            );
16459        }
16460
16461        /// A module the supervisor did not spawn registers without a launch
16462        /// nonce, so it is never an owner's current launch.
16463        #[tokio::test]
16464        async fn a_module_without_a_supervised_launch_cannot_sync() {
16465            let handler = ControlHandler::new(Arc::new(Registry::default()));
16466            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16467            assert_eq!(
16468                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16469                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16470            );
16471        }
16472
16473        #[tokio::test]
16474        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16475            let supervisor = SupervisorHandle::new();
16476            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16477            let handler = ControlHandler::new(Arc::new(Registry::default()))
16478                .with_supervisor(supervisor.clone());
16479            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16480            sync(&handler, &incumbent, 1, vec![head("s", 1)])
16481                .await
16482                .expect("the current launch syncs");
16483
16484            // A swap candidate registers with the swap token and is refused
16485            // while the incumbent keeps syncing.
16486            supervisor.open_swap(OWNER, "n2".to_string());
16487            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16488            assert_eq!(
16489                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16490                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16491            );
16492            sync(&handler, &incumbent, 2, vec![head("s", 1)])
16493                .await
16494                .expect("the serving owner syncs during the swap");
16495
16496            // The swap fails and is rolled back. The candidate never held sync
16497            // authority, and still cannot sync.
16498            supervisor.close_swap(OWNER);
16499            assert_eq!(
16500                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16501                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16502            );
16503            sync(&handler, &incumbent, 3, vec![head("s", 1)])
16504                .await
16505                .expect("the serving owner syncs after the rollback");
16506            handler.cleanup_connection(candidate.connection_id).unwrap();
16507
16508            // A swap that cuts over. Promotion records the candidate's nonce as
16509            // the module's spawn nonce, which is what `set_spawn_nonce` does
16510            // here; the promoted connection then takes authority at any
16511            // generation and the superseded incumbent is refused.
16512            supervisor.open_swap(OWNER, "n3".to_string());
16513            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16514            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16515            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16516                .await
16517                .expect("the promoted launch takes authority");
16518            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16519                panic!("unexpected reply {reply:?}");
16520            };
16521            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16522            assert_eq!(
16523                sync(&handler, &incumbent, 4, Vec::new()).await,
16524                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16525            );
16526        }
16527
16528        /// Authority dies with its connection: the cleanup path releases it,
16529        /// so the owner's next connection takes it at any generation.
16530        #[tokio::test]
16531        async fn closing_the_authority_connection_frees_sync_authority() {
16532            let supervisor = SupervisorHandle::new();
16533            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16534            let handler = ControlHandler::new(Arc::new(Registry::default()))
16535                .with_supervisor(supervisor.clone());
16536            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16537            sync(&handler, &first, 10, vec![head("s", 1)])
16538                .await
16539                .unwrap();
16540            handler.cleanup_connection(first.connection_id).unwrap();
16541
16542            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16543            sync(&handler, &second, 1, vec![head("s", 1)])
16544                .await
16545                .expect("the next connection takes the released authority");
16546        }
16547
16548        #[tokio::test]
16549        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16550            let supervisor = SupervisorHandle::new();
16551            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16552            let handler =
16553                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16554            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16555            sync(&handler, &first, 10, vec![head("s", 1)])
16556                .await
16557                .unwrap();
16558            handler
16559                .handle_control_frame(
16560                    &first,
16561                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16562                )
16563                .await
16564                .unwrap();
16565            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16566            sync(&handler, &second, 1, vec![head("s", 1)])
16567                .await
16568                .expect("GOODBYE releases authority even if the old socket remains open");
16569        }
16570
16571        #[tokio::test]
16572        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16573            let registry = Arc::new(Registry::default());
16574            let supervisor_handle = SupervisorHandle::new();
16575            let supervisor =
16576                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16577                    .with_handle(supervisor_handle.clone())
16578                    .with_daemon_incarnation("incarnation-7".to_string());
16579            // Configured with enabled: false, so the supervisor lists the
16580            // module without spawning a process for it.
16581            supervisor
16582                .supervise_configured(
16583                    ModuleSpec {
16584                        module_id: OWNER.to_string(),
16585                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16586                        args: Vec::new(),
16587                        env: Vec::new(),
16588                        reserved: false,
16589                        reserved_prefixes: Vec::new(),
16590                        protocol: ModuleProtocol::Subc,
16591                        overlap: Default::default(),
16592                    },
16593                    false,
16594                )
16595                .unwrap();
16596            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16597            let handler =
16598                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
16599            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16600
16601            // Configured but not yet synced: a reader waits for the owner.
16602            let ModuleControlResponseToModule::ScopeDescribe {
16603                status,
16604                daemon_incarnation,
16605                owner_synced,
16606                owner_configured,
16607                scope,
16608                ..
16609            } = describe(&handler, &reader, OWNER, "s").await
16610            else {
16611                panic!("not a describe reply");
16612            };
16613            assert_eq!(status, ScopeStatus::NotLive);
16614            assert_eq!(daemon_incarnation, "incarnation-7");
16615            assert!(!owner_synced);
16616            assert!(owner_configured);
16617            assert!(scope.is_none());
16618
16619            // Not a supervised module: the owner will never sync, and a reader
16620            // refuses rather than waits.
16621            let ModuleControlResponseToModule::ScopeDescribe {
16622                status,
16623                owner_configured,
16624                ..
16625            } = describe(&handler, &reader, "ghost", "s").await
16626            else {
16627                panic!("not a describe reply");
16628            };
16629            assert_eq!(status, ScopeStatus::NotLive);
16630            assert!(!owner_configured);
16631
16632            // Live, with the stamp fields and the computed owner_authorized.
16633            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16634            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16635            let ModuleControlResponseToModule::ScopeDescribe {
16636                status,
16637                scope_epoch,
16638                owner_synced,
16639                scope,
16640                ..
16641            } = describe(&handler, &reader, OWNER, "s").await
16642            else {
16643                panic!("not a describe reply");
16644            };
16645            assert_eq!(status, ScopeStatus::Live);
16646            assert_eq!(scope_epoch, Some(4));
16647            assert!(owner_synced);
16648            let stamp = scope.expect("a live scope carries its stamp");
16649            assert!(
16650                stamp.owner_authorized,
16651                "prefrontal-core is the default authority"
16652            );
16653            assert_eq!(stamp.kind, ScopeKind::Head);
16654        }
16655
16656        #[tokio::test]
16657        async fn scope_authority_owners_decides_owner_authorized() {
16658            let supervisor = SupervisorHandle::new();
16659            supervisor.set_spawn_nonce("broca", "b1".to_string());
16660            let handler = ControlHandler::new(Arc::new(Registry::default()))
16661                .with_supervisor(supervisor)
16662                .with_scope_authority_owners(vec!["broca".to_string()]);
16663            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16664            let mut gated = head("s", 1);
16665            gated.attributes.agent_id = Some("agent".to_string());
16666            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16667            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16668                describe(&handler, &broca, "broca", "s").await
16669            else {
16670                panic!("not a describe reply");
16671            };
16672            assert!(scope.unwrap().owner_authorized);
16673        }
16674
16675        /// With route admission, the stamp, the commit re-check and drains in
16676        /// place, the feature is advertised: the module ops in HELLO_ACK, and
16677        /// `scopes/v1` in HELLO_ACK and `server.describe`.
16678        #[tokio::test]
16679        async fn scope_ops_and_the_scopes_capability_are_advertised() {
16680            let handler = ControlHandler::new(Arc::new(Registry::default()));
16681            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16682            let ack = hello_via_sink(
16683                &handler,
16684                &ctx,
16685                &mut rx,
16686                hello_frame("m", PROTOCOL_VERSION, 1),
16687            )
16688            .await;
16689            let ack = parse_ack(&ack);
16690            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16691                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16692            }
16693            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16694
16695            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16696            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16697            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16698            let reply = handler
16699                .handle_control_frame(&client, frame)
16700                .await
16701                .unwrap()
16702                .pop()
16703                .unwrap();
16704            let ClientControlResponse::ServerDescribe { capabilities, .. } =
16705                serde_json::from_slice(&reply.body).unwrap()
16706            else {
16707                panic!("not a server.describe reply");
16708            };
16709            assert!(
16710                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16711                "{capabilities:?}"
16712            );
16713        }
16714
16715        // ---- route admission, stamps, commit re-check and drains ----------
16716
16717        const PLEXUS: &str = "plexus";
16718        const OTHER: &str = "other";
16719        const AFT: &str = "aft";
16720        const BROCA: &str = "broca";
16721        const MAGIC: &str = "magic-context";
16722
16723        fn nonce(module_id: &str) -> String {
16724            format!("nonce-{module_id}")
16725        }
16726
16727        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16728            let (tx, rx) = mpsc::channel(64);
16729            (
16730                RouteCtx {
16731                    connection_id: ConnectionId::new(connection),
16732                    egress: FrameSink::new(tx),
16733                },
16734                rx,
16735            )
16736        }
16737
16738        /// A daemon with a configured owner (prefrontal-core) registered on its
16739        /// own module connection, two routable targets (plexus, other), and
16740        /// launch nonces minted for the modules that open routes as carriers.
16741        struct Rig {
16742            handler: ControlHandler,
16743            forwarding: Arc<ForwardingTable>,
16744            owner: RouteCtx,
16745            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16746            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16747            generation: u64,
16748            next_connection: u64,
16749            _supervisor: Supervisor,
16750        }
16751
16752        async fn rig() -> Rig {
16753            rig_with_flow_support(true).await
16754        }
16755
16756        async fn rig_with_flow_support(flow_support: bool) -> Rig {
16757            rig_with_scope_capabilities(flow_support, false).await
16758        }
16759
16760        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16761            let mut provides = Vec::new();
16762            if flow_support {
16763                provides.push("flow-scopes/v1");
16764            }
16765            if run_support {
16766                provides.push("agent-run-scopes/v1");
16767            }
16768            rig_with_target_capabilities(&provides).await
16769        }
16770
16771        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16772            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16773            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16774            // Decoder versions alone cannot admit scoped calls: only a capability
16775            // declaration promises the required run or flow behaviour.
16776            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16777            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16778            Frame::build(
16779                FrameType::Hello,
16780                control_flags(),
16781                0,
16782                0,
16783                connection,
16784                serde_json::to_vec(&body).unwrap(),
16785            )
16786            .unwrap()
16787        }
16788
16789        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16790            let registry = Arc::new(Registry::default());
16791            let forwarding = Arc::new(ForwardingTable::default());
16792            let supervisor_handle = SupervisorHandle::new();
16793            let supervisor =
16794                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16795                    .with_handle(supervisor_handle.clone());
16796            supervisor
16797                .supervise_configured(
16798                    ModuleSpec {
16799                        module_id: OWNER.to_string(),
16800                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16801                        args: Vec::new(),
16802                        env: Vec::new(),
16803                        reserved: false,
16804                        reserved_prefixes: Vec::new(),
16805                        protocol: ModuleProtocol::Subc,
16806                        overlap: Default::default(),
16807                    },
16808                    false,
16809                )
16810                .unwrap();
16811            for module_id in [OWNER, AFT, BROCA, MAGIC] {
16812                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16813            }
16814            let handler =
16815                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16816                    .with_supervisor(supervisor_handle);
16817            let (owner, mut owner_rx) = wide_ctx(1);
16818            hello_via_sink(
16819                &handler,
16820                &owner,
16821                &mut owner_rx,
16822                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16823            )
16824            .await;
16825            let mut modules = BTreeMap::new();
16826            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16827                let (ctx, mut rx) = wide_ctx(connection);
16828                let hello = hello_with_capabilities(module_id, connection, provides);
16829                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16830                modules.insert(module_id.to_string(), (ctx, rx));
16831            }
16832            Rig {
16833                handler,
16834                forwarding,
16835                owner,
16836                _owner_rx: owner_rx,
16837                modules,
16838                generation: 0,
16839                next_connection: 100,
16840                _supervisor: supervisor,
16841            }
16842        }
16843
16844        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16845            ScopeCarrier::new(Principal::Reserved {
16846                module_id: module_id.to_string(),
16847            })
16848            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16849        }
16850
16851        /// The scope most tests open under: aft carries to any module, broca
16852        /// only to plexus and other, and the owner delegates as agent-1.
16853        fn session(scope_epoch: u64) -> ScopeRecord {
16854            let mut record = head("s", scope_epoch);
16855            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16856            record.attributes.agent_id = Some("agent-1".to_string());
16857            record.attributes.delegates = true;
16858            record
16859        }
16860
16861        impl Rig {
16862            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16863                self.generation += 1;
16864                sync(&self.handler, &self.owner, self.generation, scopes)
16865                    .await
16866                    .expect("the owner's sync is accepted");
16867            }
16868
16869            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16870                ScopeSelector::new(
16871                    Principal::Reserved {
16872                        module_id: OWNER.to_string(),
16873                    },
16874                    scope_ref.to_string(),
16875                    scope_epoch,
16876                )
16877            }
16878
16879            fn open_frame(
16880                &mut self,
16881                opener: Option<&str>,
16882                target: &str,
16883                scope: Option<ScopeSelector>,
16884            ) -> (
16885                RouteCtx,
16886                mpsc::Receiver<crate::router::OutboundFrame>,
16887                Frame,
16888            ) {
16889                self.next_connection += 1;
16890                let (ctx, rx) = wide_ctx(self.next_connection);
16891                let root = unique_project_root("scoped-open");
16892                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16893                    target: RouteTarget::ToolProvider {
16894                        module_id: target.to_string(),
16895                    },
16896                    identity: BindIdentity::new(
16897                        root.path().to_path_buf(),
16898                        "unit".to_string(),
16899                        "session".to_string(),
16900                    ),
16901                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
16902                        module_id: module_id.to_string(),
16903                        launch_nonce: nonce(module_id),
16904                    }),
16905                    consumer_capabilities: None,
16906                    role_versions: None,
16907                    admission_facts: None,
16908                    scope,
16909                })
16910                .unwrap();
16911                let frame = Frame::build(
16912                    FrameType::Request,
16913                    control_flags(),
16914                    0,
16915                    0,
16916                    self.next_connection,
16917                    body,
16918                )
16919                .unwrap();
16920                (ctx, rx, frame)
16921            }
16922
16923            /// Open and expect a refusal before anything is relayed.
16924            async fn refused(
16925                &mut self,
16926                opener: Option<&str>,
16927                target: &str,
16928                scope: Option<ScopeSelector>,
16929            ) -> String {
16930                self.refusal_body(opener, target, scope).await["code"]
16931                    .as_str()
16932                    .unwrap()
16933                    .to_string()
16934            }
16935
16936            async fn refusal_body(
16937                &mut self,
16938                opener: Option<&str>,
16939                target: &str,
16940                scope: Option<ScopeSelector>,
16941            ) -> Value {
16942                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16943                let replies = tokio::time::timeout(
16944                    Duration::from_secs(2),
16945                    self.handler.handle_control_frame(&ctx, frame),
16946                )
16947                .await
16948                .expect("the open must be refused before waiting for a bind ack")
16949                .unwrap();
16950                assert_eq!(replies.len(), 1, "{replies:?}");
16951                assert_eq!(replies[0].header.ty, FrameType::Error);
16952                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16953                assert!(
16954                    module_rx.try_recv().is_err(),
16955                    "a refused open relays nothing"
16956                );
16957                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16958                parse_error(&replies[0])
16959            }
16960
16961            /// Start an open and return its task and the bind the target got.
16962            async fn relayed(
16963                &mut self,
16964                opener: Option<&str>,
16965                target: &str,
16966                scope: Option<ScopeSelector>,
16967            ) -> Relayed {
16968                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16969                let handler = self.handler.clone();
16970                let task_ctx = ctx.clone();
16971                let task = tokio::spawn(async move {
16972                    handler
16973                        .handle_control_frame(&task_ctx, frame)
16974                        .await
16975                        .unwrap()
16976                });
16977                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16978                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16979                    .await
16980                    .expect("the target receives the relayed route.bind")
16981                    .unwrap()
16982                    .frame;
16983                Relayed {
16984                    target: target.to_string(),
16985                    client: ctx,
16986                    client_rx: rx,
16987                    task,
16988                    bind,
16989                }
16990            }
16991
16992            async fn ack(&self, relayed: &Relayed) {
16993                let (module, _) = &self.modules[&relayed.target];
16994                self.handler
16995                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16996                    .await
16997                    .unwrap();
16998            }
16999
17000            /// Open, ack and return the bound route.
17001            async fn bound(
17002                &mut self,
17003                opener: Option<&str>,
17004                target: &str,
17005                scope: Option<ScopeSelector>,
17006            ) -> Bound {
17007                let relayed = self.relayed(opener, target, scope).await;
17008                self.ack(&relayed).await;
17009                let Relayed {
17010                    target,
17011                    client,
17012                    mut client_rx,
17013                    task,
17014                    bind,
17015                } = relayed;
17016                assert!(
17017                    task.await.unwrap().is_empty(),
17018                    "the open is answered by commit"
17019                );
17020                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17021                Bound {
17022                    target,
17023                    client,
17024                    client_rx,
17025                    channel,
17026                    epoch,
17027                    bind,
17028                }
17029            }
17030
17031            fn live(&self, route: &Bound) -> bool {
17032                matches!(
17033                    self.forwarding
17034                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17035                        .unwrap(),
17036                    DataRoute::Client(DataRouteState::Bound(_))
17037                )
17038            }
17039        }
17040
17041        struct Relayed {
17042            target: String,
17043            client: RouteCtx,
17044            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17045            task: tokio::task::JoinHandle<Vec<Frame>>,
17046            bind: Frame,
17047        }
17048
17049        struct Bound {
17050            target: String,
17051            client: RouteCtx,
17052            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17053            channel: u16,
17054            epoch: u32,
17055            bind: Frame,
17056        }
17057
17058        impl Bound {
17059            /// The reason of the `route.closed` this client was sent, after
17060            /// checking it also got a GOODBYE on exactly this route.
17061            fn closed_reason(&mut self) -> RouteCloseReason {
17062                let mut reason = None;
17063                let mut goodbye = false;
17064                while let Ok(outbound) = self.client_rx.try_recv() {
17065                    let frame = outbound.frame;
17066                    match frame.header.ty {
17067                        FrameType::Goodbye => {
17068                            assert_eq!(
17069                                (frame.header.channel, frame.header.epoch),
17070                                (self.channel, self.epoch)
17071                            );
17072                            goodbye = true;
17073                        }
17074                        FrameType::Push => {
17075                            let ClientControlPush::RouteClosed {
17076                                reason: r,
17077                                module_id,
17078                                ..
17079                            } = serde_json::from_slice(&frame.body).unwrap()
17080                            else {
17081                                panic!("unexpected push");
17082                            };
17083                            assert_eq!(module_id, self.target);
17084                            reason = Some(r);
17085                        }
17086                        other => panic!("unexpected frame {other:?}"),
17087                    }
17088                }
17089                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17090                reason.expect("the client is told why the route closed")
17091            }
17092
17093            fn untouched(&mut self) -> bool {
17094                self.client_rx.try_recv().is_err()
17095            }
17096
17097            fn stamp(&self) -> Option<ScopeStamp> {
17098                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17099                    ModuleControlRequest::RouteBind { scope, .. } => scope,
17100                    other => panic!("expected a route.bind, got {other:?}"),
17101                }
17102            }
17103        }
17104
17105        #[tokio::test]
17106        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17107        ) {
17108            let mut rig = rig().await;
17109            let mut record = session(1);
17110            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17111            record.child_owners = vec![Principal::Reserved {
17112                module_id: MAGIC.to_string(),
17113            }];
17114            rig.sync(vec![record]).await;
17115            let scope = || Some(rig_selector("s", Some(1)));
17116
17117            // Admitted: the owner, a bare carrier to any module, a targeted
17118            // carrier to its listed module.
17119            rig.bound(Some(OWNER), PLEXUS, scope()).await;
17120            rig.bound(Some(AFT), OTHER, scope()).await;
17121            rig.bound(Some(BROCA), PLEXUS, scope()).await;
17122
17123            // Refused scope_not_carrier: a targeted carrier to an unlisted
17124            // module, a module that is not listed at all (a child owner is not
17125            // a carrier), and a direct key-holder.
17126            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17127                assert_eq!(
17128                    rig.refused(opener, target, scope()).await,
17129                    error_codes::SCOPE_NOT_CARRIER,
17130                    "{opener:?} -> {target}"
17131                );
17132            }
17133        }
17134
17135        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17136            ScopeSelector::new(
17137                Principal::Reserved {
17138                    module_id: OWNER.to_string(),
17139                },
17140                scope_ref.to_string(),
17141                scope_epoch,
17142            )
17143        }
17144
17145        #[tokio::test]
17146        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17147            let mut rig = rig().await;
17148            rig.sync(vec![session(1)]).await;
17149            for opener in [OWNER, AFT] {
17150                assert_eq!(
17151                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17152                        .await,
17153                    error_codes::SCOPE_EPOCH_REQUIRED,
17154                    "{opener}"
17155                );
17156            }
17157        }
17158
17159        #[tokio::test]
17160        async fn admission_separates_not_synced_not_live_and_ended() {
17161            let mut rig = rig().await;
17162            // Before the configured owner's first sync: retryable.
17163            let code = rig
17164                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17165                .await;
17166            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17167            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17168
17169            // An owner that is not configured will never sync: terminal.
17170            let ghost = ScopeSelector::new(
17171                Principal::Reserved {
17172                    module_id: "ghost".to_string(),
17173                },
17174                "s".to_string(),
17175                Some(1),
17176            );
17177            assert_eq!(
17178                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17179                error_codes::SCOPE_NOT_LIVE
17180            );
17181
17182            rig.sync(vec![session(2)]).await;
17183            assert_eq!(
17184                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17185                    .await,
17186                error_codes::SCOPE_NOT_LIVE
17187            );
17188            for epoch in [1, 3] {
17189                assert_eq!(
17190                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17191                        .await,
17192                    error_codes::SCOPE_ENDED,
17193                    "epoch {epoch}"
17194                );
17195            }
17196            // Control: the live epoch is admitted.
17197            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17198                .await;
17199        }
17200
17201        #[tokio::test]
17202        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17203            let mut rig = rig().await;
17204            rig.sync(vec![session(1)]).await;
17205            let route = rig
17206                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17207                .await;
17208            let stamp = route.stamp().expect("a scoped bind carries the stamp");
17209            assert_eq!(stamp.scope_ref, "s");
17210            assert_eq!(stamp.scope_epoch, 1);
17211            assert_eq!(stamp.kind, ScopeKind::Head);
17212            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17213            assert!(stamp.attributes.delegates);
17214            assert!(stamp.owner_authorized);
17215            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17216            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17217
17218            // broca owns a scope of its own on its own module connection; it is
17219            // not in scope_authority_owners, so its stamp is not authorized.
17220            let (broca, mut broca_rx) = wide_ctx(50);
17221            hello_via_sink(
17222                &rig.handler,
17223                &broca,
17224                &mut broca_rx,
17225                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17226            )
17227            .await;
17228            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17229                .await
17230                .unwrap();
17231            let own = ScopeSelector::new(
17232                Principal::Reserved {
17233                    module_id: BROCA.to_string(),
17234                },
17235                "b".to_string(),
17236                Some(1),
17237            );
17238            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17239            assert!(!route.stamp().unwrap().owner_authorized);
17240        }
17241
17242        #[tokio::test]
17243        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17244            let mut rig = rig().await;
17245            let mut record = head("s", 1);
17246            record.carriers = vec![carrier(AFT, None)];
17247            let flow_id = "Flow:run-7/step_2!~";
17248            record.attributes.flow_id = Some(flow_id.to_string());
17249            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17250                .await
17251                .unwrap();
17252            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17253                panic!("not a sync reply");
17254            };
17255            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17256            let route = rig
17257                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17258                .await;
17259            assert!(rig.live(&route), "the stamped bind committed");
17260            let stamp = route.stamp().expect("a flow scope carries a stamp");
17261            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17262            assert_eq!(stamp.attributes.agent_id, None);
17263            assert!(!stamp.attributes.delegates);
17264            assert!(stamp.owner_authorized);
17265            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17266            assert_eq!(unscoped.stamp(), None);
17267        }
17268
17269        #[tokio::test]
17270        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17271            let mut rig = rig_with_flow_support(false).await;
17272            let mut record = session(1);
17273            record.attributes.flow_id = Some("flow:7".to_string());
17274            rig.sync(vec![record]).await;
17275            for opener in [OWNER, AFT] {
17276                let body = rig
17277                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17278                    .await;
17279                assert_eq!(body["code"], "target_flow_unsupported");
17280                let message = body["message"].as_str().unwrap();
17281                for required in [PLEXUS, "flow-scopes/v1"] {
17282                    assert!(message.contains(required), "{message}");
17283                }
17284            }
17285        }
17286
17287        #[tokio::test]
17288        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17289            let mut rig = rig_with_flow_support(true).await;
17290            let mut record = session(1);
17291            record.attributes.flow_id = Some("flow:7".to_string());
17292            rig.sync(vec![record]).await;
17293            for opener in [OWNER, AFT] {
17294                let route = rig
17295                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17296                    .await;
17297                assert!(rig.live(&route));
17298                assert_eq!(
17299                    route.stamp().unwrap().attributes.flow_id.as_deref(),
17300                    Some("flow:7")
17301                );
17302            }
17303        }
17304
17305        #[tokio::test]
17306        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17307            use std::future::Future;
17308
17309            let mut rig = rig_with_flow_support(true).await;
17310            let mut record = session(1);
17311            record.attributes.flow_id = Some("flow:7".to_string());
17312            rig.sync(vec![record]).await;
17313            let (client, mut client_rx, frame) =
17314                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17315            // Hold the route-open response permit so admission sees the first
17316            // target but relay reservation cannot capture an endpoint yet.
17317            for _ in 0..64 {
17318                client.egress.try_send(route_bind_ack(1)).unwrap();
17319            }
17320            let handler = rig.handler.clone();
17321            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17322            std::future::poll_fn(|cx| {
17323                assert!(open.as_mut().poll(cx).is_pending());
17324                std::task::Poll::Ready(())
17325            })
17326            .await;
17327            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17328
17329            let old_connection = rig.modules[PLEXUS].0.connection_id;
17330            rig.handler.cleanup_connection(old_connection).unwrap();
17331            let (replacement, mut replacement_rx) = wide_ctx(200);
17332            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17333            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17334            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17335            let hello = Frame::build(
17336                FrameType::Hello,
17337                control_flags(),
17338                0,
17339                0,
17340                200,
17341                serde_json::to_vec(&body).unwrap(),
17342            )
17343            .unwrap();
17344            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17345
17346            client_rx.try_recv().unwrap();
17347            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17348                .await
17349                .expect("the replacement is refused without waiting for a bind ack")
17350                .unwrap();
17351            assert_eq!(replies.len(), 1);
17352            let body = parse_error(&replies[0]);
17353            assert_eq!(body["code"], "target_flow_unsupported");
17354            for required in [PLEXUS, "flow-scopes/v1"] {
17355                assert!(body["message"].as_str().unwrap().contains(required));
17356            }
17357            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17358            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17359            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17360            assert!(rig
17361                .handler
17362                .registry
17363                .get_module_by_connection(replacement.connection_id)
17364                .unwrap()
17365                .is_some());
17366        }
17367
17368        #[tokio::test]
17369        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17370            use std::future::Future;
17371            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17372            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17373            const NEXT_HASH: &str =
17374                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17375            let mut rig = rig().await;
17376            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17377            record.carriers.push(ScopeCarrier::new(Principal::Direct));
17378            rig.sync(vec![record.clone()]).await;
17379            let (client, mut rx, frame) = rig.open_frame(
17380                None,
17381                PLEXUS,
17382                Some(
17383                    rig.selector("s", Some(1))
17384                        .with_bind_proof(Some(PROOF.into())),
17385                ),
17386            );
17387            // Hold egress reservation while the scope owner changes its bind-proof
17388            // hash, so a proof valid at admission must be checked again at relay.
17389            for _ in 0..64 {
17390                client.egress.try_send(route_bind_ack(1)).unwrap();
17391            }
17392            let handler = rig.handler.clone();
17393            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17394            std::future::poll_fn(|cx| {
17395                assert!(open.as_mut().poll(cx).is_pending());
17396                std::task::Poll::Ready(())
17397            })
17398            .await;
17399            record.bind_proof_sha256 = Some(NEXT_HASH.into());
17400            rig.sync(vec![record]).await;
17401            rx.try_recv().unwrap();
17402            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17403                .await
17404                .unwrap()
17405                .unwrap();
17406            assert_eq!(
17407                parse_error(&replies[0])["code"],
17408                error_codes::SCOPE_BIND_PROOF_MISMATCH
17409            );
17410            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17411            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17412            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17413        }
17414
17415        #[tokio::test]
17416        async fn scope_bind_proof_reserved_openers_ignore_proof() {
17417            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17418            let mut rig = rig().await;
17419            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17420                .await;
17421            for opener in [OWNER, AFT] {
17422                for proof in [None, Some("malformed+proof=")] {
17423                    let route = rig
17424                        .bound(
17425                            Some(opener),
17426                            PLEXUS,
17427                            Some(
17428                                rig.selector("s", Some(1))
17429                                    .with_bind_proof(proof.map(str::to_string)),
17430                            ),
17431                        )
17432                        .await;
17433                    assert!(rig.live(&route));
17434                }
17435            }
17436        }
17437
17438        #[tokio::test]
17439        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17440            use std::future::Future;
17441            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17442            const NEXT_HASH: &str =
17443                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17444            let mut rig = rig().await;
17445            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17446                .await;
17447            let (client, mut rx, frame) =
17448                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17449            for _ in 0..64 {
17450                client.egress.try_send(route_bind_ack(1)).unwrap();
17451            }
17452            let handler = rig.handler.clone();
17453            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17454            std::future::poll_fn(|cx| {
17455                assert!(open.as_mut().poll(cx).is_pending());
17456                std::task::Poll::Ready(())
17457            })
17458            .await;
17459            rig.sync(vec![
17460                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17461            ])
17462            .await;
17463            rx.try_recv().unwrap();
17464            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17465            let bind = std::future::poll_fn(|cx| {
17466                assert!(open.as_mut().poll(cx).is_pending());
17467                module_rx.poll_recv(cx)
17468            })
17469            .await
17470            .unwrap()
17471            .frame;
17472            handler
17473                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17474                .await
17475                .unwrap();
17476            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17477                .await
17478                .unwrap()
17479                .unwrap();
17480            assert_eq!(replies.len(), 1);
17481            // Proof rechecking allowed the relay. Independent tag settlement
17482            // still asks the opener to retry after any same-epoch content change.
17483            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17484            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17485            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17486            let route = rig
17487                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17488                .await;
17489            assert!(rig.live(&route));
17490        }
17491
17492        #[tokio::test]
17493        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17494        {
17495            let mut rig = rig_with_flow_support(false).await;
17496            rig.sync(vec![session(1)]).await;
17497            let route = rig
17498                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17499                .await;
17500            assert!(rig.live(&route));
17501            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17502            let mut record = session(1);
17503            record.attributes.flow_id = Some("flow:7".to_string());
17504            rig.sync(vec![record]).await;
17505            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17506            assert!(rig.live(&unscoped));
17507            assert_eq!(unscoped.stamp(), None);
17508        }
17509
17510        #[tokio::test]
17511        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17512            let mut rig = rig().await;
17513            let mut record = session(1);
17514            record.attributes.flow_id = Some("flow:7".to_string());
17515            rig.sync(vec![record.clone()]).await;
17516            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17517            let mut owner_route = rig
17518                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17519                .await;
17520            let mut carrier_route = rig
17521                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17522                .await;
17523            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17524            rig.sync(vec![record.clone()]).await;
17525            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17526            assert!(rig.live(&owner_route) && owner_route.untouched());
17527            assert!(rig.live(&carrier_route) && carrier_route.untouched());
17528
17529            record.attributes.flow_id = Some("flow:8".to_string());
17530            rig.sync(vec![record]).await;
17531            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17532            let before = before.unwrap();
17533            assert_eq!(after.scope_epoch, before.scope_epoch);
17534            assert!(after.version > before.version);
17535            for route in [&mut owner_route, &mut carrier_route] {
17536                assert!(!rig.live(route));
17537                assert_eq!(
17538                    route.closed_reason(),
17539                    RouteCloseReason::ScopeDelegationChanged
17540                );
17541            }
17542            assert!(rig.live(&unscoped) && unscoped.untouched());
17543            // Each provider also receives a GOODBYE for its drained route;
17544            // consume it before expecting the next route.bind on that sink.
17545            for target in [PLEXUS, OTHER] {
17546                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17547                let goodbye = module_rx
17548                    .try_recv()
17549                    .expect("the provider sees the drain")
17550                    .frame;
17551                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17552                assert!(module_rx.try_recv().is_err());
17553            }
17554            let rebound = rig
17555                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17556                .await;
17557            assert_eq!(
17558                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17559                Some("flow:8")
17560            );
17561        }
17562
17563        /// The owner's sync lands between admission and the module's ack. The
17564        /// open is refused by name, the module's other routes stay up, and the
17565        /// reserved pair is released. Changed content is retryable; an ended
17566        /// scope is not.
17567        #[tokio::test]
17568        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17569            let mut rig = rig().await;
17570            rig.sync(vec![session(1)]).await;
17571            let mut cotenant = rig
17572                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17573                .await;
17574
17575            let mut changed = session(1);
17576            changed.child_owners.push(Principal::Reserved {
17577                module_id: MAGIC.to_string(),
17578            });
17579            let mut ended = None;
17580            for (code, next) in [
17581                (error_codes::SCOPE_CHANGED, vec![changed]),
17582                (error_codes::SCOPE_ENDED, Vec::new()),
17583            ] {
17584                let relayed = rig
17585                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17586                    .await;
17587                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17588                ended = Some(next.is_empty());
17589                rig.sync(next).await;
17590                rig.ack(&relayed).await;
17591                let replies = relayed.task.await.unwrap();
17592                assert_eq!(replies.len(), 1, "{replies:?}");
17593                assert_eq!(parse_error(&replies[0])["code"], code);
17594                assert_eq!(
17595                    subc_protocol::error_codes::is_retryable_route_open(code),
17596                    code == error_codes::SCOPE_CHANGED
17597                );
17598                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17599                // The module is told to drop just the binding it created.
17600                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17601                // Collected, because ending the scope also closes the co-tenant
17602                // route, whose GOODBYE comes first.
17603                let mut goodbyes = Vec::new();
17604                while let Ok(outbound) = plexus_rx.try_recv() {
17605                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17606                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17607                }
17608                assert!(
17609                    goodbyes.contains(&(bind_channel, bind_epoch)),
17610                    "{goodbyes:?}"
17611                );
17612                assert!(rig
17613                    .handler
17614                    .registry
17615                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17616                    .unwrap()
17617                    .is_some());
17618            }
17619            assert_eq!(ended, Some(true));
17620            // The co-tenant stayed up through the change, and closed only when
17621            // the scope ended, by the drain rule rather than by the commit.
17622            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17623        }
17624
17625        /// Each row of the drain table on one set of routes: the owner's, a
17626        /// bare carrier's, and a targeted carrier's to each of its targets.
17627        #[tokio::test]
17628        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17629            struct Case {
17630                name: &'static str,
17631                change: fn(&mut ScopeRecord),
17632                /// Closed routes by index: owner->plexus, aft->plexus,
17633                /// broca->plexus, broca->other.
17634                closed: [Option<RouteCloseReason>; 4],
17635            }
17636            use RouteCloseReason::*;
17637            let cases = [
17638                Case {
17639                    name: "a carrier entry removed",
17640                    change: |r| {
17641                        r.carriers.retain(|c| {
17642                            c.principal
17643                                != Principal::Reserved {
17644                                    module_id: AFT.to_string(),
17645                                }
17646                        })
17647                    },
17648                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17649                },
17650                Case {
17651                    name: "a target removed from a carrier",
17652                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17653                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
17654                },
17655                Case {
17656                    name: "a bare carrier narrowed to targets",
17657                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17658                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17659                },
17660                Case {
17661                    name: "delegates turned off",
17662                    change: |r| r.attributes.delegates = false,
17663                    closed: [Some(ScopeDelegationChanged); 4],
17664                },
17665                Case {
17666                    name: "agent_id changed",
17667                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17668                    closed: [Some(ScopeDelegationChanged); 4],
17669                },
17670                Case {
17671                    name: "a carrier added, child owners changed, the record re-sent",
17672                    change: |r| {
17673                        r.carriers.push(carrier(MAGIC, None));
17674                        r.child_owners.push(Principal::Reserved {
17675                            module_id: MAGIC.to_string(),
17676                        });
17677                    },
17678                    closed: [None; 4],
17679                },
17680                Case {
17681                    name: "a target added",
17682                    change: |r| {
17683                        r.carriers[1]
17684                            .targets
17685                            .as_mut()
17686                            .unwrap()
17687                            .push("third".to_string())
17688                    },
17689                    closed: [None; 4],
17690                },
17691                Case {
17692                    name: "delegates turned on",
17693                    change: |r| r.attributes.delegates = true,
17694                    closed: [None; 4],
17695                },
17696            ];
17697            for case in cases {
17698                let mut rig = rig().await;
17699                rig.sync(vec![session(1)]).await;
17700                let scope = || Some(rig_selector("s", Some(1)));
17701                let mut routes = [
17702                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
17703                    rig.bound(Some(AFT), PLEXUS, scope()).await,
17704                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
17705                    rig.bound(Some(BROCA), OTHER, scope()).await,
17706                ];
17707                let mut record = session(1);
17708                (case.change)(&mut record);
17709                rig.sync(vec![record]).await;
17710                for (index, expected) in case.closed.iter().enumerate() {
17711                    let route = &mut routes[index];
17712                    match expected {
17713                        Some(reason) => {
17714                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
17715                            assert_eq!(
17716                                route.closed_reason(),
17717                                *reason,
17718                                "{}: route {index}",
17719                                case.name
17720                            );
17721                        }
17722                        None => {
17723                            assert!(rig.live(route), "{}: route {index} closed", case.name);
17724                            assert!(
17725                                route.untouched(),
17726                                "{}: route {index} was told something",
17727                                case.name
17728                            );
17729                        }
17730                    }
17731                }
17732            }
17733        }
17734
17735        #[tokio::test]
17736        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17737            // Removed, and replaced by a higher epoch.
17738            for next in [Vec::new(), vec![session(2)]] {
17739                let mut rig = rig().await;
17740                rig.sync(vec![session(1)]).await;
17741                let mut route = rig
17742                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17743                    .await;
17744                rig.sync(next).await;
17745                assert!(!rig.live(&route));
17746                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17747            }
17748
17749            // A child whose parent ends: its routes close as parent-ended, the
17750            // child stays live, and routes under the parent close as ended.
17751            let mut rig = rig().await;
17752            let mut child = session(1);
17753            child.scope_ref = "child".to_string();
17754            child.kind = ScopeKind::Worker;
17755            child.parent = Some(ScopeParent::new(
17756                Principal::Reserved {
17757                    module_id: OWNER.to_string(),
17758                },
17759                "s".to_string(),
17760                1,
17761            ));
17762            rig.sync(vec![session(1), child.clone()]).await;
17763            let mut child_route = rig
17764                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17765                .await;
17766            assert_eq!(
17767                child_route.stamp().unwrap().parent_state,
17768                Some(ParentState::Linked)
17769            );
17770            rig.sync(vec![child]).await;
17771            assert!(!rig.live(&child_route));
17772            assert_eq!(
17773                child_route.closed_reason(),
17774                RouteCloseReason::ScopeParentEnded
17775            );
17776        }
17777
17778        #[tokio::test]
17779        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17780        ) {
17781            let mut rig = rig().await;
17782            rig.sync(vec![session(1)]).await;
17783            let mut route = rig
17784                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17785                .await;
17786            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17787            rig.sync(vec![session(1)]).await;
17788            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17789            assert!(rig.live(&route) && route.untouched());
17790
17791            // A call in flight on the route when another carrier is added. A
17792            // forwarded REQUEST holds one credit on the route's flow until the
17793            // module answers; the router takes it exactly like this.
17794            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17795                .forwarding
17796                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17797                .unwrap()
17798            else {
17799                panic!("the route is bound");
17800            };
17801            binding.flow.acquire_tagged(9, false).await.unwrap();
17802            let mut widened = session(1);
17803            widened.carriers.push(carrier(MAGIC, None));
17804            rig.sync(vec![widened]).await;
17805            assert!(rig.live(&route) && route.untouched());
17806            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17807            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17808            // The call's credit is still held on an open flow, so its answer
17809            // will be delivered: closing the route would have closed the flow.
17810            assert_eq!(binding.flow.in_flight(), 1);
17811            binding
17812                .flow
17813                .acquire_tagged(10, false)
17814                .await
17815                .expect("the flow is still open");
17816        }
17817
17818        /// A swap's superseded endpoint keeps its routes until drained; ending
17819        /// the scope closes them there too.
17820        #[tokio::test]
17821        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17822            let mut rig = rig().await;
17823            rig.sync(vec![session(1)]).await;
17824            let mut on_incumbent = rig
17825                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17826                .await;
17827
17828            // Swap plexus: register a candidate and cut over, leaving the
17829            // incumbent superseded with the route still on it.
17830            let (candidate, _candidate_rx) = wide_ctx(9);
17831            let registration = rig
17832                .handler
17833                .registry
17834                .register_candidate_with_control_ops(
17835                    manifest(PLEXUS, PROTOCOL_VERSION),
17836                    PROTOCOL_VERSION,
17837                    candidate.connection_id,
17838                    module_baseline_control_ops(),
17839                )
17840                .unwrap();
17841            rig.forwarding
17842                .register_candidate_module_connection(
17843                    candidate.connection_id,
17844                    PLEXUS.to_string(),
17845                    PROTOCOL_VERSION,
17846                    manifest_concurrency(&registration.manifest),
17847                    candidate.egress.clone(),
17848                )
17849                .unwrap();
17850            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17851            rig.handler
17852                .registry
17853                .promote_candidate(PLEXUS)
17854                .unwrap()
17855                .unwrap();
17856            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17857
17858            rig.sync(Vec::new()).await;
17859            assert!(!rig.live(&on_incumbent));
17860            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17861            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17862            let goodbye = incumbent_rx
17863                .try_recv()
17864                .expect("the superseded endpoint is told")
17865                .frame;
17866            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17867        }
17868    }
17869}
17870
17871#[cfg(test)]
17872mod concurrency_default_exposure_tests {
17873    use super::*;
17874
17875    fn hello_body(role_json: &str) -> Vec<u8> {
17876        format!(
17877            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17878        )
17879        .into_bytes()
17880    }
17881
17882    fn manifest_from(body: &[u8]) -> ModuleManifest {
17883        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17884        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17885            .expect("manifest parses")
17886    }
17887
17888    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17889
17890    #[test]
17891    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17892        let body = hello_body(&format!(
17893            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17894        ));
17895        let manifest = manifest_from(&body);
17896        // Precondition: serde really resolved it to the default, so the typed
17897        // manifest alone cannot answer the question this probe exists for.
17898        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17899        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17900    }
17901
17902    #[test]
17903    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17904        let body = hello_body(&format!(
17905            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17906        ));
17907        let manifest = manifest_from(&body);
17908        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17909        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17910    }
17911
17912    #[test]
17913    fn non_management_roles_are_never_reported() {
17914        let body = hello_body(
17915            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17916        );
17917        let manifest = manifest_from(&body);
17918        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17919    }
17920}