Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
26        validate_hello_role_declarations, validate_hello_self_signal_declarations,
27        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
28        ProviderRole,
29    },
30    scope::{
31        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
32        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
33        SCOPE_SYNC_OP,
34    },
35    session::{
36        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
37        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
38        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
39        ROLE_VERSIONS_FIELD,
40    },
41    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
42    Priority, RouteTarget, PROTOCOL_VERSION,
43};
44use tokio::time::{timeout_at, Instant};
45use tracing::{debug, info, warn};
46
47use crate::{
48    caller_identity::CallerIdentity,
49    capability_requirements::{
50        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
51        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
52        RequirementStatus, RuntimeModule,
53    },
54    child_roster::DaemonShutdownFlag,
55    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
56    forwarding::{
57        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
58        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
59        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
60    },
61    observability::{
62        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
63    },
64    privileged_prompt::{prompt_text, PrivilegedOperation},
65    provenance::{
66        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
67    },
68    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
69    router::{RouteCtx, RouterError},
70    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
71    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
72    stderr_tail::{CaptureState, TailEntry},
73    supervise::{
74        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
75        SupervisorHandle, SwapHelloAdmission,
76    },
77    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
78};
79
80/// Lowest envelope version this subc build will negotiate.
81///
82/// Module HELLO negotiation is exact: peers must use the daemon's locked
83/// protocol version. Older and newer peers receive `version_unsupported` and
84/// are not registered.
85pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
86
87const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
88const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
89const CAP_PING_PONG: &str = "ping_pong_v1";
90const CAP_SESSION_ATTACH: &str = "session_attach_v1";
91const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
92
93const SUBC_CONTROL_OPS: &[&str] = &[
94    ops::ACTIVITY_RECORD,
95    ops::ACTIVITY_QUERY,
96    ops::ACTIVITY_SUBSCRIBE,
97    ops::SERVER_DESCRIBE,
98    ops::CATALOG_LIST,
99    ops::ROUTE_OPEN,
100    ops::ROUTE_POLL,
101    ops::ROUTE_CLOSING,
102    ops::ROUTE_CLOSED,
103    ops::SUPERVISOR_LIST,
104    ops::SUPERVISOR_RESTART,
105    ops::SUPERVISOR_SWAP,
106    ops::SUPERVISOR_RELOAD,
107    ops::SUPERVISOR_RESCAN,
108    ops::SUPERVISOR_RELEASE_RESERVED,
109    ops::SUPERVISOR_SET_ENABLED,
110    ops::SUPERVISOR_HEALTH_PROBE,
111    ops::SUPERVISOR_HEALTH,
112    ops::SUPERVISOR_STDERR_TAIL,
113    ops::SUPERVISOR_TERMINALS,
114    ops::SUPERVISOR_ROUTES,
115    ops::SUPERVISOR_PROVENANCE,
116    ops::SUPERVISOR_SPAWN_SNAPSHOT,
117    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
118];
119
120/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
121/// announced by a capability in `server.describe` instead, and clients check
122/// that capability before sending it. `ck daemon --json` prints `subc_ops`
123/// verbatim and scripts read that output, so a new name there would change it.
124/// A malformed body for one of these is still refused `invalid_control_body`
125/// rather than as an unknown op.
126const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
127
128mod rescan_checked;
129use rescan_checked::{RescanMode, RescanPlan};
130
131const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
132    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
133    "supervisor.live_roots",
134    SCOPE_SYNC_OP,
135    SCOPE_APPLY_OP,
136    SCOPE_DESCRIBE_OP,
137    "operator.confirm",
138];
139
140/// Module-originated ops the daemon answers but does not advertise in
141/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
142/// belongs to is incomplete, so no module is told it works before it does.
143const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
144
145const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
146
147/// How long subc waits for a module to ack a relayed route.bind before returning
148/// `module_timeout`. The ack waits on the module's own configure, which for AFT
149/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
150/// and DB-open work — on a cold page cache or a large repo that legitimately
151/// exceeds a couple of seconds. The default is generous because rejecting a VALID
152/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
153/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
154pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
155
156/// How many CONSECUTIVE full-budget relay timeouts against one target module
157/// open that module's bind-relay breaker.
158///
159/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
160/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
161/// so three consecutive full-budget timeouts take ~36s to observe: every client
162/// whose open contributed to opening the breaker had already given up on its
163/// own. That is what makes opening the breaker unable to turn a call that would
164/// have succeeded into a refusal — it can only make an already-failing module
165/// fail faster.
166///
167/// Two would be reachable inside one default deadline. One would convict a
168/// module on a single cold-cache bind, which is exactly the valid-but-slow case
169/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
170pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
171
172/// How long a module's bind-relay breaker stays open before exactly one
173/// `route.open` is let through as a probe.
174///
175/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
176/// re-pays a full-budget stall almost continuously, and the breaker stops being
177/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
178/// deadline: a client that starts retrying after the module recovers has to get
179/// a probe opportunity inside its own deadline, or the breaker converts a
180/// recovered module into a failed call — the failure it exists to prevent,
181/// pointed the other way.
182///
183/// 20s sits between those with room on both sides, and it caps what a wedged
184/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
185/// rather than one per `route.open` per connection. The stall that motivated
186/// this, with its measurements, is written up in
187/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
188/// waited the whole budget out.
189pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
190
191const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
192const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
193
194fn reload_verdict(
195    configured: &Path,
196    spawned_from: Option<&Path>,
197    image: subc_control::RunningImageAgreement,
198) -> PendingReloadVerdict {
199    let path = match spawned_from {
200        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
201            ReloadPathAgreement::Match
202        }
203        Some(spawned_from) => ReloadPathAgreement::Mismatch {
204            configured: configured.to_path_buf(),
205            spawned_from: spawned_from.to_path_buf(),
206        },
207        None => ReloadPathAgreement::Unavailable {
208            reason: if matches!(
209                image,
210                subc_control::RunningImageAgreement::Unavailable {
211                    reason: subc_control::RunningImageUnavailableReason::NotRunning
212                }
213            ) {
214                ReloadPathUnavailableReason::NotRunning
215            } else {
216                ReloadPathUnavailableReason::SpawnedPathUnavailable
217            },
218        },
219    };
220    PendingReloadVerdict {
221        path,
222        image,
223        spec: None,
224    }
225}
226
227fn launch_spec_agreement(
228    configured: &crate::supervise::ModuleSpec,
229    running: bool,
230    spawned: Option<&crate::supervise::LaunchSpecDigest>,
231) -> subc_control::LaunchSpecAgreement {
232    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
233    if !running {
234        return Agreement::Unavailable {
235            reason: Reason::NotRunning,
236        };
237    }
238    match spawned {
239        None => Agreement::Unavailable {
240            reason: Reason::SpawnedSpecUnavailable,
241        },
242        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
243            Agreement::Match
244        }
245        Some(_) => Agreement::Mismatch,
246    }
247}
248
249fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
250    if configured == spawned_from {
251        return true;
252    }
253    #[cfg(windows)]
254    {
255        // Windows spawn records the resolved verbatim path (\\?\...), while
256        // configuration keeps its original spelling. Resolve both with the OS
257        // instead of stripping prefixes or folding case, which could conflate
258        // distinct names in case-sensitive directories. Different resolved
259        // paths still differ even if their files have identical content.
260        match (
261            std::fs::canonicalize(configured),
262            std::fs::canonicalize(spawned_from),
263        ) {
264            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
265            _ => false,
266        }
267    }
268    #[cfg(not(windows))]
269    {
270        false
271    }
272}
273
274#[derive(Clone)]
275struct DaemonProvenanceFacts {
276    build: DaemonBuildProvenance,
277    pid: Option<u32>,
278    started_at_ms: Option<u64>,
279    start_clock: Option<crate::clock::StartClock>,
280    executable_path: Option<PathBuf>,
281    executable_identity: Option<SpawnedFileIdentity>,
282    process_start_time: Option<u64>,
283    probe: ExecutableIdentityProbe,
284}
285
286impl Default for DaemonProvenanceFacts {
287    fn default() -> Self {
288        Self {
289            build: DaemonBuildProvenance {
290                build_git_sha: None,
291                build_lock_digest: None,
292            },
293            pid: None,
294            started_at_ms: None,
295            start_clock: None,
296            executable_path: None,
297            executable_identity: None,
298            process_start_time: None,
299            probe: ExecutableIdentityProbe::default(),
300        }
301    }
302}
303
304#[derive(Debug, Clone)]
305struct SupervisorRescanContext {
306    supervisor: Supervisor,
307    config_path: PathBuf,
308    configured_port: Option<u16>,
309    storage_config: Option<crate::daemon_config::StorageConfig>,
310    admission_facts_carrier_module_id: Option<String>,
311    admission_facts_targets: Option<Vec<String>>,
312    scope_authority_owners: Vec<String>,
313    privileged_operator_prompt: PrivilegedOperatorPrompt,
314    /// Counts executed rescans that set out to change the supervised modules'
315    /// staged or applied configuration, counted before the first change so a
316    /// rescan that fails partway still counts. With the daemon incarnation it
317    /// names the state a `supervisor.rescan_checked` preview was computed
318    /// against.
319    generation: Arc<std::sync::atomic::AtomicU64>,
320}
321
322/// Refusal labels passed to `observe_route_open_refusal` that mean the target
323/// module is not serving right now, and so open or extend an outage in the
324/// route outage tracker. Every one of them is only reachable after the target
325/// was found in the registry, which is what keeps an arbitrary client-chosen
326/// id from ever creating tracker state.
327///
328/// Deliberately absent: `not_registered` and `removed` (the id may be
329/// anything a client sent, and a removed module is gone on purpose),
330/// `protocol_none` (such a module never serves routes, so nothing is out),
331/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
332/// capability and admission-facts refusals (they refuse the caller, not a
333/// module outage), and `relay_reservation_failed` (its code ranges over
334/// capacity limits as well as a vanished connection). Capacity, breaker,
335/// relay-timeout and module-rejection refusals do not pass through that
336/// function at all; the breaker logs its own transitions.
337///
338/// The two not-serving refusals that bypass that function record themselves
339/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
340/// `required_capability_unprovided` is not tracked: the module itself is up,
341/// and the outage belongs to the missing provider.
342const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
343    "reloading",
344    "supervisor_not_live",
345    "registration_not_active",
346    "no_forwarding_connection",
347    "relay_send_failed",
348];
349
350/// Real channel-0 control handler for subc itself.
351#[derive(Clone)]
352pub struct ControlHandler {
353    registry: Arc<Registry>,
354    activity: Option<crate::activity::ActivityLedger>,
355    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
356    forwarding: Arc<ForwardingTable>,
357    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
358    supervisor: SupervisorHandle,
359    subc_capabilities: Arc<[String]>,
360    /// Daemon-wide route.bind relay budget. Used as the fallback when the
361    /// target module has no per-module override in
362    /// `route_bind_relay_timeouts`.
363    route_bind_relay_timeout: Duration,
364    /// Per-module route.bind relay budget overrides, keyed by module id. When
365    /// `handle_route_open` resolves the deadline for a target module, a
366    /// per-module entry wins over the daemon-wide value above.
367    route_bind_relay_timeouts: BTreeMap<String, Duration>,
368    /// Per-target-module bind-relay breaker state. Shared with the forwarding
369    /// table, which is where a new module connection resets it.
370    route_bind_breakers: RouteBindBreakers,
371    /// Live relay admissions keyed by target module. Shared through the
372    /// forwarding table so cloned or separately built handlers enforce one cap.
373    route_bind_concurrency: RouteBindConcurrency,
374    /// Start and end of each module's not-serving period as seen by
375    /// `route.open`, so an outage gets one line at each edge instead of only
376    /// the per-refusal INFO lines. Taken from the forwarding table, so every
377    /// handler built over one table shares it.
378    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
379    /// Consecutive relay timeouts that open a module's breaker.
380    route_bind_breaker_threshold: u32,
381    /// How long a breaker stays open before one probe is admitted.
382    route_bind_breaker_cooldown: Duration,
383    health_probe_timeout: Duration,
384    /// Central storage policy. When set, each registering module receives its
385    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
386    storage_config: Option<crate::daemon_config::StorageConfig>,
387    /// The machine id established at boot, served on every HELLO_ACK and on
388    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
389    /// changes the file, never this value. `None` serves no id.
390    machine_id: Option<crate::machine_id::MachineId>,
391    admission_facts_carrier_module_id: Option<String>,
392    admission_facts_targets: Option<Vec<String>>,
393    /// Scope records with their sync authorities and tombstones; see
394    /// `crate::scopes`. Shared by clones of this handler, so every connection
395    /// reads and writes one table.
396    scopes: Arc<RwLock<ScopeTable>>,
397    /// Wakes the expiry loop when an accepted scope change may have added a
398    /// deadline, so the loop can wait without ticking while no live scope has
399    /// one. A permit is stored when nothing waits, so no wake is lost.
400    scope_deadline_added: Arc<tokio::sync::Notify>,
401    /// The configured `scope_authority_owners`, kept so a rescan can report a
402    /// changed value as needing a daemon restart; rescan never applies it.
403    scope_authority_owners: Vec<String>,
404    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
405    privileged_operator_prompt: PrivilegedOperatorPrompt,
406    /// The local and peer address of each accepted TCP client connection, read
407    /// from the daemon's own socket, so a privileged request can name its caller.
408    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
409    #[cfg(test)]
410    caller_lookup: Option<TestCallerLookup>,
411    /// The launch nonce each module connection presented at HELLO, which is how
412    /// a `scope.sync` is matched to the owner's current launch.
413    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
414    rescan: Option<SupervisorRescanContext>,
415    connected_clients: ConnectedClients,
416    counters: DaemonCounters,
417    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
418    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
419    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
420    /// their usual severity (see [`log_requirement_events`]). A handler built
421    /// without one has a flag that is never set.
422    daemon_shutdown: DaemonShutdownFlag,
423    daemon_provenance: DaemonProvenanceFacts,
424    #[cfg(test)]
425    control_dispatch_delay: Option<Duration>,
426    #[cfg(test)]
427    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
428    /// Holds `route.open` after the route.bind is queued to the module and
429    /// before it is marked enqueued, so a test can make the module's reply (or
430    /// a client teardown) land exactly inside that window.
431    #[cfg(test)]
432    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
433    /// Holds a HELLO after its registry entry is reserved and before its
434    /// forwarding endpoint is installed, so a test can read the catalog and
435    /// open routes while the module is registered but not yet routable. A
436    /// blocking barrier because the HELLO handler is synchronous; the test
437    /// runs the HELLO on its own thread.
438    #[cfg(test)]
439    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
440}
441
442impl fmt::Debug for ControlHandler {
443    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
444        f.debug_struct("ControlHandler")
445            .field("registry", &self.registry)
446            .field("forwarding", &self.forwarding)
447            .field("process_liveness", &self.process_liveness.is_some())
448            .field("supervisor", &self.supervisor)
449            .field("subc_capabilities", &self.subc_capabilities)
450            .finish()
451    }
452}
453
454struct RouteOpenRequest {
455    target: RouteTarget,
456    identity: BindIdentity,
457    consumer_identity: Option<ConsumerIdentity>,
458    consumer_capabilities: Option<Vec<String>>,
459    role_versions: Option<BTreeMap<String, String>>,
460    admission_facts: Option<serde_json::Value>,
461    scope: Option<ScopeSelector>,
462}
463
464#[derive(Clone, Copy)]
465enum RouteOpenPhase {
466    Admission,
467    ProjectRoot,
468    EgressReservation,
469    BindSend,
470    BindAckWait,
471}
472
473/// Store elapsed time for each route-open phase, including attempts that end
474/// in an early refusal or backend error. Admission also includes rechecks and
475/// commit bookkeeping performed between the other timed phases.
476struct RouteOpenTimings {
477    module_id: String,
478    principal: String,
479    admission: Duration,
480    project_root: Duration,
481    egress_reservation: Duration,
482    bind_send: Duration,
483    bind_ack_wait: Duration,
484    phase: Option<RouteOpenPhase>,
485    phase_started_at: Instant,
486}
487
488impl RouteOpenTimings {
489    fn new() -> Self {
490        Self {
491            module_id: String::new(),
492            // Refusals before attestation must not claim a verified principal.
493            principal: "unresolved".to_string(),
494            admission: Duration::ZERO,
495            project_root: Duration::ZERO,
496            egress_reservation: Duration::ZERO,
497            bind_send: Duration::ZERO,
498            bind_ack_wait: Duration::ZERO,
499            phase: Some(RouteOpenPhase::Admission),
500            phase_started_at: Instant::now(),
501        }
502    }
503
504    fn finish(&mut self) {
505        let elapsed = self.phase_started_at.elapsed();
506        match self.phase.take() {
507            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
508            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
509            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
510            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
511            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
512            None => {}
513        }
514    }
515
516    fn enter(&mut self, phase: RouteOpenPhase) {
517        self.finish();
518        self.phase = Some(phase);
519        self.phase_started_at = Instant::now();
520    }
521}
522
523struct RouteBindReservationGuard {
524    forwarding: Arc<ForwardingTable>,
525    endpoint: ModuleEndpointId,
526    relay_corr: u64,
527    armed: bool,
528}
529
530struct ModuleControlRpcGuard {
531    forwarding: Arc<ForwardingTable>,
532    endpoint: ModuleEndpointId,
533    corr: u64,
534    armed: bool,
535}
536
537impl ModuleControlRpcGuard {
538    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
539        Self {
540            forwarding,
541            endpoint,
542            corr,
543            armed: true,
544        }
545    }
546
547    fn disarm(&mut self) {
548        self.armed = false;
549    }
550}
551
552impl Drop for ModuleControlRpcGuard {
553    fn drop(&mut self) {
554        if self.armed {
555            let _ = self
556                .forwarding
557                .cancel_module_control_rpc(self.endpoint, self.corr);
558        }
559    }
560}
561
562impl RouteBindReservationGuard {
563    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
564        Self {
565            forwarding,
566            endpoint,
567            relay_corr,
568            armed: true,
569        }
570    }
571
572    fn release_and_disarm(&mut self) {
573        if !self.armed {
574            return;
575        }
576        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
577            self.endpoint,
578            self.relay_corr,
579            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
580        ) {
581            send_goodbye_target_best_effort(
582                &self.forwarding.counters(),
583                &target,
584                "canceled route.bind",
585            );
586        }
587        self.armed = false;
588    }
589
590    fn disarm(&mut self) {
591        self.armed = false;
592    }
593}
594
595impl Drop for RouteBindReservationGuard {
596    fn drop(&mut self) {
597        self.release_and_disarm();
598    }
599}
600
601/// Per-target-module circuit breaker around the `route.bind` relay.
602///
603/// The connection reader is serial per connection, so a module whose `on_bind`
604/// sits on the ack blocks every LATER frame on the connections that call it,
605/// including calls to unrelated modules. This does not make any module's bind
606/// fast; it stops the daemon paying the full budget again and again for a
607/// condition it has already observed.
608///
609/// State is keyed by TARGET MODULE and shared by every connection: a wedged
610/// module wedges everyone, so what one connection learned should protect the
611/// rest.
612///
613/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
614/// relay to that module has actually timed out, and is removed again when a
615/// relay is accepted or the module reconnects, so it cannot grow with traffic
616/// or with modules that behave.
617///
618/// # Why a `std` mutex here is not the head-of-line defect again
619///
620/// Acquisition never awaits. The critical section is a hash lookup plus a few
621/// integer updates, with no I/O and no `.await` inside it, so a reader task
622/// cannot be descheduled behind it the way it can behind
623/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
624/// primitive, held for the same kind of work, as the refusal counter this very
625/// path already increments.
626///
627/// It is also NOT on the data-plane splice path: only `route.open` and module
628/// registration touch it, so bound-route frames gain no state check and no
629/// contention.
630#[derive(Debug, Clone, Default)]
631pub(crate) struct RouteBindBreakers {
632    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
633}
634
635#[derive(Debug, Clone, Default)]
636pub(crate) struct RouteBindConcurrency {
637    modules: Arc<Mutex<HashMap<String, usize>>>,
638}
639
640struct RouteBindConcurrencyGuard {
641    concurrency: RouteBindConcurrency,
642    module_id: String,
643}
644
645impl RouteBindConcurrency {
646    /// Admit without waiting. Waiting here would move the bind stall from the
647    /// module reply to a semaphore and restore reader head-of-line blocking.
648    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
649        let mut modules = self
650            .modules
651            .lock()
652            .expect("route.bind concurrency mutex poisoned");
653        let in_flight = modules.entry(module_id.to_string()).or_default();
654        if *in_flight >= limit {
655            return Err(*in_flight);
656        }
657        *in_flight += 1;
658        Ok(RouteBindConcurrencyGuard {
659            concurrency: self.clone(),
660            module_id: module_id.to_string(),
661        })
662    }
663}
664
665impl Drop for RouteBindConcurrencyGuard {
666    fn drop(&mut self) {
667        let mut modules = self
668            .concurrency
669            .modules
670            .lock()
671            .expect("route.bind concurrency mutex poisoned");
672        let remove = {
673            let in_flight = modules
674                .get_mut(&self.module_id)
675                .expect("admitted route.bind has a concurrency entry");
676            *in_flight -= 1;
677            *in_flight == 0
678        };
679        if remove {
680            modules.remove(&self.module_id);
681        }
682    }
683}
684
685#[derive(Debug, Default)]
686struct ModuleBreakerState {
687    /// Relay timeouts observed with no accepted relay in between.
688    consecutive_timeouts: u32,
689    /// `Some` while the breaker is open: the instant the cooldown expires and
690    /// the next arrival may probe. `None` means closed.
691    cooldown_until: Option<Instant>,
692    /// A half-open probe has been admitted and has not settled yet. This is
693    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
694    /// that read the cooldown, so concurrent opens arriving at the moment the
695    /// cooldown expires cannot all decide that they are the probe.
696    probe_in_flight: Option<Arc<()>>,
697}
698
699/// What the breaker decided for one `route.open`, before any relay work.
700enum RouteBindAdmission<'a> {
701    Admitted {
702        guard: RouteBindBreakerGuard<'a>,
703        /// This open is the single half-open probe, so the transition is worth
704        /// one log line.
705        probe: bool,
706    },
707    Refused {
708        consecutive_timeouts: u32,
709        /// What is left of the cooldown. Zero when the refusal is because the
710        /// one probe is already in flight rather than because the cooldown has
711        /// not elapsed.
712        retry_in: Duration,
713        probe_in_flight: bool,
714    },
715}
716
717/// An outstanding admission, which must be told how its relay settled.
718///
719/// `Drop` settles it as inconclusive, so an early return between admission and
720/// the relay -- or the whole handler being cancelled when the client
721/// disconnects -- releases a half-open probe slot instead of leaving the
722/// breaker wedged half-open with no further probes.
723struct RouteBindBreakerGuard<'a> {
724    breakers: RouteBindBreakers,
725    module_id: &'a str,
726    probe_token: Option<Arc<()>>,
727    settled: bool,
728}
729
730impl RouteBindBreakerGuard<'_> {
731    /// The module answered within the budget and took the bind. THE ONLY
732    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
733    /// breaker, which is a transition worth logging.
734    fn record_accepted(&mut self) -> bool {
735        self.settled = true;
736        self.breakers.record_accepted(self.module_id)
737    }
738
739    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
740    /// COUNTS TOWARD OPENING.
741    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
742        self.settled = true;
743        self.breakers.record_timeout(
744            self.module_id,
745            self.probe_token.as_ref(),
746            threshold,
747            cooldown,
748        )
749    }
750
751    /// Everything else: the module REJECTED the bind, its connection went away
752    /// mid-relay, or the waiter was cancelled.
753    ///
754    /// None of these is evidence that a module is slow, and each already has
755    /// its own refusal with its own code. A module that rejects a bind in
756    /// microseconds is healthy and must never be convicted for it; a module
757    /// that died has said nothing about the module that replaces it. So these
758    /// neither increment nor reset the count -- they only release a probe slot.
759    fn record_inconclusive(&mut self) {
760        self.settled = true;
761        self.breakers
762            .record_inconclusive(self.module_id, self.probe_token.as_ref());
763    }
764}
765
766impl Drop for RouteBindBreakerGuard<'_> {
767    fn drop(&mut self) {
768        if !self.settled {
769            self.breakers
770                .record_inconclusive(self.module_id, self.probe_token.as_ref());
771        }
772    }
773}
774
775/// The breaker moved to open, reported so the caller can log it outside the
776/// lock. Opening is rare and load-bearing; the refusals that follow are
777/// frequent and are counted rather than logged.
778struct BreakerOpened {
779    consecutive_timeouts: u32,
780    /// True when a failed probe re-opened an already-open breaker, which reads
781    /// very differently in a log from a first opening.
782    reopened_after_probe: bool,
783}
784
785impl RouteBindBreakers {
786    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
787        self.modules
788            .lock()
789            .expect("route.bind breaker mutex poisoned")
790    }
791
792    /// Decide whether this `route.open` may attempt its relay. Takes the map
793    /// lock and nothing else, and never awaits.
794    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
795        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
796            probe: probe_token.is_some(),
797            guard: RouteBindBreakerGuard {
798                breakers: self.clone(),
799                module_id,
800                probe_token,
801                settled: false,
802            },
803        };
804
805        let mut modules = self.lock();
806        let Some(state) = modules.get_mut(module_id) else {
807            return admitted(None);
808        };
809        let Some(cooldown_until) = state.cooldown_until else {
810            return admitted(None);
811        };
812        if state.probe_in_flight.is_some() {
813            return RouteBindAdmission::Refused {
814                consecutive_timeouts: state.consecutive_timeouts,
815                retry_in: Duration::ZERO,
816                probe_in_flight: true,
817            };
818        }
819        let now = Instant::now();
820        if now < cooldown_until {
821            return RouteBindAdmission::Refused {
822                consecutive_timeouts: state.consecutive_timeouts,
823                retry_in: cooldown_until - now,
824                probe_in_flight: false,
825            };
826        }
827        let token = Arc::new(());
828        state.probe_in_flight = Some(Arc::clone(&token));
829        admitted(Some(token))
830    }
831
832    fn record_accepted(&self, module_id: &str) -> bool {
833        self.lock()
834            .remove(module_id)
835            .is_some_and(|state| state.cooldown_until.is_some())
836    }
837
838    fn record_timeout(
839        &self,
840        module_id: &str,
841        probe_token: Option<&Arc<()>>,
842        threshold: u32,
843        cooldown: Duration,
844    ) -> Option<BreakerOpened> {
845        let mut modules = self.lock();
846        let state = modules.entry(module_id.to_string()).or_default();
847        let was_open = state.cooldown_until.is_some();
848        let was_probe = Self::owns_probe(state, probe_token);
849        if was_probe {
850            state.probe_in_flight = None;
851        }
852        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
853        if state.consecutive_timeouts < threshold {
854            return None;
855        }
856        state.cooldown_until = Some(Instant::now() + cooldown);
857        Some(BreakerOpened {
858            consecutive_timeouts: state.consecutive_timeouts,
859            reopened_after_probe: was_open && was_probe,
860        })
861    }
862
863    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
864        // A relay can finish after the breaker was reset or after it opened
865        // again and started a new probe. Only the guard whose token matches the
866        // active probe may release it, so a late relay never frees a newer probe.
867        state
868            .probe_in_flight
869            .as_ref()
870            .zip(token)
871            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
872    }
873
874    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
875        if let Some(state) = self.lock().get_mut(module_id) {
876            if Self::owns_probe(state, probe_token) {
877                state.probe_in_flight = None;
878            }
879        }
880    }
881
882    /// Discard what was learned about a module, because the process it was
883    /// learned about is gone. Returns the discarded count when it was non-zero.
884    ///
885    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
886    /// `module_id` is a configuration identity that outlives any particular
887    /// child; what the breaker observed was the process behind the module
888    /// connection of the moment. When a new connection registers under that id
889    /// the verdict's subject no longer exists, so the verdict is stale by
890    /// construction rather than merely likely to be wrong. Keeping it would
891    /// apply a dead process's record to a live one, which is the same defect
892    /// class this breaker exists to stop the daemon committing.
893    ///
894    /// A half-open probe in flight is discarded with the rest: it was a
895    /// question about the old process.
896    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
897        self.lock()
898            .remove(module_id)
899            .map(|state| state.consecutive_timeouts)
900            .filter(|discarded| *discarded > 0)
901    }
902
903    /// Open breakers, for the `server.describe` counters object. `None` when
904    /// none is open, so the key stays absent rather than present-and-empty.
905    ///
906    /// This is the operator's answer to "is this module refusing instantly or
907    /// is it fine?", which look identical from a client that retries and then
908    /// succeeds.
909    fn open_snapshot(&self) -> Option<serde_json::Value> {
910        let now = Instant::now();
911        let modules = self.lock();
912        let open = modules
913            .iter()
914            .filter_map(|(module_id, state)| {
915                let cooldown_until = state.cooldown_until?;
916                Some((
917                    module_id.clone(),
918                    serde_json::json!({
919                        "consecutive_timeouts": state.consecutive_timeouts,
920                        "cooldown_remaining_ms":
921                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
922                        "probe_in_flight": state.probe_in_flight.is_some(),
923                    }),
924                ))
925            })
926            .collect::<serde_json::Map<String, serde_json::Value>>();
927        (!open.is_empty()).then_some(serde_json::Value::Object(open))
928    }
929}
930
931impl ControlHandler {
932    pub fn new(registry: Arc<Registry>) -> Self {
933        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
934    }
935
936    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
937        let counters = forwarding.counters();
938        // Taken from the forwarding table rather than created here, so that the
939        // breaker a `route.open` consults is the same one a module's
940        // registration resets, however many handlers are built over one table.
941        let route_bind_breakers = forwarding.route_bind_breakers();
942        let route_bind_concurrency = forwarding.route_bind_concurrency();
943        let route_outages = forwarding.route_outages();
944        Self {
945            registry,
946            activity: None,
947            activity_streams: Arc::new(Mutex::new(HashMap::new())),
948            forwarding,
949            process_liveness: None,
950            supervisor: SupervisorHandle::new(),
951            subc_capabilities: Arc::from([
952                CAP_MANIFEST_REGISTRATION.to_string(),
953                CAP_CHANNEL_LIFECYCLE.to_string(),
954                CAP_PING_PONG.to_string(),
955                CAP_SESSION_ATTACH.to_string(),
956                CAP_ADMISSION_FACTS_RELAY.to_string(),
957                CAP_SCOPES_V1.to_string(),
958                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
959                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
961                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
962                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
963                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
964                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
965                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
966                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
967                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
968                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
969            ]),
970            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
971            route_bind_relay_timeouts: BTreeMap::new(),
972            route_bind_breakers,
973            route_bind_concurrency,
974            route_outages,
975            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
976            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
977            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
978            storage_config: None,
979            machine_id: None,
980            admission_facts_carrier_module_id: None,
981            admission_facts_targets: None,
982            scopes: Arc::new(RwLock::new(ScopeTable::new(
983                crate::daemon_config::default_scope_authority_owners(),
984            ))),
985            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
986            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
987            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
988            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
989            #[cfg(test)]
990            caller_lookup: None,
991            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
992            rescan: None,
993            connected_clients: ConnectedClients::new(),
994            counters,
995            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
996            daemon_shutdown: DaemonShutdownFlag::default(),
997            daemon_provenance: DaemonProvenanceFacts::default(),
998            #[cfg(test)]
999            control_dispatch_delay: None,
1000            #[cfg(test)]
1001            provenance_probe_override: None,
1002            #[cfg(test)]
1003            route_bind_relay_sent_hold: None,
1004            #[cfg(test)]
1005            hello_registry_to_forwarding_hold: None,
1006        }
1007    }
1008
1009    /// Set the central storage policy: registering modules then receive their
1010    /// resolved storage descriptor in HELLO_ACK.
1011    pub fn with_storage_config(
1012        mut self,
1013        storage_config: Option<crate::daemon_config::StorageConfig>,
1014    ) -> Self {
1015        self.storage_config = storage_config;
1016        self
1017    }
1018
1019    /// Open the ledger on its own writer thread; an unavailable sink does not
1020    /// prevent the daemon from serving routing or supervisor requests.
1021    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1022        self.activity = Some(crate::activity::ActivityLedger::open(
1023            run_dir.join("activity.jsonl"),
1024            incarnation,
1025        ));
1026        self.supervisor.set_activity(self.activity.clone());
1027        self
1028    }
1029
1030    /// Set the machine id served to every registering module (HELLO_ACK) and on
1031    /// `server.describe`.
1032    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1033        self.machine_id = machine_id;
1034        self
1035    }
1036
1037    /// Configure the exact reserved module and target ids permitted to relay
1038    /// opaque admission facts. Config-file loading validates this authority;
1039    /// this builder keeps the same policy available to embedded test daemons.
1040    pub fn with_admission_facts_config(
1041        mut self,
1042        carrier_module_id: Option<String>,
1043        targets: Option<Vec<String>>,
1044    ) -> Self {
1045        self.admission_facts_carrier_module_id = carrier_module_id;
1046        self.admission_facts_targets = targets;
1047        self
1048    }
1049
1050    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1051    /// Replaces the scope table with an empty one under the new list, so call it
1052    /// while building the handler, before any module can sync.
1053    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1054        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1055        self.scope_authority_owners = owners;
1056        self
1057    }
1058
1059    /// Set the startup policy before installing the rescan context or serving requests.
1060    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1061        self.privileged_operator_prompt = mode;
1062        self
1063    }
1064
1065    /// Override the route.bind relay timeout. Used by tests that assert the
1066    /// timeout path so they don't block on the production-safe default.
1067    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1068        self.route_bind_relay_timeout = timeout;
1069        self
1070    }
1071
1072    /// Install per-module route.bind relay budget overrides. A module id
1073    /// listed here wins over the daemon-wide default set via
1074    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1075    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1076    /// the same `Duration` the bind path will use.
1077    pub fn with_route_bind_relay_timeouts(
1078        mut self,
1079        timeouts: impl IntoIterator<Item = (String, Duration)>,
1080    ) -> Self {
1081        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1082        self
1083    }
1084
1085    /// Resolve the route.bind relay budget for a specific target module id.
1086    /// Per-module overrides win; the daemon-wide value (set via
1087    /// `with_route_bind_relay_timeout` or the built-in default) is the
1088    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1089    /// the same resolution `handle_route_open` will use.
1090    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1091        self.route_bind_relay_timeouts
1092            .get(module_id)
1093            .copied()
1094            .unwrap_or(self.route_bind_relay_timeout)
1095    }
1096
1097    /// Override the per-module bind-relay breaker policy.
1098    ///
1099    /// Used by tests, which cannot spend three production budgets opening a
1100    /// breaker or twenty seconds waiting for its cooldown. The production
1101    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1102    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1103    /// reasoning for the numbers.
1104    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1105        self.route_bind_breaker_threshold = threshold.max(1);
1106        self.route_bind_breaker_cooldown = cooldown;
1107        self
1108    }
1109
1110    #[cfg(test)]
1111    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1112        self.health_probe_timeout = timeout;
1113        self
1114    }
1115
1116    #[cfg(test)]
1117    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1118        self.control_dispatch_delay = Some(delay);
1119        self
1120    }
1121
1122    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1123    /// announce it is parked, once to be released. Build the barrier for two
1124    /// parties, the test being the other one.
1125    #[cfg(test)]
1126    pub(crate) fn with_route_bind_relay_sent_hold(
1127        mut self,
1128        hold: Arc<tokio::sync::Barrier>,
1129    ) -> Self {
1130        self.route_bind_relay_sent_hold = Some(hold);
1131        self
1132    }
1133
1134    /// A HELLO waits on `hold` twice between reserving its registry entry and
1135    /// installing its forwarding endpoint: once to announce it is parked, once
1136    /// to be released. Build the barrier for two parties, the test being the
1137    /// other one, and run the HELLO on a thread of its own.
1138    #[cfg(test)]
1139    pub(crate) fn with_hello_registry_to_forwarding_hold(
1140        mut self,
1141        hold: Arc<std::sync::Barrier>,
1142    ) -> Self {
1143        self.hello_registry_to_forwarding_hold = Some(hold);
1144        self
1145    }
1146
1147    #[cfg(test)]
1148    fn hold_between_registry_and_forwarding(&self) {
1149        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1150            hold.wait();
1151            hold.wait();
1152        }
1153    }
1154
1155    pub fn with_process_liveness(
1156        mut self,
1157        process_liveness: Arc<dyn ModuleProcessLiveness>,
1158    ) -> Self {
1159        self.process_liveness = Some(process_liveness);
1160        self
1161    }
1162
1163    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1164        self.supervisor = supervisor;
1165        self.supervisor.set_activity(self.activity.clone());
1166        self
1167    }
1168
1169    pub fn with_daemon_provenance(
1170        mut self,
1171        pid: u32,
1172        started_at_ms: u64,
1173        executable_path: Option<PathBuf>,
1174        build_git_sha: Option<String>,
1175        build_lock_digest: Option<String>,
1176    ) -> Self {
1177        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1178        let process_start_time = process_start_time(pid);
1179        self.daemon_provenance = DaemonProvenanceFacts {
1180            build: DaemonBuildProvenance {
1181                build_git_sha,
1182                build_lock_digest,
1183            },
1184            pid: Some(pid),
1185            started_at_ms: Some(started_at_ms),
1186            start_clock: None,
1187            executable_path,
1188            executable_identity,
1189            process_start_time,
1190            probe: ExecutableIdentityProbe::default(),
1191        };
1192        self
1193    }
1194
1195    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1196        self.daemon_provenance.start_clock = Some(clock);
1197        self
1198    }
1199
1200    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1201    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1202        self.scopes
1203            .write()
1204            .unwrap_or_else(|poisoned| poisoned.into_inner())
1205            .set_wall_clock(Arc::new(clock));
1206        self
1207    }
1208
1209    #[cfg(test)]
1210    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1211        self.provenance_probe_override = Some(result);
1212        self
1213    }
1214
1215    /// Install the configured module set and its reserved capability bindings.
1216    /// Bindings are configuration-scoped and may point at a provider that has not
1217    /// been installed yet, so this does not require the bound module to exist.
1218    pub fn with_capability_config(
1219        self,
1220        modules: impl IntoIterator<Item = (String, bool)>,
1221        reserved_capabilities: BTreeMap<String, String>,
1222    ) -> Self {
1223        self.capability_evaluator
1224            .configure(modules, reserved_capabilities);
1225        self
1226    }
1227
1228    /// Share the supervisor's daemon-shutdown flag with this handler, so
1229    /// requirement changes after the daemon has begun its announced shutdown
1230    /// are not reported as faults.
1231    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1232        self.daemon_shutdown = flag;
1233        self
1234    }
1235
1236    pub fn with_supervisor_rescan(
1237        mut self,
1238        supervisor: Supervisor,
1239        config_path: impl Into<PathBuf>,
1240        configured_port: Option<u16>,
1241    ) -> Self {
1242        self.rescan = Some(SupervisorRescanContext {
1243            supervisor,
1244            config_path: config_path.into(),
1245            configured_port,
1246            storage_config: self.storage_config.clone(),
1247            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1248            admission_facts_targets: self.admission_facts_targets.clone(),
1249            scope_authority_owners: self.scope_authority_owners.clone(),
1250            privileged_operator_prompt: self.privileged_operator_prompt,
1251            generation: Arc::default(),
1252        });
1253        self
1254    }
1255
1256    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1257        self.connected_clients = connected_clients;
1258        self
1259    }
1260
1261    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1262        Arc::clone(&self.forwarding)
1263    }
1264
1265    pub(crate) fn counters(&self) -> DaemonCounters {
1266        self.counters.clone()
1267    }
1268
1269    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1270    /// requirement event without depending on an operator polling a status command.
1271    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1272        tokio::spawn(async move {
1273            loop {
1274                self.capability_evaluator
1275                    .wait_for_change_or_deadline()
1276                    .await;
1277                self.refresh_capability_requirements();
1278            }
1279        });
1280    }
1281
1282    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1283    /// deadline passed during sleep is enforced on the first tick after wake.
1284    /// While no live scope carries a deadline the loop waits for an accepted
1285    /// scope change instead of ticking, so a daemon without deadlines never
1286    /// wakes for expiry.
1287    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1288        tokio::spawn(async move {
1289            loop {
1290                let has_deadlines = self
1291                    .scopes
1292                    .read()
1293                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1294                    .has_deadlines();
1295                if !has_deadlines {
1296                    self.scope_deadline_added.notified().await;
1297                    continue;
1298                }
1299                tokio::time::sleep(Duration::from_secs(1)).await;
1300                if let Err(error) = self.sweep_expired_scopes() {
1301                    warn!(%error, "scope expiry sweep failed");
1302                }
1303            }
1304        });
1305    }
1306
1307    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1308        let mut table = self
1309            .scopes
1310            .write()
1311            .unwrap_or_else(|poisoned| poisoned.into_inner());
1312        let swept = table.sweep_expired();
1313        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1314        drop(table);
1315        self.close_scope_drained_routes(drained);
1316        Ok(())
1317    }
1318
1319    fn publish_scope_changes(
1320        &self,
1321        changes: &[crate::scopes::ScopeTagChange],
1322        expired: &[crate::scopes::ScopeExpired],
1323    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1324        if changes.is_empty() {
1325            return Ok(Vec::new());
1326        }
1327        let drained = self
1328            .forwarding
1329            .publish_scope_changes(changes)
1330            .map_err(RouterError::Forwarding)?;
1331        let mut counts = HashMap::new();
1332        for route in &drained {
1333            *counts
1334                .entry((
1335                    route.scope.owner.as_str(),
1336                    route.scope.scope_ref.as_str(),
1337                    route.scope.tag.scope_epoch,
1338                ))
1339                .or_insert(0usize) += 1;
1340        }
1341        for scope in expired {
1342            let routes_closed = counts
1343                .get(&(
1344                    scope.owner.as_str(),
1345                    scope.scope_ref.as_str(),
1346                    scope.scope_epoch,
1347                ))
1348                .copied()
1349                .unwrap_or(0);
1350            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1351                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1352                routes_closed, "scope expired");
1353        }
1354        Ok(drained)
1355    }
1356
1357    fn runtime_capability_snapshot(
1358        &self,
1359    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1360        let runtime = self
1361            .supervisor
1362            .list()
1363            .into_iter()
1364            .map(|module| {
1365                let status = module.status().map_err(|err| {
1366                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1367                })?;
1368                Ok(RuntimeModule {
1369                    module_id: status.module_id,
1370                    state: status.state,
1371                    enabled: status.enabled,
1372                })
1373            })
1374            .collect::<Result<Vec<_>, RouterError>>()?;
1375        // Include every registration, published or not. The capability
1376        // evaluator counts a module as providing its capabilities as soon as it
1377        // has registered, even before `catalog.list` shows it, so a HELLO in
1378        // progress does not change other modules' required-capability
1379        // verdicts. See `Registry::list_registered_modules`.
1380        let registrations = self.registry.list_registered_modules().map_err(|err| {
1381            RouterError::backend(
1382                0,
1383                0,
1384                format!("failed to list capability registrations: {err}"),
1385            )
1386        })?;
1387        let registrations = registrations
1388            .into_iter()
1389            .map(|registration| RegisteredModule {
1390                module_id: registration.manifest.module_id,
1391                module_version: registration.manifest.module_version,
1392                capabilities: registration.manifest.capabilities,
1393            })
1394            .collect();
1395        Ok((runtime, registrations))
1396    }
1397
1398    /// The capability side effects of a module becoming the active registration
1399    /// for its id: cache its manifest (logging how it differs from the cached
1400    /// one), run the deny census when its declarations call for one, and recompute the
1401    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1402    /// candidate's does not, and the supervisor does it at promotion instead,
1403    /// through [`crate::supervise::SwapPromotionObserver`].
1404    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1405        let cached_registration = RegisteredModule {
1406            module_id: registration.manifest.module_id.clone(),
1407            module_version: registration.manifest.module_version.clone(),
1408            capabilities: registration.manifest.capabilities.clone(),
1409        };
1410        let change = self.capability_evaluator.record_hello(&cached_registration);
1411        log_manifest_change(&cached_registration.module_id, &change);
1412        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1413            self.enforce_capability_denies();
1414        }
1415        self.refresh_capability_requirements();
1416    }
1417
1418    /// Point the shared supervisor handle at this handler for swap promotions.
1419    /// Called wherever a handler is put behind the `Arc` the router serves, so
1420    /// it can be held weakly.
1421    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1422        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1423            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1424        self.supervisor.set_swap_promotion_observer(observer);
1425    }
1426
1427    pub fn refresh_capability_requirements(&self) {
1428        match self.runtime_capability_snapshot() {
1429            Ok((runtime, registrations)) => {
1430                log_requirement_events(
1431                    self.capability_evaluator
1432                        .evaluate_now(&runtime, &registrations),
1433                    &self.daemon_shutdown,
1434                );
1435            }
1436            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1437        }
1438    }
1439
1440    /// Reconcile only live, attested route bindings after a capability deny edge
1441    /// or target claim was added. This is deliberately a control-plane census:
1442    /// the opaque forwarding hot path must not grow a per-frame capability check.
1443    fn enforce_capability_denies(&self) {
1444        let registrations = match self.registry.list_registered_modules() {
1445            Ok(snapshot) => snapshot,
1446            Err(err) => {
1447                warn!(error = %err, "failed to read registrations for capability deny census");
1448                return;
1449            }
1450        };
1451        let manifests = registrations
1452            .into_iter()
1453            .map(|registration| {
1454                (
1455                    registration.manifest.module_id.clone(),
1456                    registration.manifest,
1457                )
1458            })
1459            .collect::<BTreeMap<_, _>>();
1460        let census = match self.forwarding.route_census(None) {
1461            Ok(census) => census,
1462            Err(err) => {
1463                warn!(error = %err, "failed to read route census for capability deny enforcement");
1464                return;
1465            }
1466        };
1467
1468        for (target_module_id, routes) in census {
1469            let Some(target_manifest) = manifests.get(&target_module_id) else {
1470                continue;
1471            };
1472            let mut closed_routes = Vec::new();
1473            let mut goodbyes = Vec::new();
1474            for route in routes {
1475                let Principal::Reserved {
1476                    module_id: opening_module_id,
1477                } = &route.principal
1478                else {
1479                    continue;
1480                };
1481                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1482                    continue;
1483                };
1484                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1485                    continue;
1486                };
1487
1488                match self.forwarding.release_client_route(
1489                    route.goodbye_target.connection_id,
1490                    route.goodbye_target.channel,
1491                    route.goodbye_target.epoch,
1492                ) {
1493                    Ok(RouteRelease::Removed(module_goodbye)) => {
1494                        warn!(
1495                            opening_module_id,
1496                            target_module_id,
1497                            capability,
1498                            "force-closing route because an attested capability deny edge now matches"
1499                        );
1500                        goodbyes.push(module_goodbye);
1501                        goodbyes.push(route.goodbye_target.clone());
1502                        closed_routes.push(route);
1503                    }
1504                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505                    Err(err) => warn!(
1506                        opening_module_id,
1507                        target_module_id,
1508                        capability,
1509                        error = %err,
1510                        "failed to force-close capability-denied route"
1511                    ),
1512                }
1513            }
1514
1515            if closed_routes.is_empty() {
1516                continue;
1517            }
1518            send_route_control_pushes(
1519                &self.forwarding,
1520                closed_routes,
1521                ClientControlPush::RouteClosed {
1522                    module_id: target_module_id,
1523                    channels: Vec::new(),
1524                    reason: RouteCloseReason::CapabilityDenied,
1525                    drained: false,
1526                    abandoned: 0,
1527                    excluded_subscriptions: 0,
1528                    terminal: Some(false),
1529                },
1530            );
1531            // route.closed records the reason; the route GOODBYE ends the
1532            // client's handle and settles any calls still pending on it.
1533            self.emit_route_goodbyes(goodbyes);
1534        }
1535    }
1536
1537    /// Why a registered module is not accepting new route binds, or `None` when
1538    /// it is. This is the module's effective readiness: its declared readiness
1539    /// first, then every `need: required` capability it declares evaluating to
1540    /// `provided`. `route.open` and `catalog.list` both read it here so the
1541    /// catalog never reports a module routable that `route.open` would refuse.
1542    fn not_ready_reason(
1543        &self,
1544        registration: &crate::registry::ModuleRegistration,
1545    ) -> Option<NotReadyReason> {
1546        if !registration.ready {
1547            return Some(NotReadyReason {
1548                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1549                capability: None,
1550            });
1551        }
1552        self.first_unprovided_required_capability(registration)
1553            .map(|capability| NotReadyReason {
1554                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1555                capability: Some(capability),
1556            })
1557    }
1558
1559    /// The lexicographically first capability this registration declares
1560    /// `need: required` whose evaluator verdict is not `provided`.
1561    ///
1562    /// The verdicts are the capability evaluator's own; nothing here decides
1563    /// what "provided" means. The evaluator counts a capability provided as
1564    /// soon as a module claiming it has REGISTERED, not once that module is
1565    /// ready. That distinction is what keeps two modules that require each
1566    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1567    /// is ready", each would wait for the other to become ready first and
1568    /// neither ever would. Do not tighten it to readiness.
1569    ///
1570    /// A required capability with no verdict at all means this registration's
1571    /// HELLO or catalog.update landed after the last recompute; recompute once
1572    /// rather than let a missing verdict read as either answer. If it is still
1573    /// missing (the recompute itself failed) the capability counts as
1574    /// unprovided: the refusal is retryable, and routing a module whose
1575    /// required provider is unknown is the outcome this check exists to stop.
1576    fn first_unprovided_required_capability(
1577        &self,
1578        registration: &crate::registry::ModuleRegistration,
1579    ) -> Option<String> {
1580        let required = registration
1581            .manifest
1582            .capabilities
1583            .iter()
1584            .flat_map(|declarations| declarations.requires.iter())
1585            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1586            .map(|requirement| requirement.capability.as_str())
1587            .collect::<BTreeSet<_>>();
1588        if required.is_empty() {
1589            return None;
1590        }
1591        let module_id = registration.manifest.module_id.as_str();
1592        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1593        if required
1594            .iter()
1595            .any(|capability| verdict(capability).is_none())
1596        {
1597            self.refresh_capability_requirements();
1598        }
1599        required
1600            .into_iter()
1601            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1602            .map(str::to_string)
1603    }
1604
1605    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1606        self.capability_evaluator
1607            .statuses()
1608            .into_iter()
1609            .map(capability_requirement_status)
1610            .collect()
1611    }
1612
1613    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1614    /// registration-release watch. The signal is what the supervisor waits on
1615    /// before spawning a replacement, so it must only fire once forwarding
1616    /// teardown is also done (see [`Self::cleanup_connection`] /
1617    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1618    /// admitted but its forwarding endpoint could not be installed.
1619    fn deregister_connection(
1620        &self,
1621        connection_id: ConnectionId,
1622        reason: RegistrationEndReason,
1623    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1624        self.registry
1625            .deregister_connection_with_reason(connection_id, reason)
1626    }
1627
1628    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1629        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1630            return None;
1631        }
1632        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1633            parse_client_control_request(&frame.body)
1634        else {
1635            return None;
1636        };
1637        Some(target_module_id(&target).to_string())
1638    }
1639
1640    pub(crate) fn route_open_capacity_refusal(
1641        &self,
1642        ctx: &RouteCtx,
1643        frame: &Frame,
1644        target_module_id: &str,
1645        in_flight: usize,
1646        limit: usize,
1647    ) -> Result<Frame, RouterError> {
1648        // Sent before the request reaches `handle_route_open`, so the caller
1649        // fields come from the raw frame here.
1650        let caller = self.route_open_caller_from_frame(frame);
1651        self.route_open_admission_refusal_frame(
1652            ctx,
1653            &caller,
1654            frame,
1655            target_module_id,
1656            "open_admission_full",
1657            (in_flight, limit),
1658            format!(
1659                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1660            ),
1661        )
1662    }
1663
1664    fn route_open_target_capacity_refusal(
1665        &self,
1666        ctx: &RouteCtx,
1667        caller: &RouteOpenCaller,
1668        frame: &Frame,
1669        target_module_id: &str,
1670        in_flight: usize,
1671    ) -> Result<Frame, RouterError> {
1672        self.route_open_admission_refusal_frame(
1673            ctx,
1674            caller,
1675            frame,
1676            target_module_id,
1677            "target_binds_full",
1678            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1679            format!(
1680                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1681            ),
1682        )
1683    }
1684
1685    /// Admission pressure clears as existing binds settle, so its refusal must
1686    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1687    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1688    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1689    /// cannot currently reach its target; `module_timeout` would falsely claim
1690    /// that a wait expired. A new, cleaner code would be terminal to deployed
1691    /// clients, so it requires a client-tolerance rollout before daemon emission.
1692    #[allow(clippy::too_many_arguments)]
1693    fn route_open_admission_refusal_frame(
1694        &self,
1695        ctx: &RouteCtx,
1696        caller: &RouteOpenCaller,
1697        frame: &Frame,
1698        target_module_id: &str,
1699        reason: &'static str,
1700        (in_flight, limit): (usize, usize),
1701        message: impl Into<String>,
1702    ) -> Result<Frame, RouterError> {
1703        let code = error_codes::TARGET_UNAVAILABLE;
1704        self.counters.increment_route_open_refused(code);
1705        info!(
1706            target: "control",
1707            code,
1708            reason,
1709            module_id = ?target_module_id,
1710            connection_id = ctx.connection_id.get(),
1711            principal = %caller.principal,
1712            scope = %caller.scope,
1713            harness = %caller.harness,
1714            in_flight,
1715            limit,
1716            "route.open refused"
1717        );
1718        control_error_frame(frame, code, message.into())
1719    }
1720
1721    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1722    ///
1723    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1724    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1725    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1726    #[cfg(test)]
1727    pub fn handle_control(
1728        &self,
1729        connection_id: ConnectionId,
1730        frame: Frame,
1731    ) -> Result<Vec<Frame>, RouterError> {
1732        match frame.header.ty {
1733            FrameType::Ping => Ok(vec![pong(&frame)?]),
1734            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1735            FrameType::Goodbye => self.handle_goodbye(connection_id),
1736            ty => Ok(vec![control_error_frame(
1737                &frame,
1738                "unsupported_control_frame",
1739                format!("unsupported channel-0 frame {ty:?}"),
1740            )?]),
1741        }
1742    }
1743
1744    pub async fn handle_control_frame(
1745        &self,
1746        ctx: &RouteCtx,
1747        frame: Frame,
1748    ) -> Result<Vec<Frame>, RouterError> {
1749        self.handle_control_frame_timed(ctx, frame, None).await
1750    }
1751
1752    pub(crate) fn record_caller_endpoints(
1753        &self,
1754        connection: ConnectionId,
1755        endpoints: Option<(SocketAddr, SocketAddr)>,
1756    ) {
1757        let mut callers = self
1758            .caller_endpoints
1759            .lock()
1760            .unwrap_or_else(|p| p.into_inner());
1761        callers.remove(&connection);
1762        if let Some(endpoints) = endpoints {
1763            callers.insert(connection, endpoints);
1764        }
1765    }
1766
1767    #[cfg(test)]
1768    pub(crate) fn with_caller_lookup_for_test(
1769        mut self,
1770        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1771    ) -> Self {
1772        self.caller_lookup = Some(Arc::new(lookup));
1773        self
1774    }
1775
1776    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1777        let endpoints = self
1778            .caller_endpoints
1779            .lock()
1780            .unwrap_or_else(|p| p.into_inner())
1781            .get(&connection)
1782            .copied();
1783        #[cfg(test)]
1784        let lookup = self.caller_lookup.clone();
1785        tokio::task::spawn_blocking(move || {
1786            #[cfg(test)]
1787            if let Some(lookup) = lookup {
1788                return lookup(endpoints);
1789            }
1790            endpoints
1791                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1792                .unwrap_or_default()
1793        })
1794        .await
1795        .unwrap_or_default()
1796    }
1797
1798    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1799        frame.header.channel == 0
1800            && frame.header.ty == FrameType::Request
1801            && self
1802                .forwarding
1803                .module_endpoint_for_connection(ctx.connection_id)
1804                .is_ok_and(|endpoint| endpoint.is_none())
1805            && parse_client_control_request(&frame.body)
1806                .ok()
1807                .is_some_and(|request| privileged_operation(&request).is_some())
1808    }
1809
1810    pub(crate) async fn handle_control_frame_timed(
1811        &self,
1812        ctx: &RouteCtx,
1813        frame: Frame,
1814        dispatch_started_at: Option<StdInstant>,
1815    ) -> Result<Vec<Frame>, RouterError> {
1816        match frame.header.ty {
1817            FrameType::Ping => Ok(vec![pong(&frame)?]),
1818            FrameType::Hello => {
1819                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1820            }
1821            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1822            FrameType::Cancel => {
1823                // A Cancel on channel 0 names a waiting operator.confirm or a
1824                // subscription. Successful cancellation has no ordinary reply.
1825                if self
1826                    .forwarding
1827                    .operator_confirms()
1828                    .cancel(ctx.connection_id, frame.header.corr)
1829                    || self
1830                        .supervisor
1831                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1832                    || self.cancel_activity_subscription(ctx, &frame)?
1833                {
1834                    Ok(Vec::new())
1835                } else {
1836                    Ok(vec![control_error_frame(
1837                        &frame,
1838                        "unknown_subscription",
1839                        "no subscription has this correlation id",
1840                    )?])
1841                }
1842            }
1843            FrameType::Request => {
1844                // This additive operation is not part of the existing exhaustive
1845                // module-control enum. Probe the op before decoding that enum.
1846                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1847                if op
1848                    .as_ref()
1849                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1850                {
1851                    return self.handle_activity_record(ctx, frame).await;
1852                }
1853                if op.as_ref().is_some_and(|probe| {
1854                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1855                }) {
1856                    return self.handle_activity_read(ctx, frame).await;
1857                }
1858                if op
1859                    .as_ref()
1860                    .is_some_and(|probe| probe.op == "operator.confirm")
1861                {
1862                    return self.handle_operator_confirm(ctx, frame);
1863                }
1864                if op
1865                    .as_ref()
1866                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1867                {
1868                    return self.handle_scope_served_by(ctx.connection_id, frame);
1869                }
1870                if self
1871                    .forwarding
1872                    .module_endpoint_for_connection(ctx.connection_id)
1873                    .map_err(RouterError::Forwarding)?
1874                    .is_some()
1875                {
1876                    if !is_known_module_request_op(&frame.body) {
1877                        return Ok(vec![control_error_frame(
1878                            &frame,
1879                            "unsupported_control_frame",
1880                            "module-originated channel-0 REQUEST is not supported",
1881                        )?]);
1882                    }
1883                    let request = match parse_module_control_request_from_module(&frame.body) {
1884                        Ok(request) => request,
1885                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1886                            return Ok(vec![control_error_frame(
1887                                &frame,
1888                                "unsupported_control_frame",
1889                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1890                            )?])
1891                        }
1892                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1893                            return Ok(vec![control_error_frame(
1894                                &frame,
1895                                "invalid_control_body",
1896                                format!("malformed module control body: {err}"),
1897                            )?])
1898                        }
1899                    };
1900                    let op = module_control_request_op(&request);
1901                    let corr = frame.header.corr;
1902                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1903                    let result =
1904                        self.handle_module_control_request(ctx.connection_id, frame, request);
1905                    log_slow_control_dispatch(
1906                        dispatch_started_at,
1907                        op,
1908                        ctx.connection_id,
1909                        corr,
1910                        None,
1911                    );
1912                    return result;
1913                }
1914
1915                if is_known_module_request_op(&frame.body) {
1916                    return Ok(vec![control_error_frame(
1917                        &frame,
1918                        "not_registered",
1919                        "catalog.update requires an active module registration owned by this connection",
1920                    )?]);
1921                }
1922
1923                let request = match parse_client_control_request(&frame.body) {
1924                    Ok(request) => request,
1925                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1926                        return Ok(vec![control_error_frame(
1927                            &frame,
1928                            "unknown_control_op",
1929                            format!("unknown client control op: {err}"),
1930                        )?])
1931                    }
1932                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1933                        return Ok(vec![control_error_frame(
1934                            &frame,
1935                            "invalid_control_body",
1936                            format!("malformed client control body: {err}"),
1937                        )?])
1938                    }
1939                };
1940                let op = client_control_request_op(&request);
1941                let corr = frame.header.corr;
1942                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1943                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1944                #[cfg(test)]
1945                if let Some(delay) = self.control_dispatch_delay {
1946                    tokio::time::sleep(delay).await;
1947                }
1948                let result = self
1949                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1950                    .await;
1951                if let Some(timings) = route_open_timings.as_mut() {
1952                    timings.finish();
1953                }
1954                log_slow_control_dispatch(
1955                    dispatch_started_at,
1956                    op,
1957                    ctx.connection_id,
1958                    corr,
1959                    route_open_timings.as_ref(),
1960                );
1961                result
1962            }
1963            FrameType::Push => {
1964                let Some(endpoint) = self
1965                    .forwarding
1966                    .module_endpoint_for_connection(ctx.connection_id)
1967                    .map_err(RouterError::Forwarding)?
1968                else {
1969                    return Ok(vec![control_error_frame(
1970                        &frame,
1971                        "unsupported_control_frame",
1972                        "client-originated channel-0 PUSH is not supported",
1973                    )?]);
1974                };
1975                self.handle_status_update(endpoint, frame)
1976            }
1977            FrameType::Response | FrameType::Error
1978                if self
1979                    .forwarding
1980                    .module_endpoint_for_connection(ctx.connection_id)
1981                    .map_err(RouterError::Forwarding)?
1982                    .is_some() =>
1983            {
1984                self.handle_module_relay_response(ctx.connection_id, frame)
1985            }
1986            ty => Ok(vec![control_error_frame(
1987                &frame,
1988                "unsupported_control_frame",
1989                format!("unsupported channel-0 frame {ty:?}"),
1990            )?]),
1991        }
1992    }
1993
1994    pub fn cleanup_connection(
1995        &self,
1996        connection_id: ConnectionId,
1997    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1998        self.caller_endpoints
1999            .lock()
2000            .unwrap_or_else(|p| p.into_inner())
2001            .remove(&connection_id);
2002        self.forwarding
2003            .operator_confirms()
2004            .connection_closed(connection_id);
2005        self.cleanup_connection_with_end_reason(
2006            connection_id,
2007            RegistrationEndReason::ConnectionClosed,
2008        )
2009    }
2010
2011    fn cleanup_connection_with_end_reason(
2012        &self,
2013        connection_id: ConnectionId,
2014        requested_reason: RegistrationEndReason,
2015    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2016        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2017            self.registry
2018                .get_module_by_connection(connection_id)?
2019                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2020                .and_then(|module| module.registration_end_reason().ok().flatten())
2021                .unwrap_or(requested_reason)
2022        } else {
2023            requested_reason
2024        };
2025        let crash_closed = self
2026            .registry
2027            .get_module_by_connection(connection_id)?
2028            .and_then(|registration| {
2029                self.forwarding
2030                    .module_endpoint_for_connection(connection_id)
2031                    .ok()
2032                    .flatten()
2033                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2034                    .map(|routes| (registration.manifest.module_id, routes))
2035            });
2036        let crash_closed = crash_closed.map(|(module_id, routes)| {
2037            let terminal = match self.supervisor.get(&module_id) {
2038                None => false,
2039                Some(module) => match module.will_recover_after_connection_loss() {
2040                    Ok(will_recover) => !will_recover,
2041                    Err(err) => {
2042                        warn!(
2043                            %module_id,
2044                            error = %err,
2045                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2046                        );
2047                        false
2048                    }
2049                },
2050            };
2051            // The forwarding table gates all providers at the start of daemon
2052            // shutdown, before their connections are closed. An ordinary
2053            // module disconnect still reports crash if that gate is not set.
2054            let (reason, drained) = match self
2055                .forwarding
2056                .daemon_drain_result_for_connection(connection_id)
2057            {
2058                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2059                Ok(None) => (RouteCloseReason::Crash, false),
2060                Err(err) => {
2061                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2062                    (RouteCloseReason::Crash, false)
2063                }
2064            };
2065            (module_id, routes, reason, drained, terminal)
2066        });
2067        let registrations = self.deregister_connection(connection_id, end_reason);
2068        let cleanup = if crash_closed.is_some() {
2069            self.forwarding.cleanup_connection_counted(connection_id)
2070        } else {
2071            // A module connection's teardown needs the count of abandoned
2072            // route.bind relays for its route.closed notice below. Any other
2073            // connection, such as a client's, sends no such notice and needs
2074            // only its routes released, so it uses the route-only wrapper and
2075            // reports zero.
2076            self.forwarding
2077                .cleanup_connection(connection_id)
2078                .map(|released| crate::forwarding::ConnectionCleanup {
2079                    released,
2080                    abandoned_relays: 0,
2081                })
2082        };
2083        // The route.closed push waits for forwarding teardown because only
2084        // teardown knows how many pending route.bind relays it aborted. It still
2085        // goes out before the GOODBYEs for the released routes, and its targets
2086        // were captured above, before teardown removed those routes.
2087        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2088            let abandoned = cleanup
2089                .as_ref()
2090                .map_or(0, |cleanup| cleanup.abandoned_relays);
2091            send_route_control_pushes(
2092                &self.forwarding,
2093                routes,
2094                ClientControlPush::RouteClosed {
2095                    module_id,
2096                    channels: Vec::new(),
2097                    reason,
2098                    drained,
2099                    abandoned,
2100                    excluded_subscriptions: 0,
2101                    terminal: Some(terminal),
2102                },
2103            );
2104        }
2105        if let Ok(cleanup) = cleanup {
2106            self.emit_route_goodbyes(cleanup.released);
2107        }
2108        // Signal the registration-release watch only now that BOTH registry and
2109        // forwarding teardown are done, so a supervisor waiting to spawn a
2110        // replacement never observes release while old routes still exist.
2111        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2112            crate::supervise::notify_registration_release();
2113            self.capability_evaluator.wake_deadline_loop();
2114            self.refresh_capability_requirements();
2115        }
2116        self.supervisor.remove_spawn_subscribers(connection_id);
2117        self.activity_streams
2118            .lock()
2119            .unwrap_or_else(|p| p.into_inner())
2120            .retain(|(id, _), task| {
2121                if *id == connection_id {
2122                    task.abort();
2123                    false
2124                } else {
2125                    true
2126                }
2127            });
2128        // Sync authority dies with its connection, so the owner's next
2129        // connection can take it; the owner's scopes stay as they are.
2130        self.hello_launch_nonces
2131            .lock()
2132            .unwrap_or_else(|poisoned| poisoned.into_inner())
2133            .forget(connection_id);
2134        self.scopes
2135            .write()
2136            .unwrap_or_else(|poisoned| poisoned.into_inner())
2137            .release_connection(connection_id);
2138        registrations
2139    }
2140
2141    pub(crate) fn handle_route_goodbye(
2142        &self,
2143        connection_id: ConnectionId,
2144        route_channel: u16,
2145        route_epoch: u32,
2146    ) -> Result<bool, RouterError> {
2147        debug!(
2148            connection_id = connection_id.get(),
2149            route_channel, route_epoch, "handling route GOODBYE"
2150        );
2151        let RouteRelease::Removed(released_route) = self
2152            .forwarding
2153            .release_client_route(connection_id, route_channel, route_epoch)
2154            .map_err(RouterError::Forwarding)?
2155        else {
2156            return Ok(false);
2157        };
2158        self.emit_route_goodbyes(vec![released_route]);
2159        Ok(true)
2160    }
2161
2162    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2163        for released in released_routes {
2164            let frame = match Frame::build_with_version(
2165                released.negotiated_ver,
2166                FrameType::Goodbye,
2167                control_flags(),
2168                released.channel,
2169                released.epoch,
2170                0,
2171                Vec::new(),
2172            ) {
2173                Ok(frame) => frame,
2174                Err(err) => {
2175                    warn!(
2176                        route_channel = released.channel,
2177                        error = %err,
2178                        "failed to build route GOODBYE frame"
2179                    );
2180                    continue;
2181                }
2182            };
2183            if !released.close_on_delivery_failure() {
2184                crate::forwarding::send_module_route_goodbye(
2185                    &self.counters,
2186                    &released.sink,
2187                    frame,
2188                    released.module_id.as_deref(),
2189                    "client route released",
2190                );
2191                continue;
2192            }
2193            if let Err(err) = released.sink.try_send(frame) {
2194                warn!(
2195                    target_connection_id = released.connection_id.get(),
2196                    route_channel = released.channel,
2197                    error = %err,
2198                    "route GOODBYE was not delivered to client; closing target connection"
2199                );
2200                if self
2201                    .forwarding
2202                    .escalate_client_delivery_failure(
2203                        released.connection_id,
2204                        released.channel,
2205                        released.epoch,
2206                        CloseReason::new(
2207                            "route_goodbye_delivery_failed",
2208                            format!(
2209                                "failed to enqueue route GOODBYE for channel {}: {err}",
2210                                released.channel
2211                            ),
2212                        ),
2213                        crate::forwarding::UndeliveredFrame {
2214                            module_id: released.module_id.as_deref(),
2215                            sink: &released.sink,
2216                        },
2217                    )
2218                    .unwrap_or(false)
2219                {
2220                    self.counters.increment_goodbye_relay_client_failed();
2221                }
2222            }
2223        }
2224    }
2225
2226    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2227    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2228    /// own commit failed after the module had already accepted). Without this, a
2229    /// module that accepts late keeps a binding subc has torn down, so a later
2230    /// frame on that module channel could misdeliver if the channel is reused.
2231    ///
2232    /// Never closes the shared module connection on failure: a dropped notification
2233    /// only wastes a bounded amount of warm module-side state, which the module's
2234    /// own idle reaper reclaims. Only call this once the route.bind relay was
2235    /// actually enqueued to the module — if the relay send itself failed, the
2236    /// module never created a binding and there is nothing to tear down.
2237    fn send_abandoned_route_bind_goodbye(
2238        &self,
2239        module_sink: &crate::FrameSink,
2240        negotiated_ver: u8,
2241        module_channel: u16,
2242        module_epoch: u32,
2243    ) {
2244        let frame = match Frame::build_with_version(
2245            negotiated_ver,
2246            FrameType::Goodbye,
2247            control_flags(),
2248            module_channel,
2249            module_epoch,
2250            0,
2251            Vec::new(),
2252        ) {
2253            Ok(frame) => frame,
2254            Err(err) => {
2255                warn!(
2256                    route_channel = module_channel,
2257                    error = %err,
2258                    "failed to build GOODBYE for abandoned route.bind"
2259                );
2260                return;
2261            }
2262        };
2263        crate::forwarding::send_module_route_goodbye(
2264            &self.counters,
2265            module_sink,
2266            frame,
2267            None,
2268            "abandoned route.bind",
2269        );
2270    }
2271
2272    fn handle_hello(
2273        &self,
2274        connection_id: ConnectionId,
2275        sink: Option<crate::FrameSink>,
2276        frame: Frame,
2277    ) -> Result<Vec<Frame>, RouterError> {
2278        debug!(
2279            connection_id = connection_id.get(),
2280            corr = frame.header.corr,
2281            "handling HELLO"
2282        );
2283        // A module connection has one identity for its entire lifetime. A second
2284        // registration would leave the old registry owner behind while replacing
2285        // its forwarding endpoint and launch nonce.
2286        if self
2287            .registry
2288            .get_module_by_connection(connection_id)
2289            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2290            .is_some()
2291        {
2292            return Ok(vec![control_error_frame(
2293                &frame,
2294                "invalid_hello",
2295                "connection is already registered as a module",
2296            )?]);
2297        }
2298        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2299            Ok(value) => value,
2300            Err(err) => {
2301                return Ok(vec![control_error_frame(
2302                    &frame,
2303                    "invalid_hello",
2304                    format!("malformed HELLO body: {err}"),
2305                )?])
2306            }
2307        };
2308        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2309            return Ok(vec![control_error_frame(
2310                &frame,
2311                "invalid_capability_grammar",
2312                err.to_string(),
2313            )?]);
2314        }
2315        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2316            return Ok(vec![control_error_frame(
2317                &frame,
2318                "invalid_manifest",
2319                err.to_string(),
2320            )?]);
2321        }
2322        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2323            return Ok(vec![control_error_frame(
2324                &frame,
2325                "invalid_event_declaration",
2326                err.to_string(),
2327            )?]);
2328        }
2329        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2330            return Ok(vec![control_error_frame(
2331                &frame,
2332                "invalid_manifest",
2333                err.to_string(),
2334            )?]);
2335        }
2336        if let Some(provenance) = hello_value
2337            .get("manifest")
2338            .and_then(|manifest| manifest.get("provenance"))
2339        {
2340            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2341                return Ok(vec![control_error_frame(
2342                    &frame,
2343                    "invalid_manifest",
2344                    format!("malformed manifest provenance: {err}"),
2345                )?]);
2346            }
2347        }
2348        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2349            let module_id = hello_value["manifest"]["module_id"]
2350                .as_str()
2351                .unwrap_or("<unknown>");
2352            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2353        }
2354        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2355            let module_id = hello_value["manifest"]["module_id"]
2356                .as_str()
2357                .unwrap_or("<unknown>");
2358            warn!(module_id, %reason, "dropping flow_operations from HELLO");
2359        }
2360        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2361            Ok(hello) => hello,
2362            Err(err) => {
2363                return Ok(vec![control_error_frame(
2364                    &frame,
2365                    "invalid_hello",
2366                    format!("malformed HELLO body: {err}"),
2367                )?])
2368            }
2369        };
2370
2371        if hello.protocol_ver != hello.manifest.protocol_ver {
2372            return Ok(vec![control_error_frame(
2373                &frame,
2374                "invalid_manifest",
2375                format!(
2376                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2377                    hello.protocol_ver, hello.manifest.protocol_ver
2378                ),
2379            )?]);
2380        }
2381
2382        if hello.manifest.module_id.trim().is_empty() {
2383            return Ok(vec![control_error_frame(
2384                &frame,
2385                "invalid_manifest",
2386                "manifest module_id must not be empty",
2387            )?]);
2388        }
2389
2390        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2391            Ok(negotiated_ver) => negotiated_ver,
2392            Err(message) => {
2393                return Ok(vec![control_error_frame(
2394                    &frame,
2395                    "version_unsupported",
2396                    message,
2397                )?])
2398            }
2399        };
2400
2401        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2402        // swap is open for this id, the only HELLO admitted as a second process
2403        // is the one carrying the candidate's launch nonce (the swap token), and
2404        // it registers into the candidate slot rather than being refused as a
2405        // duplicate. Run after the reserved gate, a reserved module's candidate
2406        // would be refused `reserved_module` for presenting a nonce that gate
2407        // does not know. See `SupervisorHandle::swap_hello_admission`.
2408        let swap_admission = self
2409            .supervisor
2410            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2411        if swap_admission == SwapHelloAdmission::Refused {
2412            warn!(
2413                module_id = %hello.manifest.module_id,
2414                connection_id = connection_id.get(),
2415                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2416            );
2417            return Ok(vec![control_error_frame(
2418                &frame,
2419                "swap_token_invalid",
2420                format!(
2421                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2422                    hello.manifest.module_id
2423                ),
2424            )?]);
2425        }
2426        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2427
2428        // A swap candidate proves its identity with the candidate nonce above.
2429        // Other registrations receive this catalog fact only when reserved-module
2430        // policy required and accepted a nonce; unreserved registrations require
2431        // no nonce check.
2432        let nonce_verified = if swap_candidate {
2433            true
2434        } else {
2435            match self.supervisor.reserved_hello_verification(
2436                &hello.manifest.module_id,
2437                hello.launch_nonce.as_deref(),
2438            ) {
2439                Ok(verified) => verified,
2440                Err(rejection) => {
2441                    let message = match rejection {
2442                        ReservedHelloRejection::Exact { module_id } => format!(
2443                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2444                        ),
2445                        ReservedHelloRejection::Prefix {
2446                            prefix,
2447                            owner_module_id,
2448                        } => format!(
2449                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2450                            hello.manifest.module_id
2451                        ),
2452                    };
2453                    return Ok(vec![control_error_frame(
2454                        &frame,
2455                        "reserved_module",
2456                        message,
2457                    )?]);
2458                }
2459            }
2460        };
2461
2462        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2463            &hello.manifest.module_id,
2464            hello.manifest.capabilities.as_ref(),
2465        );
2466        if let Some(refusal) = reserved_capability_refusals.first() {
2467            let capability = refusal.capability.clone();
2468            let bound_module = refusal.claimants[0].clone();
2469            log_duplicate_claim_events(reserved_capability_refusals);
2470            return Ok(vec![control_error_frame(
2471                &frame,
2472                "reserved_capability",
2473                format!(
2474                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2475                    capability, bound_module, hello.manifest.module_id
2476                ),
2477            )?]);
2478        }
2479
2480        // A connection that already opened client routes must not also register as
2481        // a module: cleanup would then release only one side and leak the other.
2482        if self
2483            .forwarding
2484            .connection_has_client_routes(connection_id)
2485            .map_err(RouterError::Forwarding)?
2486        {
2487            return Ok(vec![control_error_frame(
2488                &frame,
2489                "invalid_hello",
2490                "connection has open client routes and cannot also register as a module",
2491            )?]);
2492        }
2493
2494        // Kept for scope sync authority, which goes only to the connection that
2495        // presented the module's current launch nonce. Recorded before the
2496        // registration is attempted: a connection whose registration then fails
2497        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2498        self.hello_launch_nonces
2499            .lock()
2500            .unwrap_or_else(|poisoned| poisoned.into_inner())
2501            .record(connection_id, hello.launch_nonce.as_deref());
2502        let control_ops = effective_module_control_ops(hello.control_ops);
2503        // Built before anything is registered so an encoding failure leaves no
2504        // registry or forwarding state behind.
2505        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2506        if swap_candidate {
2507            return self.register_swap_candidate(
2508                connection_id,
2509                sink,
2510                &frame,
2511                hello.manifest,
2512                negotiated_ver,
2513                control_ops,
2514                hello_ack,
2515            );
2516        }
2517        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2518        // is refused, but `catalog.list` does not show the module and the
2519        // catalog generation does not move until it is published below, once
2520        // its forwarding endpoint exists. Listing it now would advertise a
2521        // module that `route.open` still refuses for having no forwarding
2522        // connection.
2523        let registration = match self
2524            .registry
2525            .register_unpublished_verified_with_control_ops(
2526                hello.manifest,
2527                negotiated_ver,
2528                connection_id,
2529                control_ops,
2530                nonce_verified,
2531            ) {
2532            Ok(registration) => registration,
2533            Err(RegistryError::DuplicateModuleId { module_id }) => {
2534                return Ok(vec![control_error_frame(
2535                    &frame,
2536                    "duplicate_module_id",
2537                    format!(
2538                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2539                    ),
2540                )?])
2541            }
2542            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2543                return Ok(vec![control_error_frame(
2544                    &frame,
2545                    "invalid_module_id",
2546                    err.to_string(),
2547                )?])
2548            }
2549            Err(err) => {
2550                return Ok(vec![control_error_frame(
2551                    &frame,
2552                    "registry_error",
2553                    err.to_string(),
2554                )?])
2555            }
2556        };
2557        #[cfg(test)]
2558        self.hold_between_registry_and_forwarding();
2559
2560        let reply = if let Some(sink) = sink {
2561            // The forwarding table's module store is also the daemon-to-module
2562            // control-RPC lane, so every HELLO gets a live endpoint even when the
2563            // manifest has no routable provider role. Non-routable modules still
2564            // cannot receive route.bind in production: `handle_route_open` checks
2565            // the registry manifest with `target_has_required_role` before the
2566            // only production call to `begin_route_bind_relay_for` below that
2567            // route.open path. The remaining direct relay callers are unit tests
2568            // and benchmark harnesses that construct forwarding state explicitly.
2569            //
2570            // The HELLO_ACK is queued by the forwarding table itself, before the
2571            // endpoint becomes visible, and is NOT returned as a reply. A module
2572            // reads HELLO_ACK first and exits on anything else; a reply is only
2573            // written after this handler returns, by which time a route.open on
2574            // another connection could already have queued a route.bind request
2575            // for this module ahead of it.
2576            let concurrency = manifest_concurrency(&registration.manifest);
2577            if let Err(err) = self.forwarding.register_module_connection_acked(
2578                connection_id,
2579                registration.manifest.module_id.clone(),
2580                negotiated_ver,
2581                concurrency,
2582                sink,
2583                hello_ack,
2584            ) {
2585                // Forwarding registration failed, so there is no forwarding
2586                // state to tear down. Remove the registry entry, which was never
2587                // published and so never listed or counted in the catalog
2588                // generation, and signal the release watch directly.
2589                if matches!(
2590                    self.deregister_connection(
2591                        connection_id,
2592                        RegistrationEndReason::RegistrationFailed,
2593                    ),
2594                    Ok(r) if !r.is_empty()
2595                ) {
2596                    crate::supervise::notify_registration_release();
2597                }
2598                return Ok(vec![control_error_frame(
2599                    &frame,
2600                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2601                        "invalid_hello"
2602                    } else {
2603                        forwarding_error_code(&err)
2604                    },
2605                    err.to_string(),
2606                )?]);
2607            }
2608            Vec::new()
2609        } else {
2610            // No sink means no forwarding endpoint, so nothing can be routed
2611            // ahead of the ack; it goes out as the reply.
2612            vec![hello_ack]
2613        };
2614
2615        // The module is routable now (or, without a sink, as routable as it
2616        // will ever be), so the catalog may list it. Publishing bumps the
2617        // catalog generation in the same registry critical section that makes
2618        // the module listed.
2619        if !self
2620            .registry
2621            .publish_registration(connection_id)
2622            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2623        {
2624            debug!(
2625                module_id = %registration.manifest.module_id,
2626                connection_id = connection_id.get(),
2627                "registration was removed before it could be published"
2628            );
2629        }
2630
2631        // Exposure over assumption: Concurrency's serde default is pinned to the
2632        // pre-field behavior (ModuleManaged), so a management surface that is
2633        // genuinely Serial and just never declared it inherits concurrent
2634        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2635        // turns "no module has been bitten yet" into the checkable claim "no
2636        // module is exposed" -- one read of the boot log instead of a fleet
2637        // audit. Detected from the raw HELLO bytes because the serde default
2638        // deliberately erases the absent/declared distinction from the type.
2639        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2640            info!(
2641                module_id = %registration.manifest.module_id,
2642                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2643            );
2644        }
2645
2646        self.apply_registration_capabilities(&registration);
2647
2648        self.supervisor
2649            .record_activity_registration(&registration.manifest.module_id);
2650
2651        info!(
2652            module_id = %registration.manifest.module_id,
2653            module_version = %registration.manifest.module_version,
2654            negotiated_ver,
2655            routable_provider = manifest_provides_routable_role(&registration.manifest),
2656            connection_id = connection_id.get(),
2657            "module registered"
2658        );
2659
2660        Ok(reply)
2661    }
2662
2663    /// Register a HELLO the swap gate admitted into the candidate slot of the
2664    /// registry and of forwarding, where it is reachable over its own
2665    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2666    /// nothing routes to it until the supervisor cuts over.
2667    ///
2668    /// Registry first (unpublished), then forwarding, then publication, the
2669    /// same order as an ordinary HELLO; a forwarding failure removes the
2670    /// registry entry again. The swap supervisor treats a published, ready
2671    /// candidate as registered and promotes it, and promotion needs the
2672    /// candidate's forwarding endpoint, so the candidate is published only
2673    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2674    /// recompute capability verdicts or run the capability deny census
2675    /// (`enforce_capability_denies`): both are about the module in the active
2676    /// slot, which `route.open` can target. A candidate sits in the candidate
2677    /// slot, so it neither provides capabilities to other modules nor can be
2678    /// routed to until promotion moves it into the active slot.
2679    #[allow(clippy::too_many_arguments)]
2680    fn register_swap_candidate(
2681        &self,
2682        connection_id: ConnectionId,
2683        sink: Option<crate::FrameSink>,
2684        frame: &Frame,
2685        manifest: ModuleManifest,
2686        negotiated_ver: u8,
2687        control_ops: Vec<String>,
2688        hello_ack: Frame,
2689    ) -> Result<Vec<Frame>, RouterError> {
2690        let module_id = manifest.module_id.clone();
2691        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2692            manifest,
2693            negotiated_ver,
2694            connection_id,
2695            control_ops,
2696        ) {
2697            Ok(registration) => registration,
2698            Err(RegistryError::DuplicateModuleId { module_id }) => {
2699                return Ok(vec![control_error_frame(
2700                    frame,
2701                    "duplicate_module_id",
2702                    format!(
2703                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2704                    ),
2705                )?])
2706            }
2707            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2708                return Ok(vec![control_error_frame(
2709                    frame,
2710                    "invalid_module_id",
2711                    err.to_string(),
2712                )?])
2713            }
2714            Err(err) => {
2715                return Ok(vec![control_error_frame(
2716                    frame,
2717                    "registry_error",
2718                    err.to_string(),
2719                )?])
2720            }
2721        };
2722        #[cfg(test)]
2723        self.hold_between_registry_and_forwarding();
2724        let reply = if let Some(sink) = sink {
2725            // Same ordering as an ordinary HELLO: the forwarding table queues
2726            // the HELLO_ACK before the candidate endpoint is inserted, because
2727            // a module exits if its first frame after HELLO is anything else.
2728            let concurrency = manifest_concurrency(&registration.manifest);
2729            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2730                connection_id,
2731                module_id.clone(),
2732                negotiated_ver,
2733                concurrency,
2734                sink,
2735                hello_ack,
2736            ) {
2737                if matches!(
2738                    self.deregister_connection(
2739                        connection_id,
2740                        RegistrationEndReason::RegistrationFailed,
2741                    ),
2742                    Ok(r) if !r.is_empty()
2743                ) {
2744                    crate::supervise::notify_registration_release();
2745                }
2746                return Ok(vec![control_error_frame(
2747                    frame,
2748                    forwarding_error_code(&err),
2749                    err.to_string(),
2750                )?]);
2751            }
2752            Vec::new()
2753        } else {
2754            vec![hello_ack]
2755        };
2756        if !self
2757            .registry
2758            .publish_registration(connection_id)
2759            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2760        {
2761            debug!(
2762                module_id = %module_id,
2763                connection_id = connection_id.get(),
2764                "swap candidate registration was removed before it could be published"
2765            );
2766        }
2767        self.supervisor.mark_swap_candidate_admitted(&module_id);
2768        info!(
2769            module_id = %module_id,
2770            module_version = %registration.manifest.module_version,
2771            negotiated_ver,
2772            ready = registration.ready,
2773            connection_id = connection_id.get(),
2774            "swap candidate registered; not routable until cutover"
2775        );
2776        Ok(reply)
2777    }
2778
2779    fn build_hello_ack(
2780        &self,
2781        frame: &Frame,
2782        negotiated_ver: u8,
2783        module_id: &str,
2784    ) -> Result<Frame, RouterError> {
2785        let ack = ModuleHelloAckBody {
2786            negotiated_ver,
2787            subc_ops: module_subc_ops(),
2788            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2789            storage: self
2790                .storage_config
2791                .as_ref()
2792                .map(|cfg| cfg.descriptor_for(module_id)),
2793            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2794        };
2795        let body = serde_json::to_vec(&ack).map_err(|err| {
2796            RouterError::backend(
2797                0,
2798                frame.header.corr,
2799                format!("failed to encode HELLO_ACK: {err}"),
2800            )
2801        })?;
2802
2803        Frame::build_with_version(
2804            negotiated_ver,
2805            FrameType::HelloAck,
2806            control_flags(),
2807            0,
2808            0,
2809            frame.header.corr,
2810            body,
2811        )
2812        .map_err(RouterError::FrameBuild)
2813    }
2814
2815    async fn handle_client_control_request(
2816        &self,
2817        ctx: &RouteCtx,
2818        frame: Frame,
2819        request: ClientControlRequest,
2820        route_open_timings: &mut Option<RouteOpenTimings>,
2821    ) -> Result<Vec<Frame>, RouterError> {
2822        if let Some(operation) = privileged_operation(&request) {
2823            let caller = self.resolve_caller(ctx.connection_id).await;
2824            let op = client_control_request_op(&request);
2825            let text = prompt_text(operation, &caller);
2826            match self.privileged_operator_prompt {
2827                PrivilegedOperatorPrompt::Log => {
2828                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2829                        target = ?operation, caller = %caller.description(), summary = %text,
2830                        "privileged request would prompt the operator");
2831                }
2832                PrivilegedOperatorPrompt::Enforce => {
2833                    let key =
2834                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2835                    let outcome = self
2836                        .forwarding
2837                        .operator_confirms()
2838                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2839                        .await;
2840                    if outcome.code != "confirmed" {
2841                        return Ok(vec![outcome.frame(&frame)]);
2842                    }
2843                }
2844            }
2845        }
2846        match request {
2847            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2848            ClientControlRequest::CatalogList { module_id } => {
2849                self.handle_catalog_list(frame, module_id)
2850            }
2851            ClientControlRequest::RouteOpen {
2852                target,
2853                identity,
2854                consumer_identity,
2855                consumer_capabilities,
2856                role_versions,
2857                admission_facts,
2858                scope,
2859            } => {
2860                self.handle_route_open(
2861                    ctx,
2862                    frame,
2863                    RouteOpenRequest {
2864                        target,
2865                        identity,
2866                        consumer_identity,
2867                        consumer_capabilities,
2868                        role_versions,
2869                        admission_facts,
2870                        scope,
2871                    },
2872                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2873                )
2874                .await
2875            }
2876            ClientControlRequest::RoutePoll {
2877                route_channel,
2878                route_epoch,
2879                kind,
2880            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2881            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2882            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2883                self.handle_supervisor_spawn_snapshot(frame)
2884            }
2885            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2886                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2887            }
2888            ClientControlRequest::SupervisorRestart {
2889                module_id,
2890                drain_timeout_ms,
2891            } => {
2892                self.log_supervisor_request_received(
2893                    ctx,
2894                    frame.header.corr,
2895                    ops::SUPERVISOR_RESTART,
2896                    Some(&module_id),
2897                    None,
2898                )?;
2899                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2900                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2901                    .await
2902            }
2903            ClientControlRequest::SupervisorSwap {
2904                module_id,
2905                ready_timeout_ms,
2906            } => {
2907                self.log_supervisor_request_received(
2908                    ctx,
2909                    frame.header.corr,
2910                    ops::SUPERVISOR_SWAP,
2911                    Some(&module_id),
2912                    None,
2913                )?;
2914                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2915                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2916                    .await
2917            }
2918            ClientControlRequest::SupervisorReload { module_id } => {
2919                self.log_supervisor_request_received(
2920                    ctx,
2921                    frame.header.corr,
2922                    ops::SUPERVISOR_RELOAD,
2923                    Some(&module_id),
2924                    None,
2925                )?;
2926                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2927                self.handle_supervisor_reload(frame, module_id, requester)
2928                    .await
2929            }
2930            ClientControlRequest::SupervisorRescan { preview } => {
2931                if !preview {
2932                    self.log_supervisor_request_received(
2933                        ctx,
2934                        frame.header.corr,
2935                        ops::SUPERVISOR_RESCAN,
2936                        None,
2937                        None,
2938                    )?;
2939                }
2940                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2941                self.handle_supervisor_rescan(
2942                    frame,
2943                    if preview {
2944                        RescanMode::Preview
2945                    } else {
2946                        RescanMode::Execute
2947                    },
2948                    requester,
2949                )
2950                .await
2951            }
2952            ClientControlRequest::SupervisorRescanChecked {
2953                preview,
2954                candidate,
2955                expect,
2956            } => {
2957                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2958                    .await
2959            }
2960            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2961                self.log_supervisor_request_received(
2962                    ctx,
2963                    frame.header.corr,
2964                    ops::SUPERVISOR_RELEASE_RESERVED,
2965                    Some(&module_id),
2966                    None,
2967                )?;
2968                self.handle_supervisor_release_reserved(frame, module_id)
2969                    .await
2970            }
2971            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2972                self.log_supervisor_request_received(
2973                    ctx,
2974                    frame.header.corr,
2975                    ops::SUPERVISOR_SET_ENABLED,
2976                    Some(&module_id),
2977                    Some(enabled),
2978                )?;
2979                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2980                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2981                    .await
2982            }
2983            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2984                self.handle_supervisor_health_probe(frame, module_id).await
2985            }
2986            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2987            ClientControlRequest::SupervisorRoutes { module_id } => {
2988                self.handle_supervisor_routes(frame, module_id)
2989            }
2990            ClientControlRequest::SupervisorProvenance { module_id } => {
2991                self.handle_supervisor_provenance(frame, module_id).await
2992            }
2993            ClientControlRequest::SupervisorStderrTail {
2994                module_id,
2995                max_lines,
2996                max_bytes,
2997            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2998            ClientControlRequest::SupervisorTerminals { module_id } => {
2999                self.handle_supervisor_terminals(frame, module_id).await
3000            }
3001        }
3002    }
3003
3004    fn log_supervisor_request_received(
3005        &self,
3006        ctx: &RouteCtx,
3007        corr: u64,
3008        op: &'static str,
3009        module_id: Option<&str>,
3010        enabled: Option<bool>,
3011    ) -> Result<(), RouterError> {
3012        let caller = self.supervisor_requester(ctx, corr)?;
3013        let caller = principal_label(&caller);
3014        // `Option` fields are recorded only when present, so a request with no
3015        // module id (rescan) or no enabled flag simply omits that field.
3016        info!(
3017            target: "control",
3018            op,
3019            module_id,
3020            enabled,
3021            connection_id = ctx.connection_id.get(),
3022            caller = %caller,
3023            "supervisor request received"
3024        );
3025        Ok(())
3026    }
3027
3028    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3029        Ok(self
3030            .registry
3031            .get_module_by_connection(ctx.connection_id)
3032            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3033            .map(|registration| Principal::Reserved {
3034                module_id: registration.manifest.module_id,
3035            })
3036            .unwrap_or(Principal::Direct))
3037    }
3038
3039    fn handle_module_control_request(
3040        &self,
3041        connection_id: ConnectionId,
3042        frame: Frame,
3043        request: ModuleControlRequestFromModule,
3044    ) -> Result<Vec<Frame>, RouterError> {
3045        match request {
3046            ModuleControlRequestFromModule::CatalogUpdate {
3047                provides,
3048                capabilities,
3049                ready,
3050            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3051            ModuleControlRequestFromModule::LiveRoots {} => {
3052                let registered = self
3053                    .registry
3054                    .get_module_by_connection(connection_id)
3055                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3056                let Some(registration) = registered else {
3057                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3058                };
3059                let response = self
3060                    .forwarding
3061                    .live_roots(&registration.manifest.module_id)
3062                    .map_err(RouterError::Forwarding)?;
3063                Ok(vec![control_response_body_frame(
3064                    &frame,
3065                    &response,
3066                    "ModuleControlResponseToModule::LiveRoots",
3067                )?])
3068            }
3069            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3070                self.handle_scope_sync(connection_id, frame, generation, scopes)
3071            }
3072            ModuleControlRequestFromModule::ScopeApply {
3073                generation,
3074                upsert,
3075                end,
3076            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3077            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3078                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3079            }
3080        }
3081    }
3082
3083    fn handle_scope_served_by(
3084        &self,
3085        connection_id: ConnectionId,
3086        frame: Frame,
3087    ) -> Result<Vec<Frame>, RouterError> {
3088        let registered = self
3089            .registry
3090            .get_module_by_connection(connection_id)
3091            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3092        let Some(registered) = registered else {
3093            return Ok(vec![control_error_frame(
3094                &frame,
3095                "not_registered",
3096                "scope.served_by requires an active module registration owned by this connection",
3097            )?]);
3098        };
3099        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3100        {
3101            Ok(request) => request,
3102            Err(err) => {
3103                return Ok(vec![control_error_frame(
3104                    &frame,
3105                    "invalid_control_body",
3106                    format!("malformed scope.served_by body: {err}"),
3107                )?])
3108            }
3109        };
3110        let answer = self
3111            .scopes
3112            .read()
3113            .unwrap_or_else(|poisoned| poisoned.into_inner())
3114            .served_by(
3115                &registered.manifest.module_id,
3116                &request.scope_ref,
3117                request.scope_epoch,
3118                &request.module_id,
3119            );
3120        let reply = subc_control::ScopeServedByReply::new(
3121            answer,
3122            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3123        );
3124        Ok(vec![control_response_body_frame(
3125            &frame,
3126            &reply,
3127            "ScopeServedByReply",
3128        )?])
3129    }
3130
3131    fn handle_operator_confirm(
3132        &self,
3133        ctx: &RouteCtx,
3134        frame: Frame,
3135    ) -> Result<Vec<Frame>, RouterError> {
3136        use crate::operator_confirm::{audit, Outcome};
3137        let registration = self
3138            .registry
3139            .get_module_by_connection(ctx.connection_id)
3140            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3141        let Some(registration) = registration else {
3142            let outcome = Outcome::refusal("not_registered");
3143            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3144            return Ok(vec![outcome.frame(&frame)]);
3145        };
3146        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3147            Ok(request) => request,
3148            Err(_) => {
3149                let outcome = Outcome::refusal("invalid_control_body");
3150                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3151                return Ok(vec![outcome.frame(&frame)]);
3152            }
3153        };
3154        let module_id = registration.manifest.module_id;
3155        // Read the launch nonce (under its own lock) before taking the forwarding
3156        // table's lock below: holding forwarding while waiting on another daemon
3157        // lock risks a lock-order deadlock with paths that take them the other way.
3158        let nonce = self
3159            .hello_launch_nonces
3160            .lock()
3161            .unwrap_or_else(|p| p.into_inner())
3162            .nonce(ctx.connection_id)
3163            .map(str::to_owned);
3164        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3165            self.supervisor
3166                .spawned_consumer_authorized(&module_id, nonce)
3167        });
3168        let confirms = self.forwarding.operator_confirms();
3169        self.forwarding
3170            .with_operator_route(
3171                ctx.connection_id,
3172                request.route_channel,
3173                request.route_epoch,
3174                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3175            )
3176            .map_err(RouterError::Forwarding)
3177    }
3178
3179    /// Resolve folders before taking the scope-table lock: a slow filesystem
3180    /// must not hold up admissions for unrelated scoped routes.
3181    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3182        let mut scopes = ScopeRecords::from(scopes);
3183        for (index, record) in scopes.records.iter_mut().enumerate() {
3184            if let Some(refusal) = Self::canonicalize_workdir(record) {
3185                scopes.refusals.insert(index, refusal);
3186            }
3187        }
3188        scopes
3189    }
3190
3191    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3192        let workdir = record.workdir.as_mut()?;
3193        let path = std::path::Path::new(workdir);
3194        if !path.is_absolute() {
3195            return Some((
3196                error_codes::INVALID_WORKDIR,
3197                "workdir must be absolute".to_string(),
3198            ));
3199        }
3200        match ProjectRootId::from_path_allowing_missing(path) {
3201            Ok(root) => match root.as_path().to_str() {
3202                Some(canonical) => {
3203                    *workdir = canonical.to_string();
3204                    None
3205                }
3206                None => Some((
3207                    error_codes::INVALID_WORKDIR,
3208                    "canonical workdir is not UTF-8".to_string(),
3209                )),
3210            },
3211            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3212        }
3213    }
3214
3215    /// `scope.sync`: the owner is the module registered on this connection.
3216    /// A connection with no registration (every client connection, `direct`
3217    /// included) is refused `not_registered` before the table is consulted.
3218    fn handle_scope_sync(
3219        &self,
3220        connection_id: ConnectionId,
3221        frame: Frame,
3222        generation: u64,
3223        scopes: Vec<ScopeRecord>,
3224    ) -> Result<Vec<Frame>, RouterError> {
3225        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3226    }
3227
3228    fn handle_scope_change(
3229        &self,
3230        connection_id: ConnectionId,
3231        frame: Frame,
3232        generation: u64,
3233        scopes: Vec<ScopeRecord>,
3234        end: Option<Vec<ScopeEnd>>,
3235    ) -> Result<Vec<Frame>, RouterError> {
3236        let is_apply = end.is_some();
3237        let Some(registration) = self
3238            .registry
3239            .get_module_by_connection(connection_id)
3240            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3241        else {
3242            return Ok(vec![control_error_frame(
3243                &frame,
3244                "not_registered",
3245                "scope.sync and scope.apply require an active module registration owned by this connection",
3246            )?]);
3247        };
3248        let owner = registration.manifest.module_id;
3249        let scopes = Self::prepare_scope_records(scopes);
3250        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3251        let is_current_launch = |connection: ConnectionId| {
3252            self.hello_launch_nonces
3253                .lock()
3254                .unwrap_or_else(|poisoned| poisoned.into_inner())
3255                .presented(connection, current_nonce.as_deref())
3256        };
3257        // Lock order is the scope table, then the forwarding table: the new
3258        // tags are published, and the routes the change closes are selected,
3259        // while the scope table is still write-locked, so no admission can read
3260        // a record whose tag is not yet published.
3261        let mut table = self
3262            .scopes
3263            .write()
3264            .unwrap_or_else(|poisoned| poisoned.into_inner());
3265        let outcome = match end {
3266            Some(end) => table.apply(
3267                &owner,
3268                connection_id,
3269                is_current_launch,
3270                generation,
3271                scopes,
3272                end,
3273            ),
3274            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3275        };
3276        let drained = match &outcome {
3277            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3278            Err(_) => Vec::new(),
3279        };
3280        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3281        drop(table);
3282        if has_deadlines {
3283            self.scope_deadline_added.notify_one();
3284        }
3285        match outcome {
3286            Ok(applied) => {
3287                let counts = ScopeOutcomeCounts::of(&applied.results);
3288                info!(
3289                    owner = %owner,
3290                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3291                    generation,
3292                    records = applied.results.len(),
3293                    created = counts.created,
3294                    replaced = counts.replaced,
3295                    updated = counts.updated,
3296                    unchanged = counts.unchanged,
3297                    refused = counts.refused,
3298                    ended = applied.ended.len(),
3299                    tag_changes = applied.tag_changes.len(),
3300                    routes_closed = drained.len(),
3301                    "scope change accepted"
3302                );
3303                // An accepted scope change can still refuse individual records, and the
3304                // owner is the only party that sees the reply. Name them here so
3305                // an operator can tell a refused session from a missing one
3306                // without the owner's logs. Capped so a call that refuses
3307                // thousands cannot flood the log; the count above is complete.
3308                for refused in applied
3309                    .results
3310                    .iter()
3311                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3312                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3313                {
3314                    warn!(
3315                        owner = %owner,
3316                        generation,
3317                        scope_ref = %refused.scope_ref,
3318                        scope_epoch = refused.scope_epoch,
3319                        code = refused.code.as_deref().unwrap_or(""),
3320                        "scope record refused"
3321                    );
3322                }
3323                self.close_scope_drained_routes(drained);
3324                let response = if is_apply {
3325                    ModuleControlResponseToModule::ScopeApply {
3326                        generation,
3327                        results: applied.results,
3328                        end_results: applied.end_results,
3329                        ended: applied.ended,
3330                    }
3331                } else {
3332                    ModuleControlResponseToModule::ScopeSync {
3333                        generation,
3334                        results: applied.results,
3335                        ended: applied.ended,
3336                    }
3337                };
3338                Ok(vec![control_response_body_frame(
3339                    &frame,
3340                    &response,
3341                    "scope change response",
3342                )?])
3343            }
3344            Err(refusal) => {
3345                info!(
3346                    owner = %owner,
3347                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3348                    generation,
3349                    code = refusal.code,
3350                    "scope change refused"
3351                );
3352                Ok(vec![control_error_frame(
3353                    &frame,
3354                    refusal.code,
3355                    refusal.message,
3356                )?])
3357            }
3358        }
3359    }
3360
3361    /// Tell both ends of each route a scope change closed. The module gets a
3362    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3363    /// the client's route handle. The client also gets `route.closed` with the
3364    /// scope reason, one push per module and reason, so it can tell a revoked
3365    /// route from an ordinary close and not reopen it.
3366    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3367        if drained.is_empty() {
3368            return;
3369        }
3370        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3371            BTreeMap::new();
3372        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3373        for route in drained {
3374            warn!(
3375                module_id = %route.module_id,
3376                reason = ?route.reason,
3377                client_connection_id = route.client.connection_id.get(),
3378                route_channel = route.client.channel,
3379                "closing route because its scope changed"
3380            );
3381            pushes
3382                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3383                .or_insert_with(|| (route.reason, Vec::new()))
3384                .1
3385                .push(EndpointRoute {
3386                    goodbye_target: route.client.clone(),
3387                    principal: Principal::Unverified,
3388                    bound_at: Instant::now(),
3389                    draining: false,
3390                    drain_reason: None,
3391                });
3392            goodbyes.push(route.module);
3393            goodbyes.push(route.client);
3394        }
3395        for ((module_id, _), (reason, routes)) in pushes {
3396            send_route_control_pushes(
3397                &self.forwarding,
3398                routes,
3399                ClientControlPush::RouteClosed {
3400                    module_id,
3401                    channels: Vec::new(),
3402                    reason,
3403                    drained: false,
3404                    abandoned: 0,
3405                    excluded_subscriptions: 0,
3406                    terminal: Some(false),
3407                },
3408            );
3409        }
3410        self.emit_route_goodbyes(goodbyes);
3411    }
3412
3413    /// `scope.describe`: any registered module may read any scope, because a
3414    /// provider must read the scope a route it serves is stamped with.
3415    fn handle_scope_describe(
3416        &self,
3417        connection_id: ConnectionId,
3418        frame: Frame,
3419        owner: Principal,
3420        scope_ref: String,
3421    ) -> Result<Vec<Frame>, RouterError> {
3422        let registered = self
3423            .registry
3424            .get_module_by_connection(connection_id)
3425            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3426        let Some(registered) = registered else {
3427            return Ok(vec![control_error_frame(
3428                &frame,
3429                "not_registered",
3430                "scope.describe requires an active module registration owned by this connection",
3431            )?]);
3432        };
3433        let mut description = self
3434            .scopes
3435            .read()
3436            .unwrap_or_else(|poisoned| poisoned.into_inner())
3437            .describe(&owner, &scope_ref);
3438        // Describe readers also decode strict ScopeAttributes. Attribution is
3439        // safe to omit unless the requesting module declares it understands it.
3440        if !registered
3441            .manifest
3442            .capabilities
3443            .as_ref()
3444            .is_some_and(|caps| {
3445                caps.provides
3446                    .iter()
3447                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3448            })
3449        {
3450            if let Some(stamp) = description.stamp.as_mut() {
3451                stamp.attributes.on_behalf_of_agent = None;
3452            }
3453        }
3454        if !registered
3455            .manifest
3456            .capabilities
3457            .as_ref()
3458            .is_some_and(|caps| {
3459                caps.provides
3460                    .iter()
3461                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3462            })
3463        {
3464            if let Some(stamp) = description.stamp.as_mut() {
3465                stamp.attributes.flow_version = None;
3466                stamp.attributes.flow_code_hash = None;
3467                stamp.attributes.flow_install_id = None;
3468            }
3469        }
3470        let owner_configured = match &owner {
3471            // Ask whether the owner is configured (`is_configured`), not
3472            // whether it is on the roster (`get(..).is_some()`): a supervised
3473            // module's process can register and describe a scope before the
3474            // supervisor has put it on the roster.
3475            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3476            _ => false,
3477        };
3478        let response = ModuleControlResponseToModule::ScopeDescribe {
3479            status: description.status,
3480            scope_epoch: description.scope_epoch,
3481            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3482            owner_synced: description.owner_synced,
3483            owner_configured,
3484            scope: description.stamp,
3485        };
3486        Ok(vec![control_response_body_frame(
3487            &frame,
3488            &response,
3489            "ModuleControlResponseToModule::ScopeDescribe",
3490        )?])
3491    }
3492
3493    fn handle_catalog_update(
3494        &self,
3495        connection_id: ConnectionId,
3496        frame: Frame,
3497        provides: Vec<ProviderRole>,
3498        capabilities: Option<CapabilityDeclarations>,
3499        ready: Option<bool>,
3500    ) -> Result<Vec<Frame>, RouterError> {
3501        self.refresh_capability_requirements();
3502        let Some(registration) = self
3503            .registry
3504            .get_module_by_connection(connection_id)
3505            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3506        else {
3507            return Ok(vec![control_error_frame(
3508                &frame,
3509                "not_registered",
3510                "catalog.update requires an active module registration owned by this connection",
3511            )?]);
3512        };
3513
3514        if let Some(message) =
3515            catalog_update_frozen_field_message(&registration.manifest, &provides)
3516        {
3517            return Ok(vec![control_error_frame(
3518                &frame,
3519                "catalog_update_frozen_field",
3520                message,
3521            )?]);
3522        }
3523
3524        let mut candidate = registration.manifest.clone();
3525        candidate.provides = provides.clone();
3526        candidate.capabilities = capabilities
3527            .clone()
3528            .or_else(|| registration.manifest.capabilities.clone());
3529        if let Some(rich_ui) = &candidate.rich_ui {
3530            if let Err(err) = rich_ui.validate() {
3531                return Ok(vec![control_error_frame(
3532                    &frame,
3533                    "invalid_manifest",
3534                    err.to_string(),
3535                )?]);
3536            }
3537        }
3538        if let Err(err) = candidate.validate_capability_grammar() {
3539            return Ok(vec![control_error_frame(
3540                &frame,
3541                "invalid_capability_grammar",
3542                err.to_string(),
3543            )?]);
3544        }
3545
3546        // Updates must honor the same reserved owner as initial registration;
3547        // otherwise an empty HELLO could acquire the claim after admission.
3548        let mut conflicts = self
3549            .capability_evaluator
3550            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3551        if let Some(conflict) = conflicts.first() {
3552            let message = format!(
3553                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3554                conflict.capability, conflict.claimants[0], candidate.module_id
3555            );
3556            for conflict in &mut conflicts {
3557                conflict.source = DuplicateClaimSource::CatalogUpdate;
3558            }
3559            log_duplicate_claim_events(conflicts);
3560            return Ok(vec![control_error_frame(
3561                &frame,
3562                "reserved_capability",
3563                message,
3564            )?]);
3565        }
3566
3567        let updated = self
3568            .registry
3569            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3570            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3571        if updated.is_none() {
3572            return Ok(vec![control_error_frame(
3573                &frame,
3574                "not_registered",
3575                "catalog.update requires an active module registration owned by this connection",
3576            )?]);
3577        }
3578        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3579            log_duplicate_claim_events(
3580                self.capability_evaluator
3581                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3582            );
3583        }
3584        if capability_census_trigger(
3585            registration.manifest.capabilities.as_ref(),
3586            updated
3587                .as_ref()
3588                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3589        ) {
3590            self.enforce_capability_denies();
3591        }
3592        self.refresh_capability_requirements();
3593
3594        let response = ModuleControlResponseToModule::CatalogUpdate {};
3595        control_response_body_frame(
3596            &frame,
3597            &response,
3598            "ModuleControlResponseToModule::CatalogUpdate",
3599        )
3600        .map(|frame| vec![frame])
3601    }
3602
3603    pub(crate) fn with_activity_ledger_config(
3604        mut self,
3605        run_dir: &Path,
3606        incarnation: String,
3607        config: crate::activity::ActivityConfig,
3608    ) -> Self {
3609        self.activity = Some(crate::activity::ActivityLedger::open_config(
3610            run_dir.join("activity.jsonl"),
3611            incarnation,
3612            config,
3613        ));
3614        self.supervisor.set_activity(self.activity.clone());
3615        self
3616    }
3617
3618    pub(crate) async fn configure_activity(
3619        &self,
3620        config: crate::activity::ActivityConfig,
3621    ) -> Result<(), &'static str> {
3622        match &self.activity {
3623            Some(ledger) => ledger.configure(config).await,
3624            None => Ok(()),
3625        }
3626    }
3627
3628    fn cancel_activity_subscription(
3629        &self,
3630        ctx: &RouteCtx,
3631        frame: &Frame,
3632    ) -> Result<bool, RouterError> {
3633        let task = self
3634            .activity_streams
3635            .lock()
3636            .unwrap_or_else(|p| p.into_inner())
3637            .remove(&(ctx.connection_id, frame.header.corr));
3638        let Some(task) = task else {
3639            return Ok(false);
3640        };
3641        task.abort();
3642        let end = Frame::build_with_version(
3643            response_version(frame),
3644            FrameType::StreamEnd,
3645            control_flags(),
3646            0,
3647            0,
3648            frame.header.corr,
3649            Vec::new(),
3650        )
3651        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3652        let sink = ctx.egress.clone();
3653        tokio::spawn(async move {
3654            let _ = sink.send(end).await;
3655        });
3656        Ok(true)
3657    }
3658
3659    async fn handle_activity_read(
3660        &self,
3661        ctx: &RouteCtx,
3662        frame: Frame,
3663    ) -> Result<Vec<Frame>, RouterError> {
3664        use subc_control::{
3665            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3666            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3667        };
3668        #[derive(Deserialize)]
3669        struct Identity {
3670            #[serde(default)]
3671            consumer_identity: Option<ConsumerIdentity>,
3672        }
3673        // A registered module connection already identifies its caller. Ignore
3674        // consumer_identity there so a module cannot borrow a mirror's identity.
3675        // Only separate consumer connections use the Direct/nonce trust boundary.
3676        let principal = match self
3677            .forwarding
3678            .module_id_for_connection(ctx.connection_id)
3679            .map_err(RouterError::Forwarding)?
3680        {
3681            Some(module_id) => Ok(Principal::Reserved { module_id }),
3682            None => {
3683                let identity: Identity = match serde_json::from_slice(&frame.body) {
3684                    Ok(identity) => identity,
3685                    Err(_) => {
3686                        return Ok(vec![control_error_frame(
3687                            &frame,
3688                            ACTIVITY_MALFORMED,
3689                            "malformed activity read",
3690                        )?])
3691                    }
3692                };
3693                self.route_open_principal(&frame, identity.consumer_identity)?
3694            }
3695        };
3696        let admitted = matches!(principal, Ok(Principal::Direct))
3697            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3698        if !admitted {
3699            return Ok(vec![control_error_frame(
3700                &frame,
3701                ACTIVITY_READ_FORBIDDEN,
3702                "activity reads require an operator or trusted mirror principal",
3703            )?]);
3704        }
3705        let Some(ledger) = &self.activity else {
3706            return Ok(vec![control_error_frame(
3707                &frame,
3708                ACTIVITY_UNAVAILABLE,
3709                "activity ledger unavailable",
3710            )?]);
3711        };
3712        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3713            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3714        {
3715            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3716                Ok(request) => ledger.query(request).await,
3717                Err(_) => Err(ACTIVITY_MALFORMED),
3718            };
3719            return match result {
3720                Ok(reply) => Ok(vec![control_response_body_frame(
3721                    &frame,
3722                    &reply,
3723                    ops::ACTIVITY_QUERY,
3724                )?]),
3725                Err(code) => Ok(vec![control_error_frame(
3726                    &frame,
3727                    code,
3728                    "activity query refused",
3729                )?]),
3730            };
3731        }
3732        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3733            Ok(request) => request,
3734            Err(_) => {
3735                return Ok(vec![control_error_frame(
3736                    &frame,
3737                    ACTIVITY_MALFORMED,
3738                    "malformed activity subscription",
3739                )?])
3740            }
3741        };
3742        let mut subscription = match ledger.subscribe(request.since).await {
3743            Ok(subscription) => subscription,
3744            Err(code) => {
3745                return Ok(vec![control_error_frame(
3746                    &frame,
3747                    code,
3748                    "activity subscription refused",
3749                )?])
3750            }
3751        };
3752        let key = (ctx.connection_id, frame.header.corr);
3753        let sink = ctx.egress.clone();
3754        let mut streams = self
3755            .activity_streams
3756            .lock()
3757            .unwrap_or_else(|p| p.into_inner());
3758        streams.retain(|_, task| !task.is_finished());
3759        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3760            return Ok(vec![control_error_frame(
3761                &frame,
3762                "duplicate_subscription",
3763                "activity subscription correlation id already in use",
3764            )?]);
3765        }
3766        let task = tokio::spawn(async move {
3767            loop {
3768                let entry = if let Some(entry) = subscription.replay.pop_front() {
3769                    Some(entry)
3770                } else {
3771                    subscription.receiver.recv().await
3772                };
3773                let Some(entry) = entry else {
3774                    break;
3775                };
3776                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3777                    return;
3778                };
3779                let Ok(next) = Frame::build_with_version(
3780                    response_version(&frame),
3781                    FrameType::StreamData,
3782                    control_flags(),
3783                    0,
3784                    0,
3785                    frame.header.corr,
3786                    body,
3787                ) else {
3788                    return;
3789                };
3790                if sink.send(next).await.is_err() {
3791                    return;
3792                }
3793            }
3794            // Dropping a full subscriber's sender drains its accepted entries
3795            // before the terminal error, preserving their order on the wire.
3796            if let Some(code) = subscription.terminal_code() {
3797                if let Ok(end) =
3798                    control_error_frame(&frame, code, "activity subscriber fell behind")
3799                {
3800                    let _ = sink.send(end).await;
3801                }
3802            }
3803        });
3804        streams.insert(key, task.abort_handle());
3805        Ok(Vec::new())
3806    }
3807
3808    async fn handle_activity_record(
3809        &self,
3810        ctx: &RouteCtx,
3811        frame: Frame,
3812    ) -> Result<Vec<Frame>, RouterError> {
3813        use subc_control::{
3814            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3815            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3816        };
3817        let result = async {
3818            let module = self
3819                .forwarding
3820                .module_id_for_connection(ctx.connection_id)
3821                .ok()
3822                .flatten()
3823                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3824            let request: ActivityRecordRequest =
3825                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3826            crate::activity::validate(&module, &request)?;
3827            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3828            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3829            let at_ms = std::time::SystemTime::now()
3830                .duration_since(std::time::UNIX_EPOCH)
3831                .unwrap_or_default()
3832                .as_millis() as u64;
3833            ledger.record(module, request, actor, at_ms).await
3834        }
3835        .await;
3836        match result {
3837            Ok(entry) => Ok(vec![control_response_body_frame(
3838                &frame,
3839                &ActivityRecordResponse { seq: entry.seq },
3840                "activity.record",
3841            )?]),
3842            Err(code) => Ok(vec![control_error_frame(
3843                &frame,
3844                code,
3845                "activity recording refused",
3846            )?]),
3847        }
3848    }
3849
3850    fn activity_actor(
3851        &self,
3852        connection: ConnectionId,
3853        module: &str,
3854        on: Option<&subc_control::ActivityOn>,
3855    ) -> Result<subc_control::ActivityActor, &'static str> {
3856        use crate::forwarding::{DataRoute, DataRouteState};
3857        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3858        let principal = |id: &str| Principal::Reserved {
3859            module_id: id.to_owned(),
3860        };
3861        let mut actor = ActivityActor {
3862            principal: principal(module),
3863            scope_owner: None,
3864            agent_id: None,
3865            run_id: None,
3866            flow_id: None,
3867            project_root: None,
3868            workdir: None,
3869            project_id: None,
3870        };
3871        let scope = match on {
3872            None => return Ok(actor),
3873            Some(ActivityOn::Scope(scope)) => {
3874                let selector = subc_protocol::scope::ScopeSelector::new(
3875                    principal(module),
3876                    scope.scope_ref.clone(),
3877                    Some(scope.epoch),
3878                );
3879                self.scopes
3880                    .read()
3881                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3882                    .attribute(&principal(module), module, &selector, true)
3883                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3884            }
3885            Some(ActivityOn::Route(route)) => {
3886                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3887                    .forwarding
3888                    .lookup_data_route(connection, route.channel, route.epoch)
3889                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3890                else {
3891                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3892                };
3893                if binding.module_id != module {
3894                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3895                }
3896                actor.principal = binding.principal.clone();
3897                actor.project_root = binding
3898                    .project_root
3899                    .as_ref()
3900                    .map(|root| root.as_path().to_path_buf());
3901                let Some(bound) = &binding.scope else {
3902                    return Ok(actor);
3903                };
3904                let owner = principal(&bound.owner);
3905                let selector = subc_protocol::scope::ScopeSelector::new(
3906                    owner.clone(),
3907                    bound.scope_ref.clone(),
3908                    Some(bound.tag.scope_epoch),
3909                );
3910                let admitted = self
3911                    .scopes
3912                    .read()
3913                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3914                    .attribute(&owner, module, &selector, true)
3915                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3916                if admitted.tag != bound.tag {
3917                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3918                }
3919                admitted
3920            }
3921        };
3922        actor.scope_owner = Some(scope.stamp.owner);
3923        actor.agent_id = scope.stamp.attributes.agent_id;
3924        actor.run_id = scope.stamp.attributes.run_id;
3925        actor.flow_id = scope.stamp.attributes.flow_id;
3926        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3927        actor.project_id = scope.stamp.project_id;
3928        Ok(actor)
3929    }
3930
3931    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3932        self.refresh_capability_requirements();
3933        // A bare connection count is ambiguous between many clients holding a
3934        // route each and one client accumulating hundreds, so publish the
3935        // concentration alongside it. Route state is best-effort here: a
3936        // diagnostic endpoint must still answer if the forwarding lock is
3937        // contended.
3938        let mut counters = self.counters.snapshot();
3939        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3940            obj.insert(
3941                "activity_unparseable_lines".into(),
3942                ledger.read_errors().into(),
3943            );
3944            obj.insert(
3945                "activity_lifecycle_append_failures".into(),
3946                ledger.lifecycle_failures().into(),
3947            );
3948            obj.insert(
3949                "activity_started_digest_skips".into(),
3950                ledger.lifecycle_digest_skips().into(),
3951            );
3952        }
3953        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3954            self.forwarding.client_route_concentration(),
3955            counters.as_object_mut(),
3956        ) {
3957            obj.insert(
3958                "client_connections_with_routes".into(),
3959                connections_with_routes.into(),
3960            );
3961            obj.insert("max_routes_on_one_connection".into(), max.into());
3962        }
3963        // A module that is being fast-refused and a module that is fine look
3964        // identical from a client that retries and succeeds, so name the open
3965        // breakers here. This rides the existing free-form counters object
3966        // rather than a new wire field, so no sibling that deserializes
3967        // `ServerDescribe` has to be rebuilt to keep reading it.
3968        if let (Some(open_breakers), Some(obj)) = (
3969            self.route_bind_breakers.open_snapshot(),
3970            counters.as_object_mut(),
3971        ) {
3972            obj.insert("route_bind_breakers_open".into(), open_breakers);
3973        }
3974        let response = ClientControlResponse::ServerDescribe {
3975            protocol_ver: PROTOCOL_VERSION,
3976            subc_ops: subc_ops(),
3977            capabilities: self.describe_capabilities(),
3978            connected_clients: self.connected_clients.count(),
3979            counters: Some(counters),
3980            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3981            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3982            capability_requirements: self.capability_requirement_statuses(),
3983            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3984            config_path: self
3985                .rescan
3986                .as_ref()
3987                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3988        };
3989        Ok(vec![control_response_body_frame(
3990            &frame,
3991            &response,
3992            "ClientControlResponse::ServerDescribe",
3993        )?])
3994    }
3995
3996    fn handle_catalog_list(
3997        &self,
3998        frame: Frame,
3999        module_id: Option<String>,
4000    ) -> Result<Vec<Frame>, RouterError> {
4001        let (generation, modules) = self
4002            .registry
4003            .list_modules_with_nonce_verification()
4004            .map_err(|err| {
4005                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4006            })?;
4007        let entries = modules
4008            .into_iter()
4009            .filter(|(registration, _)| {
4010                module_id
4011                    .as_deref()
4012                    .map(|wanted| registration.manifest.module_id == wanted)
4013                    .unwrap_or(true)
4014            })
4015            .map(|(registration, nonce_verified)| {
4016                let not_ready = self.not_ready_reason(&registration);
4017                let roles = registration.manifest.provides;
4018                CatalogEntry::new(
4019                    registration.manifest.module_id,
4020                    roles,
4021                    registration.control_ops,
4022                )
4023                .with_ready(not_ready.is_none())
4024                .with_not_ready(not_ready)
4025                .with_module_version(Some(registration.manifest.module_version))
4026                .with_capabilities(registration.manifest.capabilities)
4027                .with_self_signals(registration.manifest.self_signals)
4028                .with_events(registration.manifest.events)
4029                .with_operation_delivery(registration.manifest.operation_delivery)
4030                .with_operation_payload(registration.manifest.operation_payload)
4031                .with_flow_operations(registration.manifest.flow_operations)
4032                .with_rich_ui(registration.manifest.rich_ui)
4033                .with_role_declarations(registration.manifest.role_declarations)
4034                .with_nonce_verified(nonce_verified)
4035            })
4036            .collect();
4037        let response = ClientControlResponse::CatalogList {
4038            generation,
4039            modules: entries,
4040            subc_ops: subc_ops(),
4041        };
4042        Ok(vec![control_response_body_frame(
4043            &frame,
4044            &response,
4045            "ClientControlResponse::CatalogList",
4046        )?])
4047    }
4048
4049    /// The principal a route.open's `consumer_identity` attests: `Direct`
4050    /// when it presents none, `Reserved` when its launch nonce verifies, and
4051    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4052    /// can run before any refusal check without changing which check refuses.
4053    fn attest_route_open_consumer(
4054        &self,
4055        consumer_identity: Option<&ConsumerIdentity>,
4056    ) -> Option<Principal> {
4057        let Some(consumer_identity) = consumer_identity else {
4058            return Some(Principal::Direct);
4059        };
4060        self.supervisor
4061            .spawned_consumer_authorized(
4062                &consumer_identity.module_id,
4063                &consumer_identity.launch_nonce,
4064            )
4065            .then(|| Principal::Reserved {
4066                module_id: consumer_identity.module_id.clone(),
4067            })
4068    }
4069
4070    /// The caller fields for a refusal that only has the raw request frame,
4071    /// such as the connection-level admission refusal sent before the request
4072    /// reaches [`Self::handle_route_open`].
4073    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4074        let Ok(ClientControlRequest::RouteOpen {
4075            identity,
4076            consumer_identity,
4077            scope,
4078            ..
4079        }) = parse_client_control_request(&frame.body)
4080        else {
4081            return RouteOpenCaller::unknown();
4082        };
4083        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4084        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4085    }
4086
4087    fn route_open_principal(
4088        &self,
4089        frame: &Frame,
4090        consumer_identity: Option<ConsumerIdentity>,
4091    ) -> Result<Result<Principal, Frame>, RouterError> {
4092        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4093        self.route_open_principal_attested(frame, consumer_identity, attested)
4094    }
4095
4096    /// [`Self::route_open_principal`] with `attested` already taken by
4097    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4098    /// open checks the launch nonce once.
4099    fn route_open_principal_attested(
4100        &self,
4101        frame: &Frame,
4102        consumer_identity: Option<ConsumerIdentity>,
4103        attested: Option<Principal>,
4104    ) -> Result<Result<Principal, Frame>, RouterError> {
4105        if let Some(principal) = attested {
4106            return Ok(Ok(principal));
4107        }
4108        let Some(consumer_identity) = consumer_identity else {
4109            return Ok(Ok(Principal::Direct));
4110        };
4111
4112        Ok(Err(control_error_frame(
4113            frame,
4114            "bad_consumer_identity",
4115            format!(
4116                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4117                consumer_identity.module_id
4118            ),
4119        )?))
4120    }
4121
4122    /// Ordinary `route.open` refusals go through here; admission and breaker
4123    /// refusals log separately with their capacity or breaker state. The daemon can
4124    /// attest which code it sent: without the event, a client's "the daemon
4125    /// refused me" and the daemon's own view could only be reconciled by
4126    /// argument. Malformed input (`invalid_project_root`) does not come here;
4127    /// rejecting a request that was never a valid open is not a refusal of one.
4128    #[allow(clippy::too_many_arguments)]
4129    fn route_open_refusal_frame(
4130        &self,
4131        ctx: &RouteCtx,
4132        caller: &RouteOpenCaller,
4133        frame: &Frame,
4134        module_id: &str,
4135        reason: &'static str,
4136        code: &'static str,
4137        message: impl Into<String>,
4138    ) -> Result<Frame, RouterError> {
4139        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4140        control_error_frame(frame, code, message.into())
4141    }
4142
4143    /// Refuse a `route.open` because the target module's bind-relay breaker is
4144    /// open, without attempting the relay.
4145    ///
4146    /// The wire code is `module_timeout`, which is the truth (the module has
4147    /// not been answering binds) and which both SDKs already classify as
4148    /// retryable with capped backoff. Reusing it is what keeps this change out
4149    /// of both SDKs; the daemon-side distinction lives in the counter key
4150    /// instead.
4151    ///
4152    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4153    /// While a breaker is open this fires on every open to that module, and the
4154    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4155    /// produced 261 lines about a single module inside 3000 lines of daemon
4156    /// log. The rare transitions are logged at warn/info instead and the volume
4157    /// is carried by the counter, so the evidence survives without the flood.
4158    /// The debug line keeps a per-refusal record reachable for whoever turns
4159    /// the level up.
4160    #[allow(clippy::too_many_arguments)]
4161    fn route_open_breaker_refusal_frame(
4162        &self,
4163        ctx: &RouteCtx,
4164        caller: &RouteOpenCaller,
4165        frame: &Frame,
4166        module_id: &str,
4167        consecutive_timeouts: u32,
4168        retry_in: Duration,
4169        probe_in_flight: bool,
4170    ) -> Result<Frame, RouterError> {
4171        self.counters
4172            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4173        debug!(
4174            target: "control",
4175            code = "module_timeout",
4176            module_id = ?module_id,
4177            connection_id = ctx.connection_id.get(),
4178            principal = %caller.principal,
4179            scope = %caller.scope,
4180            harness = %caller.harness,
4181            consecutive_timeouts,
4182            retry_in_ms = retry_in.as_millis() as u64,
4183            probe_in_flight,
4184            "route.open refused by open bind-relay breaker"
4185        );
4186        // Say what a caller can act on. An open bind-relay breaker means the
4187        // module timed out accepting several new routes in a row. The module
4188        // is still running and its established routes keep working; only new
4189        // route.open requests are refused until the cooldown ends and one
4190        // test route (the probe) gets through. A message that only counts
4191        // failed relays reads as "the module is down" to a worker that sees it.
4192        let detail = if probe_in_flight {
4193            "one test route is already being tried; retry once it settles".to_string()
4194        } else {
4195            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4196        };
4197        control_error_frame(
4198            frame,
4199            "module_timeout",
4200            format!(
4201                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4202                 timed out in a row); {detail}; its established routes are unaffected"
4203            ),
4204        )
4205    }
4206
4207    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4208    /// requester's bytes (an unknown target is whatever the client sent) and
4209    /// is Debug-formatted so control characters land in the log escaped
4210    /// rather than as terminal sequences for whoever tails it.
4211    ///
4212    /// `reason` names the check that refused, because one wire code has
4213    /// several senders: after a module registers, `target_unavailable` can
4214    /// come from a missing role, an inactive registration, a supervisor that
4215    /// has not marked the process live, a missing forwarding connection, or a
4216    /// failed relay, and a log that records only the code cannot say which of
4217    /// them fired. It is a static, daemon-chosen label per branch, so it is
4218    /// safe to print plainly and stays a closed set.
4219    fn observe_route_open_refusal(
4220        &self,
4221        ctx: &RouteCtx,
4222        caller: &RouteOpenCaller,
4223        module_id: &str,
4224        reason: &'static str,
4225        code: &'static str,
4226    ) {
4227        self.counters.increment_route_open_refused(code);
4228        info!(
4229            target: "control",
4230            code,
4231            reason,
4232            module_id = ?module_id,
4233            connection_id = ctx.connection_id.get(),
4234            principal = %caller.principal,
4235            scope = %caller.scope,
4236            harness = %caller.harness,
4237            "route.open refused"
4238        );
4239        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4240            self.route_outages.record_not_serving(module_id, reason);
4241        }
4242    }
4243
4244    /// Record an ACCEPTED route.open.
4245    ///
4246    /// Refusals have been logged and counted since the attestation work; accepts
4247    /// were invisible, so the daemon knew every principal it stamped and wrote
4248    /// none of them down. The party that attests the identity was the only party
4249    /// not recording it, which left a credential vault unable to name the sender
4250    /// of a call that reached it (claustrum #43) and left the launch-nonce
4251    /// concurrency question unanswerable from the outside.
4252    ///
4253    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4254    /// `code`/`module_id`/`connection_id` returns both directions of the same
4255    /// decision rather than two shapes a reader has to join by hand.
4256    ///
4257    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4258    /// and the difference carries information rather than being an
4259    /// inconsistency. This line is only reachable after a successful bind to a
4260    /// REGISTERED module, so the value has already passed HELLO validation
4261    /// including the path-hazard refusal and cannot contain control bytes. A
4262    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4263    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4264    ///
4265    /// Bare is also what every other daemon line already emits (`module
4266    /// registered`, `configured module supervised`). Shipping `?module_id` here
4267    /// made this instrument the only one in the file whose ids did not answer
4268    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4269    /// line whose whole purpose is being grepped beside its sibling.
4270    ///
4271    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4272    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4273    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4274    /// one are recorded identically. A test written against that harness passes
4275    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4276    /// green assertion that cannot fail. The same limit applies to the escaping
4277    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4278    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4279    /// Fencing either needs the real formatter, not the capture layer.
4280    ///
4281    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4282    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4283    /// to record. The ephemeral port would decay within minutes and answer only a
4284    /// live question. The identity question is instead answered by counting
4285    /// distinct live connections presenting one module's `consumer_identity` --
4286    /// "is anyone else holding this secret" rather than "is this the right
4287    /// process".
4288    fn observe_route_open_accept(
4289        &self,
4290        ctx: &RouteCtx,
4291        module_id: &str,
4292        principal: &str,
4293        scope: &str,
4294    ) {
4295        self.route_outages.record_accepted(module_id);
4296        self.counters.increment_route_open_accepted(principal);
4297        info!(
4298            target: "control",
4299            principal,
4300            module_id,
4301            connection_id = ctx.connection_id.get(),
4302            scope = %scope,
4303            "route.open accepted"
4304        );
4305    }
4306
4307    fn supervised_absent_route_open_refusal_frame(
4308        &self,
4309        ctx: &RouteCtx,
4310        caller: &RouteOpenCaller,
4311        frame: &Frame,
4312        module_id: &str,
4313        code: &'static str,
4314        status: &crate::supervise::ModuleStatus,
4315    ) -> Result<Frame, RouterError> {
4316        self.counters.increment_route_open_refused(code);
4317        info!(
4318            target: "control",
4319            code,
4320            reason = "supervised_not_registered",
4321            module_id = ?module_id,
4322            connection_id = ctx.connection_id.get(),
4323            principal = %caller.principal,
4324            scope = %caller.scope,
4325            harness = %caller.harness,
4326            state = %status.state,
4327            enabled = status.enabled,
4328            live = status.live,
4329            "route.open refused"
4330        );
4331        // A supervised module whose process has not registered is not
4332        // serving, whatever the reason; the supervisor knows this id, so it is
4333        // safe to track.
4334        self.route_outages
4335            .record_not_serving(module_id, "supervised_not_registered");
4336        let message = if let Some(pid) = status.waiting_for_previous_pid {
4337            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4338                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4339            })
4340        } else {
4341            format!(
4342                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4343                status.state, status.enabled, status.live
4344            )
4345        };
4346        control_error_frame(frame, code, message)
4347    }
4348
4349    async fn handle_route_open(
4350        &self,
4351        ctx: &RouteCtx,
4352        frame: Frame,
4353        request: RouteOpenRequest,
4354        timings: &mut RouteOpenTimings,
4355    ) -> Result<Vec<Frame>, RouterError> {
4356        let RouteOpenRequest {
4357            target,
4358            mut identity,
4359            consumer_identity,
4360            consumer_capabilities,
4361            role_versions,
4362            admission_facts,
4363            scope,
4364        } = request;
4365        let target_module_id = target_module_id(&target).to_string();
4366        timings.module_id.clone_from(&target_module_id);
4367        if self
4368            .registry
4369            .get_module_by_connection(ctx.connection_id)
4370            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4371            .is_some()
4372        {
4373            return Ok(vec![control_error_frame(
4374                &frame,
4375                "invalid_request",
4376                "module connections cannot open client routes",
4377            )?]);
4378        }
4379        debug!(
4380            connection_id = ctx.connection_id.get(),
4381            corr = frame.header.corr,
4382            module_id = %target_module_id,
4383            "handling route.open"
4384        );
4385
4386        // The caller's attested principal, requested scope and harness, for
4387        // every `route.open refused` line below. The launch nonce is checked
4388        // here, once, and the answer is reused where the principal is
4389        // enforced further down, so the order of refusals is unchanged.
4390        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4391        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4392
4393        // A malformed declaration is refused first, before anything about the
4394        // target is looked up: the same body would be refused against any
4395        // module, so the caller learns nothing by retrying or waiting. An empty
4396        // map declares nothing and travels as no field at all, so a provider
4397        // only ever sees a missing field or a non-empty one.
4398        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4399        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4400            self.observe_route_open_refusal(
4401                ctx,
4402                &caller,
4403                &target_module_id,
4404                "invalid_role_versions",
4405                error_codes::INVALID_REQUEST,
4406            );
4407            return Ok(vec![control_error_body_frame(
4408                &frame,
4409                ErrorBody {
4410                    code: error_codes::INVALID_REQUEST.to_string(),
4411                    message: error.to_string(),
4412                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4413                },
4414            )?]);
4415        }
4416
4417        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4418            if status.waiting_for_previous_pid.is_some() {
4419                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4420                    ctx,
4421                    &caller,
4422                    &frame,
4423                    &target_module_id,
4424                    "module_warming",
4425                    &status,
4426                )?]);
4427            }
4428        }
4429
4430        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4431        // opposite. Below, a caller learns whether a module is unregistered,
4432        // supervised-but-down (with state/enabled/live), or registered without the
4433        // requested role. Elsewhere that is an enumeration leak: a probe learning
4434        // the shape of a fleet it cannot otherwise see.
4435        //
4436        // It is not one here, and the reason is the ACCESS MODEL rather than
4437        // anything about these errors. Reaching route.open requires the
4438        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4439        // connection file, so any caller who completes it already runs as this
4440        // user -- and can read subc.jsonc for the module list and `ck module
4441        // status` for live state. The reply discloses nothing the caller cannot
4442        // read more easily from disk, while the precision is load-bearing:
4443        // `unknown_module` is retryable and a missing role is not.
4444        //
4445        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4446        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4447        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4448        let Some(registration) = self
4449            .registry
4450            .get_module(&target_module_id)
4451            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4452        else {
4453            if let Some((status, warming)) =
4454                self.supervisor_status(&target_module_id, frame.header.corr)?
4455            {
4456                // BEFORE the two availability codes below, because for a module
4457                // that speaks no subc wire both of them are false comfort: they
4458                // say "not right now" and are retried, and this module will
4459                // never register no matter how long the caller waits. The
4460                // absence here is the declaration being honoured, not a module
4461                // that is late.
4462                if status.protocol == ModuleProtocol::None {
4463                    return Ok(vec![self.route_open_refusal_frame(
4464                        ctx,
4465                        &caller,
4466                        &frame,
4467                        &target_module_id,
4468                        "protocol_none",
4469                        error_codes::MODULE_NO_PROTOCOL,
4470                        format!(
4471                            "module_id '{target_module_id}' is declared protocol: none; \
4472                             it speaks no subc wire and serves no routes"
4473                        ),
4474                    )?]);
4475                }
4476                let code = if warming {
4477                    "module_warming"
4478                } else {
4479                    "target_unavailable"
4480                };
4481                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4482                    ctx,
4483                    &caller,
4484                    &frame,
4485                    &target_module_id,
4486                    code,
4487                    &status,
4488                )?]);
4489            }
4490            if let Some(removed_ago_ms) =
4491                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4492            {
4493                return Ok(vec![self.route_open_refusal_frame(
4494                    ctx,
4495                    &caller,
4496                    &frame,
4497                    &target_module_id,
4498                    "removed",
4499                    error_codes::MODULE_REMOVED,
4500                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4501                )?]);
4502            }
4503            return Ok(vec![self.route_open_refusal_frame(
4504                ctx,
4505                &caller,
4506                &frame,
4507                &target_module_id,
4508                "not_registered",
4509                error_codes::UNKNOWN_MODULE,
4510                format!("module_id '{target_module_id}' is not registered"),
4511            )?]);
4512        };
4513
4514        // Best-effort only: registry readiness and forwarding reservation use
4515        // different locks, so a module can flip readiness between this read and
4516        // the relay. Modules must still tolerate an `on_bind` while not ready.
4517        if !registration.ready {
4518            self.counters
4519                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4520            info!(
4521                target: "control",
4522                code = error_codes::MODULE_WARMING,
4523                module_id = ?target_module_id,
4524                connection_id = ctx.connection_id.get(),
4525                principal = %caller.principal,
4526                scope = %caller.scope,
4527                harness = %caller.harness,
4528                reason = "declared_not_ready",
4529                "route.open refused"
4530            );
4531            // The module is registered but says it cannot take work, which is
4532            // an outage from the caller's side even though its process is up.
4533            self.route_outages
4534                .record_not_serving(&target_module_id, "declared_not_ready");
4535            return Ok(vec![control_error_body_frame(
4536                &frame,
4537                ErrorBody {
4538                    code: error_codes::MODULE_WARMING.to_string(),
4539                    message: format!(
4540                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4541                    ),
4542                    detail: Some(serde_json::json!({
4543                        "reason": "declared_not_ready"
4544                    })),
4545                },
4546            )?]);
4547        }
4548
4549        // Effective readiness, second half: a module that declares a capability
4550        // `need: required` is not routable while that capability has no
4551        // registered provider. It is enforced HERE, as a retryable routing
4552        // refusal, and deliberately not as spawn ordering or a boot block. The
4553        // module is still started and registered and can make its own calls;
4554        // spawn ordering is a promise that cannot be kept once a provider
4555        // crashes at runtime, and refusing to boot would stop the whole
4556        // machine, including the tools needed to fix its configuration.
4557        //
4558        // "Provided" is the evaluator's verdict, which counts a provider as
4559        // soon as it has REGISTERED, not once it is ready. Two modules that
4560        // require each other's capabilities are therefore both routable once
4561        // both register; counting readiness instead would deadlock them.
4562        //
4563        // Only new opens are refused. Routes already bound when a provider
4564        // goes away stay bound: nothing here tears them down, and the module
4565        // answers them as it can. Like the readiness read above this is
4566        // best-effort against a provider registering or leaving concurrently.
4567        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4568            self.counters
4569                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4570            info!(
4571                target: "control",
4572                code = error_codes::MODULE_WARMING,
4573                module_id = ?target_module_id,
4574                connection_id = ctx.connection_id.get(),
4575                principal = %caller.principal,
4576                scope = %caller.scope,
4577                harness = %caller.harness,
4578                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4579                capability = %capability,
4580                "route.open refused"
4581            );
4582            return Ok(vec![control_error_body_frame(
4583                &frame,
4584                ErrorBody {
4585                    code: error_codes::MODULE_WARMING.to_string(),
4586                    message: format!(
4587                        "module_id '{target_module_id}' requires capability '{capability}', \
4588                         which no registered module provides; retry"
4589                    ),
4590                    detail: Some(serde_json::json!({
4591                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4592                        "capability": capability,
4593                    })),
4594                },
4595            )?]);
4596        }
4597
4598        if !target_has_required_role(&target, &registration.manifest.provides) {
4599            return Ok(vec![self.route_open_refusal_frame(
4600                ctx,
4601                &caller,
4602                &frame,
4603                &target_module_id,
4604                "role_not_provided",
4605                "target_unavailable",
4606                format!("module_id '{target_module_id}' does not provide the requested target"),
4607            )?]);
4608        }
4609
4610        if registration.state != ChannelState::Active {
4611            return Ok(vec![self.route_open_refusal_frame(
4612                ctx,
4613                &caller,
4614                &frame,
4615                &target_module_id,
4616                "registration_not_active",
4617                "target_unavailable",
4618                format!("module_id '{target_module_id}' is not active"),
4619            )?]);
4620        }
4621
4622        if self
4623            .forwarding
4624            .module_is_draining(&target_module_id)
4625            .map_err(RouterError::Forwarding)?
4626        {
4627            return Ok(vec![self.route_open_refusal_frame(
4628                ctx,
4629                &caller,
4630                &frame,
4631                &target_module_id,
4632                "reloading",
4633                "module_reloading",
4634                format!("module_id '{target_module_id}' is reloading"),
4635            )?]);
4636        }
4637
4638        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4639            process_liveness.process_live(&target_module_id) == Some(false)
4640        }) {
4641            // A module the supervisor is restarting or reloading can still hold
4642            // a registration: the old process before its connection closes, or
4643            // a new one that registered while the supervisor was draining. The
4644            // forwarding table does not see that as draining, but the consumer
4645            // should still be told to retry soon, exactly as for the drain
4646            // above, rather than that the target is unavailable.
4647            if process_liveness.process_replacing(&target_module_id) {
4648                return Ok(vec![self.route_open_refusal_frame(
4649                    ctx,
4650                    &caller,
4651                    &frame,
4652                    &target_module_id,
4653                    "reloading",
4654                    "module_reloading",
4655                    format!("module_id '{target_module_id}' is reloading"),
4656                )?]);
4657            }
4658            return Ok(vec![self.route_open_refusal_frame(
4659                ctx,
4660                &caller,
4661                &frame,
4662                &target_module_id,
4663                "supervisor_not_live",
4664                "target_unavailable",
4665                format!("module_id '{target_module_id}' is not live"),
4666            )?]);
4667        }
4668
4669        if !self
4670            .forwarding
4671            .has_live_module_connection(&target_module_id)
4672            .map_err(RouterError::Forwarding)?
4673        {
4674            return Ok(vec![self.route_open_refusal_frame(
4675                ctx,
4676                &caller,
4677                &frame,
4678                &target_module_id,
4679                "no_forwarding_connection",
4680                "target_unavailable",
4681                format!("module_id '{target_module_id}' has no live forwarding connection"),
4682            )?]);
4683        }
4684
4685        if let Some(error) =
4686            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4687        {
4688            self.observe_route_open_refusal(
4689                ctx,
4690                &caller,
4691                &target_module_id,
4692                "op_not_allowed",
4693                "op_not_allowed",
4694            );
4695            return Ok(vec![error]);
4696        }
4697
4698        let principal =
4699            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4700                Ok(principal) => principal,
4701                Err(error) => {
4702                    self.observe_route_open_refusal(
4703                        ctx,
4704                        &caller,
4705                        &target_module_id,
4706                        "bad_consumer_identity",
4707                        "bad_consumer_identity",
4708                    );
4709                    return Ok(vec![error]);
4710                }
4711            };
4712        timings.principal = principal_label(&principal);
4713
4714        // This is attested, control-plane policy for supervised module origins.
4715        // Keep it before route reservation and out of the opaque forwarding hot
4716        // path: data frames must never acquire a per-frame capability check.
4717        if let Principal::Reserved {
4718            module_id: opening_module_id,
4719        } = &principal
4720        {
4721            if let Some(opening_registration) = self
4722                .registry
4723                .get_module(opening_module_id)
4724                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4725            {
4726                if let Some(capability) =
4727                    denied_capability(&opening_registration.manifest, &registration.manifest)
4728                {
4729                    warn!(
4730                        opening_module_id,
4731                        target_module_id,
4732                        capability,
4733                        "refusing route.open because an attested capability deny edge matches"
4734                    );
4735                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4736                        &frame,
4737                        &target_module_id,
4738                        "capability_deny_edge",
4739                        "capability_forbidden",
4740                        format!(
4741                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4742                        ),
4743                    )?]);
4744                }
4745            }
4746        }
4747
4748        if admission_facts.is_some() {
4749            let carrier_matches = matches!(
4750                &principal,
4751                Principal::Reserved { module_id }
4752                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4753            );
4754            if !carrier_matches {
4755                return Ok(vec![self.route_open_refusal_frame(
4756                    ctx,
4757                    &caller,
4758                    &frame,
4759                    &target_module_id,
4760                    "admission_facts_carrier_not_permitted",
4761                    "admission_facts_not_permitted",
4762                    "admission facts may only be carried by the configured reserved module",
4763                )?]);
4764            }
4765
4766            let target_allowed = self
4767                .admission_facts_targets
4768                .as_ref()
4769                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4770            if !target_allowed {
4771                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4772                    &frame,
4773                    &target_module_id,
4774                    "admission_facts_target_not_listed",
4775                    "admission_facts_target_not_allowed",
4776                    format!(
4777                        "admission facts are not permitted for target module_id '{target_module_id}'"
4778                    ),
4779                )?]);
4780            }
4781
4782            // Keep the value opaque to subc. The downstream admission validator owns
4783            // schema and semantic checks; this daemon only enforces carrier authority
4784            // and the configured destination allowlist.
4785        }
4786
4787        // Scope admission, on the attested principal above and never on the
4788        // request body. The tag read here travels with the pending bind and is
4789        // compared with the published one at commit, so a sync between here
4790        // and the module's ack refuses the open instead of binding a stamp
4791        // that is no longer true.
4792        let (bound_scope, mut scope_stamp) = match &scope {
4793            None => (None, None),
4794            Some(selector) => {
4795                let owner_configured = match &selector.owner {
4796                    // A reserved owner counts as configured from before its
4797                    // process is spawned (see `SupervisorHandle::is_configured`).
4798                    // So an owner that has not synced its scopes yet is refused
4799                    // as retryable (`scope_not_synced`), not as one that will
4800                    // never sync.
4801                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4802                    _ => false,
4803                };
4804                let mut scopes = self
4805                    .scopes
4806                    .write()
4807                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4808                let admitted = scopes
4809                    .admit(&principal, &target_module_id, selector, owner_configured)
4810                    .and_then(|admission| {
4811                        crate::scopes::check_target_flow_support(
4812                            &admission.stamp,
4813                            &target_module_id,
4814                            registration.manifest.capabilities.as_ref(),
4815                        )?;
4816                        crate::scopes::check_target_agent_run_support(
4817                            &admission.stamp,
4818                            &target_module_id,
4819                            registration.manifest.capabilities.as_ref(),
4820                        )?;
4821                        let bound = scopes.reserve_bind(
4822                            &admission,
4823                            &principal,
4824                            ctx.connection_id,
4825                            selector.hold_presence,
4826                        );
4827                        Ok((admission, bound))
4828                    });
4829                match admitted {
4830                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4831                    Err(refusal) => {
4832                        return Ok(vec![self.route_open_refusal_frame(
4833                            ctx,
4834                            &caller,
4835                            &frame,
4836                            &target_module_id,
4837                            refusal.code,
4838                            refusal.code,
4839                            refusal.message,
4840                        )?]);
4841                    }
4842                }
4843            }
4844        };
4845        // From here on a refusal names the scope by the stamp admission
4846        // issued, exactly as the `route.open accepted` line does.
4847        if let Some(stamp) = &scope_stamp {
4848            caller.scope = route_open_scope_label(stamp);
4849        }
4850
4851        // Bind admits a root that no longer exists on disk, because refusing here
4852        // closes the only exit from a paused run: cancel needs a bound route, and a
4853        // renamed or reclaimed directory makes that route unopenable forever. The
4854        // run itself is intact and still addressable by its recorded identity.
4855        //
4856        // This does NOT relax the rule the strict constructor protects. That rule is
4857        // that no root is ever aliased into NEW durable state -- a missing component
4858        // can reappear as a symlink elsewhere, which would move the identity and
4859        // split a session's history across two of them. The engine now refuses the
4860        // two operations that create such state (send and import) at admission,
4861        // which is a narrower way to hold the same invariant: reads and terminations
4862        // are admitted, writes are not. That refusal had to ship before this line
4863        // changed, or there is an interval where a send commits under a provisional
4864        // identity -- the exact failure the original policy existed to prevent.
4865        //
4866        // Resolution follows realpath rather than lexical cleanup: the longest
4867        // existing ancestor is canonicalized and the missing tail re-appended, so a
4868        // live root is unchanged and a vanished leaf keeps the identity it was
4869        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4870        // same caller the moment the directory vanished, which strands the run more
4871        // quietly than refusing it.
4872        timings.enter(RouteOpenPhase::ProjectRoot);
4873        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4874            Ok(project_root) => project_root,
4875            Err(err) => {
4876                return Ok(vec![control_error_frame(
4877                    &frame,
4878                    "invalid_project_root",
4879                    err.to_string(),
4880                )?])
4881            }
4882        };
4883        timings.enter(RouteOpenPhase::Admission);
4884        identity.project_root = project_root.as_path().to_path_buf();
4885
4886        // Last gate before any relay work, and deliberately after the cheap
4887        // registry and availability checks above: those name a more precise
4888        // condition (unknown, removed, reloading) and a caller is better served
4889        // by the precise code than by this one.
4890        //
4891        // Everything below this point costs an egress permit, a reserved handle
4892        // pair and, if the module does not answer, the whole relay budget. The
4893        // reader no longer waits for that budget, so cap each target explicitly;
4894        // serial dispatch used to provide the accidental cap of one relay per
4895        // connection. Admission is a mutex-protected count and never waits.
4896        let _concurrency_guard = match self
4897            .route_bind_concurrency
4898            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4899        {
4900            Ok(guard) => guard,
4901            Err(in_flight) => {
4902                return Ok(vec![self.route_open_target_capacity_refusal(
4903                    ctx,
4904                    &caller,
4905                    &frame,
4906                    &target_module_id,
4907                    in_flight,
4908                )?]);
4909            }
4910        };
4911
4912        // A module that has already burned the whole budget `threshold` times
4913        // in a row does not get to charge it again until a probe says it recovered.
4914        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4915            RouteBindAdmission::Admitted { guard, probe } => {
4916                if probe {
4917                    info!(
4918                        module_id = %target_module_id,
4919                        connection_id = ctx.connection_id.get(),
4920                        "route.bind breaker half-open: admitting one probe"
4921                    );
4922                }
4923                guard
4924            }
4925            RouteBindAdmission::Refused {
4926                consecutive_timeouts,
4927                retry_in,
4928                probe_in_flight,
4929            } => {
4930                return Ok(vec![self.route_open_breaker_refusal_frame(
4931                    ctx,
4932                    &caller,
4933                    &frame,
4934                    &target_module_id,
4935                    consecutive_timeouts,
4936                    retry_in,
4937                    probe_in_flight,
4938                )?]);
4939            }
4940        };
4941
4942        // Resolve the per-module budget here so the wait matches the operator's
4943        // intent for this specific target. A per-module override in
4944        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4945        // daemons) wins over the daemon-wide default.
4946        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4947        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4948        timings.enter(RouteOpenPhase::EgressReservation);
4949        let pending = match self
4950            .forwarding
4951            .begin_route_bind_relay_for(
4952                ctx.connection_id,
4953                ctx.egress.clone(),
4954                response_version(&frame),
4955                frame.header.corr,
4956                &target_module_id,
4957                principal.clone(),
4958                bound_scope,
4959                Some(project_root),
4960                relay_deadline,
4961            )
4962            .await
4963        {
4964            Ok(pending) => pending,
4965            Err(err) => {
4966                return Ok(vec![self.route_open_refusal_frame(
4967                    ctx,
4968                    &caller,
4969                    &frame,
4970                    &target_module_id,
4971                    "relay_reservation_failed",
4972                    forwarding_error_code(&err),
4973                    err.to_string(),
4974                )?])
4975            }
4976        };
4977        timings.enter(RouteOpenPhase::Admission);
4978        let crate::forwarding::PendingRouteBindRelay {
4979            endpoint,
4980            module_sink,
4981            negotiated_ver,
4982            client_channel,
4983            client_epoch,
4984            module_channel,
4985            module_epoch,
4986            corr: relay_corr,
4987            receiver,
4988        } = pending;
4989        let mut reservation =
4990            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4991
4992        // Reserving egress can wait while a module reconnects or a swap cuts
4993        // over. Check the relay's captured connection, not the earlier by-id
4994        // lookup: the original module's flow/run capabilities cannot authorize
4995        // its replacement. The captured sink stays bound to that connection.
4996        let relay_registration = self
4997            .registry
4998            .get_module_by_connection(endpoint.connection_id)
4999            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5000        let relay_capabilities = relay_registration
5001            .as_ref()
5002            .and_then(|registration| registration.manifest.capabilities.as_ref());
5003        if let Some(selector) = &scope {
5004            // Egress reservation may wait across a scope update. Recheck the
5005            // proof against the current record before sending the captured bind.
5006            if let Err(refusal) = self
5007                .scopes
5008                .read()
5009                .unwrap_or_else(|poisoned| poisoned.into_inner())
5010                .check_bind_proof(&principal, selector)
5011            {
5012                reservation.release_and_disarm();
5013                return Ok(vec![self.route_open_refusal_frame(
5014                    ctx,
5015                    &caller,
5016                    &frame,
5017                    &target_module_id,
5018                    refusal.code,
5019                    refusal.code,
5020                    refusal.message,
5021                )?]);
5022            }
5023        }
5024        if let Some(stamp) = scope_stamp
5025            .as_ref()
5026            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5027        {
5028            if let Err(refusal) = crate::scopes::check_target_flow_support(
5029                stamp,
5030                &target_module_id,
5031                relay_capabilities,
5032            )
5033            .and_then(|()| {
5034                crate::scopes::check_target_agent_run_support(
5035                    stamp,
5036                    &target_module_id,
5037                    relay_capabilities,
5038                )
5039            }) {
5040                reservation.release_and_disarm();
5041                return Ok(vec![self.route_open_refusal_frame(
5042                    ctx,
5043                    &caller,
5044                    &frame,
5045                    &target_module_id,
5046                    refusal.code,
5047                    refusal.code,
5048                    refusal.message,
5049                )?]);
5050            }
5051        }
5052
5053        // Older readers may reject unknown fields. Select optional fields from
5054        // the captured endpoint's declaration, including on unscoped binds.
5055        if !relay_capabilities.is_some_and(|caps| {
5056            caps.provides
5057                .iter()
5058                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5059        }) {
5060            if let Some(stamp) = scope_stamp.as_mut() {
5061                stamp.workdir = None;
5062                stamp.project_id = None;
5063            }
5064        }
5065        // A worker run has no agent_id; on_behalf_of_agent is its only agent
5066        // attribution. The run gate above has already required the target to
5067        // declare `worker-runs/v1`, which promises to read both
5068        // `on_behalf_of_agent` and `call_mode`.
5069        let worker_run = scope_stamp
5070            .as_ref()
5071            .is_some_and(crate::scopes::is_worker_run)
5072            && relay_capabilities.is_some_and(|caps| {
5073                caps.provides
5074                    .iter()
5075                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5076            });
5077        if !worker_run
5078            && !relay_capabilities.is_some_and(|caps| {
5079                caps.provides
5080                    .iter()
5081                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5082            })
5083        {
5084            if let Some(stamp) = scope_stamp.as_mut() {
5085                stamp.attributes.on_behalf_of_agent = None;
5086            }
5087        }
5088        if !relay_capabilities.is_some_and(|caps| {
5089            caps.provides
5090                .iter()
5091                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5092        }) {
5093            if let Some(stamp) = scope_stamp.as_mut() {
5094                stamp.attributes.flow_version = None;
5095                stamp.attributes.flow_code_hash = None;
5096                stamp.attributes.flow_install_id = None;
5097            }
5098        }
5099        let call_mode = if worker_run
5100            || relay_capabilities
5101                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5102        {
5103            Some(derive_call_mode(scope_stamp.as_ref()))
5104        } else {
5105            if let Some(stamp) = scope_stamp.as_mut() {
5106                stamp.expires_at_ms = None;
5107            }
5108            None
5109        };
5110
5111        debug!(
5112            connection_id = ctx.connection_id.get(),
5113            client_channel,
5114            client_epoch,
5115            module_channel,
5116            module_epoch,
5117            "reserved route handle pair"
5118        );
5119        // Rendered BEFORE the move into the relay, because the accept arm below
5120        // is where it is logged and the principal is gone by then.
5121        let route_principal_label = principal_label(&principal);
5122        // The opener's build class is an unattested claim meant only for the
5123        // daemon's own prompt decision. A module must never see it, so that no
5124        // module can come to rely on a claim any local process could make.
5125        identity.build_class = None;
5126        let relay = ModuleControlRequest::RouteBind {
5127            route_channel: module_channel,
5128            epoch: module_epoch,
5129            target,
5130            identity,
5131            principal: Some(principal),
5132            consumer_capabilities,
5133            role_versions,
5134            admission_facts,
5135            scope: scope_stamp,
5136            call_mode,
5137        };
5138        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5139            RouterError::backend(
5140                0,
5141                frame.header.corr,
5142                format!("failed to encode route.bind request: {err}"),
5143            )
5144        })?;
5145        let relay_frame = Frame::build_with_version(
5146            negotiated_ver,
5147            FrameType::Request,
5148            control_flags(),
5149            0,
5150            0,
5151            relay_corr,
5152            relay_body,
5153        )
5154        .map_err(RouterError::FrameBuild)?;
5155
5156        timings.enter(RouteOpenPhase::BindSend);
5157        let sent = module_sink.send(relay_frame).await;
5158        timings.enter(RouteOpenPhase::Admission);
5159        if let Err(err) = sent {
5160            reservation.release_and_disarm();
5161            return Ok(vec![self.route_open_refusal_frame(
5162                ctx,
5163                &caller,
5164                &frame,
5165                &target_module_id,
5166                "relay_send_failed",
5167                "target_unavailable",
5168                err.to_string(),
5169            )?]);
5170        }
5171
5172        #[cfg(test)]
5173        if let Some(hold) = &self.route_bind_relay_sent_hold {
5174            // First wait: tell the test the bind is queued. Second wait: stay
5175            // here until the test has driven whatever it wants to happen
5176            // between queueing the bind and marking it enqueued.
5177            hold.wait().await;
5178            hold.wait().await;
5179        }
5180
5181        // A missing relay is not by itself an abandoned bind: the module may
5182        // already have acked it and the route may be committed. Only a relay
5183        // dropped without any module reply needs this GOODBYE; one a reply
5184        // settled has already had its GOODBYE decided by the reply path.
5185        if self
5186            .forwarding
5187            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5188            .map_err(RouterError::Forwarding)?
5189            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5190        {
5191            self.send_abandoned_route_bind_goodbye(
5192                &module_sink,
5193                negotiated_ver,
5194                module_channel,
5195                module_epoch,
5196            );
5197        }
5198
5199        timings.enter(RouteOpenPhase::BindAckWait);
5200        let outcome = timeout_at(relay_deadline, receiver).await;
5201        timings.enter(RouteOpenPhase::Admission);
5202        match outcome {
5203            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5204                reservation.disarm();
5205                if breaker.record_accepted() {
5206                    info!(
5207                        module_id = %target_module_id,
5208                        "route.bind breaker closed: the probe was accepted"
5209                    );
5210                }
5211                self.observe_route_open_accept(
5212                    ctx,
5213                    &target_module_id,
5214                    &route_principal_label,
5215                    &caller.scope,
5216                );
5217                Ok(Vec::new())
5218            }
5219            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5220                reservation.release_and_disarm();
5221                // A module that says no in microseconds is healthy. Rejection
5222                // is a different condition with its own refusal and must not
5223                // move the breaker.
5224                breaker.record_inconclusive();
5225                // The daemon's own commit re-check refused the bind because the
5226                // scope ended or changed after admission. The module accepted;
5227                // counting it as a module rejection would blame the module.
5228                let scope_code = match body.code.as_str() {
5229                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5230                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5231                    _ => None,
5232                };
5233                if let Some(code) = scope_code {
5234                    self.observe_route_open_refusal(
5235                        ctx,
5236                        &caller,
5237                        &target_module_id,
5238                        "scope_changed_before_commit",
5239                        code,
5240                    );
5241                    return Ok(vec![control_error_body_frame(&frame, body)?]);
5242                }
5243                self.counters
5244                    .increment_route_open_refused("module_rejected");
5245                info!(
5246                    target: "control",
5247                    code = "module_rejected",
5248                    module_code = ?body.code,
5249                    module_id = ?target_module_id,
5250                    connection_id = ctx.connection_id.get(),
5251                    principal = %caller.principal,
5252                    scope = %caller.scope,
5253                    harness = %caller.harness,
5254                    "route.open refused"
5255                );
5256                Ok(vec![control_error_body_frame(&frame, body)?])
5257            }
5258            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5259                reservation.release_and_disarm();
5260                breaker.record_inconclusive();
5261                // Fires when the module's connection closes while a relayed
5262                // bind is pending -- typically a caller racing a module restart
5263                // whose bind was relayed BEFORE the drain mark went up. Logged
5264                // because the caller sees only its own error and the fleet has
5265                // already spent one diagnosis round unable to tell this arm
5266                // from a relay timeout without daemon-side evidence.
5267                tracing::warn!(
5268                    module_id = %target_module_id,
5269                    "route.bind relay abandoned: {message}"
5270                );
5271                Ok(vec![self.route_open_refusal_frame(
5272                    ctx,
5273                    &caller,
5274                    &frame,
5275                    &target_module_id,
5276                    "relay_abandoned",
5277                    "target_unavailable",
5278                    message,
5279                )?])
5280            }
5281            Ok(Err(_)) => {
5282                reservation.release_and_disarm();
5283                breaker.record_inconclusive();
5284                Ok(vec![self.route_open_refusal_frame(
5285                    ctx,
5286                    &caller,
5287                    &frame,
5288                    &target_module_id,
5289                    "relay_waiter_canceled",
5290                    "target_unavailable",
5291                    "route.bind relay waiter was canceled before the module responded",
5292                )?])
5293            }
5294            Err(_) => {
5295                reservation.release_and_disarm();
5296                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5297                // answer at all is the one condition a fast refusal can
5298                // usefully stand in for; every other arm already answered.
5299                if let Some(opened) = breaker.record_timeout(
5300                    self.route_bind_breaker_threshold,
5301                    self.route_bind_breaker_cooldown,
5302                ) {
5303                    warn!(
5304                        module_id = %target_module_id,
5305                        consecutive_timeouts = opened.consecutive_timeouts,
5306                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5307                        reopened_after_probe = opened.reopened_after_probe,
5308                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5309                    );
5310                }
5311                // The generous budget just burned to no answer: the module is
5312                // registered and its connection is up, but its bind handler sat
5313                // on the ack for the full budget (warm-on-bind, cold configure,
5314                // or a wedged handler). Every earlier unavailability shape
5315                // fast-refuses BEFORE the relay, so this arm firing means the
5316                // slowness is module-side -- log it so the per-module timeline
5317                // is reconstructable without client audit rows.
5318                tracing::warn!(
5319                    module_id = %target_module_id,
5320                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5321                    "route.bind relay timed out: module did not ack within budget"
5322                );
5323                Ok(vec![self.route_open_refusal_frame(
5324                    ctx,
5325                    &caller,
5326                    &frame,
5327                    &target_module_id,
5328                    "relay_timed_out",
5329                    "module_timeout",
5330                    format!(
5331                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5332                        route_bind_relay_timeout
5333                    ),
5334                )?])
5335            }
5336        }
5337    }
5338
5339    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5340        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5341            snapshot: self.supervisor.spawn_snapshot(),
5342        };
5343        Ok(vec![control_response_body_frame(
5344            &frame,
5345            &response,
5346            "ClientControlResponse::SupervisorSpawnSnapshot",
5347        )?])
5348    }
5349
5350    fn handle_supervisor_spawn_subscribe(
5351        &self,
5352        ctx: &RouteCtx,
5353        frame: Frame,
5354        since: Option<SpawnCursor>,
5355    ) -> Result<Vec<Frame>, RouterError> {
5356        match self.supervisor.subscribe_spawns(
5357            ctx.connection_id,
5358            frame.header.corr,
5359            response_version(&frame),
5360            since,
5361            ctx.egress.clone(),
5362        ) {
5363            Ok(()) => Ok(Vec::new()),
5364            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5365                Ok(vec![control_error_body_frame(
5366                    &frame,
5367                    ErrorBody {
5368                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5369                        message: "spawn cursor belongs to a different daemon incarnation"
5370                            .to_string(),
5371                        detail: Some(serde_json::json!({
5372                            "current_daemon_incarnation": current
5373                        })),
5374                    },
5375                )?])
5376            }
5377            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5378                &frame,
5379                ErrorBody {
5380                    code: "spawn_cursor_too_old".to_string(),
5381                    message: "spawn cursor predates the retained event ring".to_string(),
5382                    detail: Some(serde_json::json!({
5383                        "oldest_retained_cursor": oldest
5384                    })),
5385                },
5386            )?]),
5387            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5388                0,
5389                frame.header.corr,
5390                format!("failed to open supervisor spawn subscription: {error}"),
5391            )),
5392        }
5393    }
5394
5395    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5396        let generation = self
5397            .registry
5398            .generation()
5399            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5400        let mut modules = Vec::new();
5401        for module in self.supervisor.list() {
5402            let status = module.status_for_control("list").map_err(|err| {
5403                RouterError::backend(
5404                    0,
5405                    frame.header.corr,
5406                    format!("failed to read supervisor status: {err}"),
5407                )
5408            })?;
5409            let (configured, _) = module.configuration().map_err(|err| {
5410                RouterError::backend(
5411                    0,
5412                    frame.header.corr,
5413                    format!("failed to read module configuration: {err}"),
5414                )
5415            })?;
5416            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5417                RouterError::backend(
5418                    0,
5419                    frame.header.corr,
5420                    format!("failed to read retry time: {err}"),
5421                )
5422            })?;
5423            // Status and configuration snapshots release their locks before the image probe awaits.
5424            let image = module.running_image_agreement().await;
5425            // Read per request so the figure is current when the operator asks;
5426            // the daemon samples nothing in between.
5427            let resources = Some(module.child_resource_usage());
5428            let mut verdict =
5429                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5430            verdict.spec = Some(launch_spec_agreement(
5431                &configured,
5432                status.process_alive,
5433                status.launch_spec_digest.as_ref(),
5434            ));
5435            let pending_reload = Some(verdict);
5436            let configured_enabled = self
5437                .capability_evaluator
5438                .configured_enabled(&status.module_id);
5439            // Keep the retired policy field on the wire for one release so
5440            // existing status consumers still receive the platform policy.
5441            modules.push(
5442                SupervisorEntry::new(
5443                    status.module_id,
5444                    status.state.to_string(),
5445                    status.enabled,
5446                    status.live,
5447                    status.health.status,
5448                )
5449                .with_configured_enabled(configured_enabled)
5450                .with_next_retry_at_ms(next_retry_at_ms)
5451                .with_launch_nonce_env(Some(!cfg!(unix)))
5452                .with_launch_nonce_source(status.launch_nonce_source)
5453                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5454                .with_protocol(status.protocol)
5455                .with_pending_reload(pending_reload)
5456                .with_last_probe_ms(status.health.last_probe_ms)
5457                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5458                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5459                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5460                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5461                .with_restart_count(Some(status.restart_count))
5462                .with_max_restarts(Some(status.max_restarts))
5463                .with_lifetime_restarts(Some(status.lifetime_restarts))
5464                .with_spawn_generation(Some(status.spawn_generation))
5465                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5466                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5467                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5468                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5469                .with_resources(resources),
5470            );
5471        }
5472        let response = ClientControlResponse::SupervisorList {
5473            generation,
5474            modules,
5475        };
5476        Ok(vec![control_response_body_frame(
5477            &frame,
5478            &response,
5479            "ClientControlResponse::SupervisorList",
5480        )?])
5481    }
5482
5483    fn handle_supervisor_stderr_tail(
5484        &self,
5485        frame: Frame,
5486        module_id: String,
5487        max_lines: Option<u32>,
5488        max_bytes: Option<u32>,
5489    ) -> Result<Vec<Frame>, RouterError> {
5490        let Some(module) = self.supervisor.get(&module_id) else {
5491            return Ok(vec![control_error_frame(
5492                &frame,
5493                "unknown_module",
5494                format!("module_id '{module_id}' is not supervised"),
5495            )?]);
5496        };
5497
5498        let snapshot = module.stderr_tail(
5499            max_lines.map(|value| value as usize),
5500            max_bytes.map(|value| value as usize),
5501        );
5502
5503        let response = ClientControlResponse::SupervisorStderrTail {
5504            module_id,
5505            tail: StderrTail {
5506                capture: match snapshot.capture {
5507                    CaptureState::Captured => StderrCaptureState::Captured,
5508                    CaptureState::Incomplete { reason } => {
5509                        StderrCaptureState::Incomplete { reason }
5510                    }
5511                    CaptureState::NotCaptured { reason } => {
5512                        StderrCaptureState::NotCaptured { reason }
5513                    }
5514                },
5515                entries: snapshot
5516                    .entries
5517                    .into_iter()
5518                    .map(|entry| match entry {
5519                        TailEntry::Line {
5520                            text,
5521                            truncated,
5522                            at_ms,
5523                        } => StderrTailEntry::Line {
5524                            text,
5525                            truncated,
5526                            at_ms,
5527                        },
5528                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5529                    })
5530                    .collect(),
5531                dropped_lines: snapshot.dropped_lines,
5532            },
5533        };
5534        Ok(vec![control_response_body_frame(
5535            &frame,
5536            &response,
5537            "ClientControlResponse::SupervisorStderrTail",
5538        )?])
5539    }
5540
5541    async fn handle_supervisor_terminals(
5542        &self,
5543        frame: Frame,
5544        module_id: String,
5545    ) -> Result<Vec<Frame>, RouterError> {
5546        let Some(module) = self.supervisor.get(&module_id) else {
5547            return Ok(vec![control_error_frame(
5548                &frame,
5549                "unknown_module",
5550                format!("module_id '{module_id}' is not supervised"),
5551            )?]);
5552        };
5553
5554        // The journal read runs on a blocking thread: it can be megabytes of
5555        // file I/O and must not occupy a runtime worker.
5556        let terminals = module
5557            .read_durable_terminal_history()
5558            .await
5559            .map_err(|error| {
5560                RouterError::backend(
5561                    0,
5562                    frame.header.corr,
5563                    format!("failed to read terminal history: {error}"),
5564                )
5565            })?;
5566        let response = ClientControlResponse::SupervisorTerminals {
5567            module_id,
5568            terminals,
5569        };
5570        Ok(vec![control_response_body_frame(
5571            &frame,
5572            &response,
5573            "ClientControlResponse::SupervisorTerminals",
5574        )?])
5575    }
5576
5577    fn handle_supervisor_routes(
5578        &self,
5579        frame: Frame,
5580        module_id: Option<String>,
5581    ) -> Result<Vec<Frame>, RouterError> {
5582        let modules = self
5583            .forwarding
5584            .route_census(module_id.as_deref())
5585            .map_err(RouterError::Forwarding)?
5586            .into_iter()
5587            .map(|(module_id, routes)| SupervisorRouteModule {
5588                module_id,
5589                routes: routes
5590                    .into_iter()
5591                    .map(|route| SupervisorRoute {
5592                        consumer: match route.principal {
5593                            Principal::Reserved { module_id } => {
5594                                SupervisorRouteConsumer::Reserved { module_id }
5595                            }
5596                            Principal::Direct | Principal::Unverified => {
5597                                SupervisorRouteConsumer::Direct {
5598                                    connection_id: route.goodbye_target.connection_id.get(),
5599                                }
5600                            }
5601                        },
5602                        age_ms: Instant::now()
5603                            .saturating_duration_since(route.bound_at)
5604                            .as_millis()
5605                            .try_into()
5606                            .unwrap_or(u64::MAX),
5607                        draining: route.draining,
5608                        drain_reason: route.drain_reason,
5609                    })
5610                    .collect(),
5611            })
5612            .collect();
5613        let response = ClientControlResponse::SupervisorRoutes { modules };
5614        Ok(vec![control_response_body_frame(
5615            &frame,
5616            &response,
5617            "ClientControlResponse::SupervisorRoutes",
5618        )?])
5619    }
5620
5621    async fn handle_supervisor_provenance(
5622        &self,
5623        frame: Frame,
5624        module_id: Option<String>,
5625    ) -> Result<Vec<Frame>, RouterError> {
5626        let mut selected = if let Some(module_id) = module_id {
5627            let Some(module) = self.supervisor.get(&module_id) else {
5628                return Ok(vec![control_error_frame(
5629                    &frame,
5630                    "unknown_module",
5631                    format!("module_id '{module_id}' is not supervised"),
5632                )?]);
5633            };
5634            vec![module]
5635        } else {
5636            self.supervisor.list()
5637        };
5638
5639        let mut modules = Vec::with_capacity(selected.len());
5640        for module in selected.drain(..) {
5641            let (status, observed_image) = module
5642                .status_and_running_image_agreement()
5643                .await
5644                .map_err(|err| {
5645                    RouterError::backend(
5646                        0,
5647                        frame.header.corr,
5648                        format!("failed to read supervisor status: {err}"),
5649                    )
5650                })?;
5651            let module_declared = self
5652                .registry
5653                .get_module(&status.module_id)
5654                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5655                .and_then(|registration| registration.manifest.provenance)
5656                .map(|build| ModuleDeclaredProvenance::Reported { build })
5657                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5658            #[cfg(test)]
5659            let running_image = match &self.provenance_probe_override {
5660                Some(result) => result.clone(),
5661                None => observed_image,
5662            };
5663            #[cfg(not(test))]
5664            let running_image = observed_image;
5665            modules.push(SupervisorModuleProvenance {
5666                module_id: status.module_id,
5667                module_declared,
5668                daemon_observed: SupervisorObservedProcess {
5669                    launch_nonce_source: status.launch_nonce_source,
5670                    pid: status.pid,
5671                    spawned_at_ms: status.spawned_at_ms,
5672                    spawned_from: status.spawned_from,
5673                    running_image,
5674                },
5675            });
5676        }
5677        let daemon = SupervisorDaemonProvenance {
5678            daemon_build: self.daemon_provenance.build.clone(),
5679            daemon_observed: DaemonObservedProcess {
5680                pid: self.daemon_provenance.pid,
5681                started_at_ms: self
5682                    .daemon_provenance
5683                    .start_clock
5684                    .map(|clock| clock.started_at_ms())
5685                    .or(self.daemon_provenance.started_at_ms),
5686                running_image: self
5687                    .daemon_provenance
5688                    .probe
5689                    .observe(
5690                        self.daemon_provenance.pid,
5691                        self.daemon_provenance.executable_path.as_deref(),
5692                        self.daemon_provenance.executable_identity.clone(),
5693                        self.daemon_provenance.process_start_time,
5694                    )
5695                    .await,
5696            },
5697        };
5698        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5699        Ok(vec![control_response_body_frame(
5700            &frame,
5701            &response,
5702            "ClientControlResponse::SupervisorProvenance",
5703        )?])
5704    }
5705
5706    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5707        self.refresh_capability_requirements();
5708        let generation = self
5709            .registry
5710            .generation()
5711            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5712        let modules = self
5713            .supervisor
5714            .list()
5715            .into_iter()
5716            .map(|module| {
5717                let status = module.status_for_control("health").map_err(|err| {
5718                    RouterError::backend(
5719                        0,
5720                        frame.header.corr,
5721                        format!("failed to read supervisor health: {err}"),
5722                    )
5723                })?;
5724                let module_id = status.module_id;
5725                let capability_detail = self
5726                    .capability_evaluator
5727                    .required_problem_detail(&module_id);
5728                Ok(SupervisorHealthEntry {
5729                    module_id,
5730                    status: status.health.status,
5731                    detail: append_capability_problem_detail(
5732                        status.health.detail,
5733                        capability_detail,
5734                    ),
5735                    metrics: status.health.metrics,
5736                    consecutive_failures: status.health.consecutive_failures,
5737                    late_answer_count: status.health.late_answer_count,
5738                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5739                    last_action: status.health.last_action,
5740                    last_action_ms: status.health.last_action_ms,
5741                    last_probe_ms: status.health.last_probe_ms,
5742                })
5743            })
5744            .collect::<Result<Vec<_>, RouterError>>()?;
5745        let response = ClientControlResponse::SupervisorHealth {
5746            generation,
5747            modules,
5748        };
5749        Ok(vec![control_response_body_frame(
5750            &frame,
5751            &response,
5752            "ClientControlResponse::SupervisorHealth",
5753        )?])
5754    }
5755
5756    async fn handle_supervisor_restart(
5757        &self,
5758        frame: Frame,
5759        module_id: String,
5760        drain_timeout_ms: Option<u64>,
5761        requester: Principal,
5762    ) -> Result<Vec<Frame>, RouterError> {
5763        let operation_lock = self.supervisor.operation_lock();
5764        let _operation_guard = operation_lock.lock().await;
5765        let Some(module) = self.supervisor.get(&module_id) else {
5766            return Ok(vec![control_error_frame(
5767                &frame,
5768                "unknown_module",
5769                format!("module_id '{module_id}' is not supervised"),
5770            )?]);
5771        };
5772
5773        self.route_outages.mark_operator_action(&module_id);
5774        if let Err(err) = module
5775            .restart_requested(drain_timeout_ms, Some(requester))
5776            .await
5777        {
5778            self.route_outages
5779                .operator_action_ended_unrefused(&module_id);
5780            let (code, message) = match err {
5781                crate::supervise::SuperviseError::Disabled { .. } => {
5782                    ("module_disabled", err.to_string())
5783                }
5784                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5785                    ("swap_in_progress", err.to_string())
5786                }
5787                _ => (
5788                    "target_unavailable",
5789                    format!("failed to restart module_id '{module_id}': {err}"),
5790                ),
5791            };
5792            return Ok(vec![control_error_frame(&frame, code, message)?]);
5793        }
5794
5795        let response = ClientControlResponse::SupervisorAck {
5796            module_id,
5797            applied: true,
5798        };
5799        Ok(vec![control_response_body_frame(
5800            &frame,
5801            &response,
5802            "ClientControlResponse::SupervisorAck",
5803        )?])
5804    }
5805
5806    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5807    /// when the old process has finished draining: a caller whose own lane
5808    /// rides the old process must get its reply before that drain waits on it.
5809    async fn handle_supervisor_swap(
5810        &self,
5811        frame: Frame,
5812        module_id: String,
5813        ready_timeout_ms: Option<u64>,
5814        requester: Principal,
5815    ) -> Result<Vec<Frame>, RouterError> {
5816        // The daemon-wide operation lock is held only to resolve the handle,
5817        // not across the swap. The swap can take its whole readiness budget,
5818        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5819        // holding it here would park an operator's stop behind the swap it is
5820        // meant to abort. A rescan or stop that reaches the module during the
5821        // swap is served by the swap itself (see `supervise_swap`).
5822        let module = {
5823            let operation_lock = self.supervisor.operation_lock();
5824            let _operation_guard = operation_lock.lock().await;
5825            self.supervisor.get(&module_id)
5826        };
5827        let Some(module) = module else {
5828            return Ok(vec![control_error_frame(
5829                &frame,
5830                "unknown_module",
5831                format!("module_id '{module_id}' is not supervised"),
5832            )?]);
5833        };
5834
5835        self.route_outages.mark_operator_action(&module_id);
5836        if let Err(err) = module
5837            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5838            .await
5839        {
5840            self.route_outages
5841                .operator_action_ended_unrefused(&module_id);
5842            use crate::supervise::SuperviseError;
5843            let message = err.to_string();
5844            let error = match err {
5845                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5846                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5847                    code: "swap_refused".to_string(),
5848                    message,
5849                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5850                },
5851                SuperviseError::SwapFailed {
5852                    arm,
5853                    candidate_exit,
5854                    ..
5855                } => ErrorBody {
5856                    code: "swap_failed".to_string(),
5857                    message,
5858                    detail: Some(serde_json::json!({
5859                        "arm": arm.as_str(),
5860                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5861                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5862                    })),
5863                },
5864                _ => ErrorBody::new(
5865                    "target_unavailable",
5866                    format!("failed to swap module_id '{module_id}': {message}"),
5867                ),
5868            };
5869            return Ok(vec![control_error_body_frame(&frame, error)?]);
5870        }
5871        // A completed swap kept the incumbent serving until cutover, so it
5872        // usually opened no outage; a mark left behind would make the next,
5873        // unrelated outage read as requested.
5874        self.route_outages
5875            .operator_action_ended_unrefused(&module_id);
5876
5877        let response = ClientControlResponse::SupervisorAck {
5878            module_id,
5879            applied: true,
5880        };
5881        Ok(vec![control_response_body_frame(
5882            &frame,
5883            &response,
5884            "ClientControlResponse::SupervisorAck",
5885        )?])
5886    }
5887
5888    async fn handle_supervisor_reload(
5889        &self,
5890        frame: Frame,
5891        module_id: String,
5892        requester: Principal,
5893    ) -> Result<Vec<Frame>, RouterError> {
5894        let operation_lock = self.supervisor.operation_lock();
5895        let _operation_guard = operation_lock.lock().await;
5896        let Some(module) = self.supervisor.get(&module_id) else {
5897            return Ok(vec![control_error_frame(
5898                &frame,
5899                "unknown_module",
5900                format!("module_id '{module_id}' is not supervised"),
5901            )?]);
5902        };
5903
5904        self.route_outages.mark_operator_action(&module_id);
5905        if let Err(err) = module.reload_requested(Some(requester)).await {
5906            self.route_outages
5907                .operator_action_ended_unrefused(&module_id);
5908            let (code, message) = match err {
5909                crate::supervise::SuperviseError::Disabled { .. } => {
5910                    ("module_disabled", err.to_string())
5911                }
5912                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5913                    ("swap_in_progress", err.to_string())
5914                }
5915                _ => (
5916                    "reload_failed",
5917                    format!("failed to reload module_id '{module_id}': {err}"),
5918                ),
5919            };
5920            return Ok(vec![control_error_frame(&frame, code, message)?]);
5921        }
5922
5923        let response = ClientControlResponse::SupervisorAck {
5924            module_id,
5925            applied: true,
5926        };
5927        Ok(vec![control_response_body_frame(
5928            &frame,
5929            &response,
5930            "ClientControlResponse::SupervisorAck",
5931        )?])
5932    }
5933
5934    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5935    /// the configuration comes from and whether to apply it; every mode then
5936    /// runs the same comparison, so a preview describes the apply it previews.
5937    async fn handle_supervisor_rescan(
5938        &self,
5939        frame: Frame,
5940        mode: RescanMode,
5941        requester: Principal,
5942    ) -> Result<Vec<Frame>, RouterError> {
5943        let preview = mode.previews();
5944        let Some(context) = self.rescan.clone() else {
5945            return Ok(vec![control_error_frame(
5946                &frame,
5947                "rescan_unavailable",
5948                "the daemon was not started with a reloadable config path".to_string(),
5949            )?]);
5950        };
5951
5952        let operation_lock = self.supervisor.operation_lock();
5953        let _operation_guard = operation_lock.lock().await;
5954        // Loaded under the operation lock, so no other rescan can change the
5955        // staged configuration between a checked apply's comparison and the
5956        // apply itself.
5957        let loaded = match self.load_rescan_config(&mode, &context) {
5958            Ok(config) => config,
5959            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5960        };
5961        // `load` reports a missing file as Ok(None), which is correct at boot
5962        // (no config, nothing to supervise) and catastrophic here: rescan treats
5963        // "not in the config" as "remove it", so an absent file would read as an
5964        // empty module list and retire the entire running fleet. An editor
5965        // writing via write-new-then-rename, or a half-finished edit, is enough
5966        // to open that window. Refuse instead: a config that cannot be read
5967        // carries no instruction to remove anything.
5968        let Some((config, activity_config)) = loaded else {
5969            return Ok(vec![control_error_frame(
5970                &frame,
5971                "invalid_daemon_config",
5972                format!(
5973                    "daemon config not found at {}; refusing to rescan (an absent config would \
5974                     retire every supervised module)",
5975                    context.config_path.display()
5976                ),
5977            )?]);
5978        };
5979        let (
5980            configured_port,
5981            storage_config,
5982            admission_facts_carrier_module_id,
5983            admission_facts_targets,
5984            scope_authority_owners,
5985            privileged_operator_prompt,
5986            modules,
5987            reserved_capabilities,
5988        ) = (
5989            config.port,
5990            config.storage,
5991            config.admission_facts_carrier_module_id,
5992            config.admission_facts_targets,
5993            config.scope_authority_owners,
5994            config.privileged_operator_prompt,
5995            config.modules,
5996            config.reserved_capabilities,
5997        );
5998
5999        // Collect the sections rescan cannot apply, so the REPLY carries them.
6000        //
6001        // The warning below has always been correct and has always gone only to
6002        // the journal -- addressed to whoever reads logs, while the person who
6003        // just edited the config is looking at the CLI. Naming each section
6004        // individually rather than setting a flag: "something outside modules
6005        // changed" sends the operator back to diffing their own file, which is
6006        // the work this is meant to save.
6007        let mut restart_required = Vec::new();
6008        for section in RestartRequiredSection::ALL {
6009            let changed = match section {
6010                RestartRequiredSection::Port => configured_port != context.configured_port,
6011                RestartRequiredSection::Storage => storage_config != context.storage_config,
6012                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6013                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6014                }
6015                RestartRequiredSection::AdmissionFactsTargets => {
6016                    admission_facts_targets != context.admission_facts_targets
6017                }
6018                RestartRequiredSection::ScopeAuthorityOwners => {
6019                    scope_authority_owners != context.scope_authority_owners
6020                }
6021                RestartRequiredSection::PrivilegedOperatorPrompt => {
6022                    privileged_operator_prompt != context.privileged_operator_prompt
6023                }
6024            };
6025            if changed {
6026                restart_required.push(section.label().to_string());
6027            }
6028        }
6029        if !restart_required.is_empty() {
6030            warn!(
6031                config_path = %context.config_path.display(),
6032                sections = %restart_required.join(", "),
6033                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6034            );
6035        }
6036
6037        for configured in &modules {
6038            if let Err(err) = validate_spec(&configured.module_spec()) {
6039                return Ok(vec![control_error_frame(
6040                    &frame,
6041                    "invalid_daemon_config",
6042                    format!("supervisor rescan rejected daemon config: {err}"),
6043                )?]);
6044            }
6045        }
6046
6047        let configured_capabilities = modules
6048            .iter()
6049            .map(|module| (module.module_id.clone(), module.enabled))
6050            .collect::<Vec<_>>();
6051        let preview_capability_warnings = if preview {
6052            let (_, registrations) = self.runtime_capability_snapshot()?;
6053            let current_modules = self
6054                .supervisor
6055                .list()
6056                .into_iter()
6057                .map(|module| module.module_id().to_string())
6058                .collect::<BTreeSet<_>>();
6059            let resulting_modules = configured_capabilities.clone();
6060            let removed = current_modules
6061                .into_iter()
6062                .filter(|module_id| {
6063                    !resulting_modules
6064                        .iter()
6065                        .any(|(configured_id, _)| configured_id == module_id)
6066                })
6067                .collect::<Vec<_>>();
6068            // A candidate preview also names providers the candidate disables.
6069            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6070            let disabled = match &mode {
6071                RescanMode::Candidate(_) => resulting_modules
6072                    .iter()
6073                    .filter(|(_, enabled)| !enabled)
6074                    .map(|(module_id, _)| module_id.clone())
6075                    .collect::<Vec<_>>(),
6076                _ => Vec::new(),
6077            };
6078            self.capability_evaluator.preview_warnings(
6079                resulting_modules,
6080                &removed,
6081                &disabled,
6082                &registrations,
6083            )
6084        } else {
6085            Vec::new()
6086        };
6087        let reconciled = match &mode {
6088            RescanMode::Candidate(candidate) => self
6089                .preview_candidate(&context, modules, candidate)
6090                .map(|result| (result, Vec::new())),
6091            _ => {
6092                self.reconcile_supervised_modules(
6093                    &context.supervisor,
6094                    modules,
6095                    preview,
6096                    requester.clone(),
6097                )
6098                .await
6099            }
6100        };
6101        let (result, changed) = match reconciled {
6102            Ok(result) => result,
6103            Err(message) => {
6104                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6105            }
6106        };
6107        if !preview {
6108            if let Some(ledger) = &self.activity {
6109                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6110                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6111            }
6112            if let Err(code) = self.configure_activity(activity_config).await {
6113                warn!(
6114                    code,
6115                    "activity retention could not be applied during rescan"
6116                );
6117            }
6118            self.capability_evaluator
6119                .configure(configured_capabilities, reserved_capabilities);
6120            self.capability_evaluator.wake_deadline_loop();
6121            self.refresh_capability_requirements();
6122        }
6123        let mut result = result;
6124        result.restart_required = restart_required;
6125        result.capability_warnings = preview_capability_warnings;
6126        let response = ClientControlResponse::SupervisorRescan { result };
6127        Ok(vec![control_response_body_frame(
6128            &frame,
6129            &response,
6130            "ClientControlResponse::SupervisorRescan",
6131        )?])
6132    }
6133
6134    async fn handle_supervisor_release_reserved(
6135        &self,
6136        frame: Frame,
6137        module_id: String,
6138    ) -> Result<Vec<Frame>, RouterError> {
6139        let Some(context) = self.rescan.clone() else {
6140            return Ok(vec![control_error_frame(
6141                &frame,
6142                "release_unavailable",
6143                "reserved-id release requires a daemon started with a reloadable config path",
6144            )?]);
6145        };
6146        let operation_lock = self.supervisor.operation_lock();
6147        let _operation_guard = operation_lock.lock().await;
6148        let loaded = match crate::daemon_config::load(&context.config_path) {
6149            Ok(Some(config)) => config,
6150            Ok(None) => {
6151                return Ok(vec![control_error_frame(
6152                    &frame,
6153                    "invalid_daemon_config",
6154                    format!(
6155                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6156                        context.config_path.display()
6157                    ),
6158                )?])
6159            }
6160            Err(err) => {
6161                return Ok(vec![control_error_frame(
6162                    &frame,
6163                    "invalid_daemon_config",
6164                    format!("unable to verify reserved-id release against daemon config: {err}"),
6165                )?])
6166            }
6167        };
6168        if loaded
6169            .modules
6170            .iter()
6171            .any(|configured| configured.module_id == module_id)
6172        {
6173            return Ok(vec![control_error_frame(
6174                &frame,
6175                "reserved_module_configured",
6176                format!(
6177                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6178                ),
6179            )?]);
6180        }
6181        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6182            return Ok(vec![control_error_frame(
6183                &frame,
6184                "reserved_gate_not_retained",
6185                format!(
6186                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6187                ),
6188            )?]);
6189        }
6190
6191        let response = ClientControlResponse::SupervisorAck {
6192            module_id,
6193            applied: true,
6194        };
6195        Ok(vec![control_response_body_frame(
6196            &frame,
6197            &response,
6198            "ClientControlResponse::SupervisorAck",
6199        )?])
6200    }
6201
6202    /// Compare the configured module set with the supervised one, changing
6203    /// nothing. The staged spec each supervised module holds is what the
6204    /// configured entry is compared with.
6205    fn plan_rescan(
6206        &self,
6207        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6208    ) -> Result<RescanPlan, String> {
6209        let mut current = BTreeMap::new();
6210        for module in self.supervisor.list() {
6211            let (spec, health) = module.configuration().map_err(|err| {
6212                format!(
6213                    "failed to read configuration for module_id '{}': {err}",
6214                    module.module_id()
6215                )
6216            })?;
6217            let enabled = module
6218                .status()
6219                .map_err(|err| {
6220                    format!(
6221                        "failed to read status for module_id '{}': {err}",
6222                        module.module_id()
6223                    )
6224                })?
6225                .enabled;
6226            current.insert(
6227                module.module_id().to_string(),
6228                (module, spec, health, enabled),
6229            );
6230        }
6231        let configured = configured_modules
6232            .into_iter()
6233            .map(|module| (module.module_id.clone(), module))
6234            .collect::<BTreeMap<_, _>>();
6235
6236        let added = configured
6237            .keys()
6238            .filter(|module_id| !current.contains_key(*module_id))
6239            .cloned()
6240            .collect::<Vec<_>>();
6241        let removed = current
6242            .keys()
6243            .filter(|module_id| !configured.contains_key(*module_id))
6244            .cloned()
6245            .collect::<Vec<_>>();
6246        let mut changed_pending_reload = Vec::new();
6247        let mut configuration_changes = BTreeSet::new();
6248        let mut enabled_changes = BTreeSet::new();
6249        let mut unchanged = 0_u32;
6250
6251        for (module_id, configured_module) in &configured {
6252            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6253            else {
6254                continue;
6255            };
6256            // Compare the whole launch spec so a future launch field cannot
6257            // accidentally become a live-only policy change. Health is stored
6258            // separately and applies live without replacing the process.
6259            let launch_changed = *current_spec != configured_module.module_spec();
6260            let configuration_changed =
6261                launch_changed || *current_health != configured_module.health;
6262            let enabled_changed = *current_enabled != configured_module.enabled;
6263            if configuration_changed {
6264                configuration_changes.insert(module_id.clone());
6265            }
6266            if launch_changed {
6267                changed_pending_reload.push(module_id.clone());
6268            }
6269            if enabled_changed {
6270                enabled_changes.insert(module_id.clone());
6271            }
6272            if !configuration_changed && !enabled_changed {
6273                unchanged = unchanged.saturating_add(1);
6274            }
6275        }
6276        Ok(RescanPlan {
6277            current,
6278            configured,
6279            added,
6280            removed,
6281            changed_pending_reload,
6282            configuration_changes,
6283            enabled_changes,
6284            unchanged,
6285        })
6286    }
6287
6288    /// Reconcile the running module set against the configured one.
6289    ///
6290    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6291    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6292    /// deliberately shares this function with the executing path rather than
6293    /// computing the same diff somewhere else -- two implementations of one
6294    /// decision agree until they do not, and the whole value of a preview is that
6295    /// it describes the operation that will actually run.
6296    async fn reconcile_supervised_modules(
6297        &self,
6298        supervisor: &Supervisor,
6299        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6300        preview: bool,
6301        requester: Principal,
6302    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6303        let plan = self.plan_rescan(configured_modules)?;
6304
6305        // Everything above this point is pure computation over two snapshots.
6306        // Everything below MUTATES. The preview returns here so the boundary is a
6307        // single early return rather than a condition repeated at each mutation
6308        // site, where one missed guard would apply part of a change the caller was
6309        // told would not happen.
6310        if preview {
6311            return Ok((plan.preview_result(), Vec::new()));
6312        }
6313        let changed = plan
6314            .configuration_changes
6315            .union(&plan.enabled_changes)
6316            .cloned()
6317            .collect::<Vec<_>>();
6318        let RescanPlan {
6319            current,
6320            configured,
6321            added,
6322            removed,
6323            changed_pending_reload,
6324            configuration_changes,
6325            enabled_changes,
6326            unchanged,
6327        } = plan;
6328        // The staged generation moves before the first change rather than
6329        // after the last, so a rescan that fails partway still invalidates
6330        // every earlier `supervisor.rescan_checked` preview: part of its diff
6331        // may have run.
6332        if !(added.is_empty()
6333            && removed.is_empty()
6334            && configuration_changes.is_empty()
6335            && enabled_changes.is_empty())
6336        {
6337            if let Some(context) = &self.rescan {
6338                context
6339                    .generation
6340                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6341            }
6342        }
6343
6344        for module_id in &removed {
6345            let module = &current
6346                .get(module_id)
6347                .expect("removed module came from current supervisor state")
6348                .0;
6349            module
6350                .retire_requested(Some(requester.clone()))
6351                .await
6352                .map_err(|err| {
6353                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6354                })?;
6355            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6356            //
6357            // `handle_route_open` resolves an absent module in three steps:
6358            // registry, then supervisor status, then tombstone. Retiring first
6359            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6360            // went with the teardown above, the supervisor entry went with
6361            // `retire`, and the tombstone does not exist yet -- so a route.open
6362            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6363            // it") for a module that was deliberately removed and whose caller
6364            // should get `module_removed` (TERMINAL, carrying a removal age).
6365            //
6366            // Writing the tombstone first closes it: during the window the
6367            // supervisor entry still answers, so the caller gets
6368            // `target_unavailable` -- retryable, and TRUE, because the module
6369            // is mid-teardown. After both statements it is `module_removed`.
6370            // No instant remains where a removed module reads as one that
6371            // never existed.
6372            //
6373            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6374            // the absence of a test beside a fix invites deletion: these are
6375            // two sync statements with no await between them, so reaching the
6376            // window needs a second worker thread to land exactly between them
6377            // and there is no hook to force it. MEASURED: the 25 daemon_config
6378            // tests pass identically with the old order and the new one, so
6379            // the existing suite cannot see this and a green run is not
6380            // evidence either way. What the suite does hold is the
6381            // post-condition -- a removed module answers `module_removed` --
6382            // which this preserves.
6383            //
6384            // Found by an Athena panel reading the shipped tree against a
6385            // design note (2026-09-19), as the one concrete instance of that
6386            // note's class that survived contact with source. Direction is
6387            // benign: retryable where terminal was intended, never the reverse.
6388            self.supervisor.record_rescan_removal(module_id);
6389            self.supervisor.retire(module_id);
6390            self.route_outages.forget(module_id);
6391        }
6392
6393        for module_id in configured.keys() {
6394            let Some((module, _, _, _)) = current.get(module_id) else {
6395                continue;
6396            };
6397            let configured_module = configured
6398                .get(module_id)
6399                .expect("configured module id came from configured map");
6400            if configuration_changes.contains(module_id) {
6401                module
6402                    .update_configuration(
6403                        configured_module.module_spec(),
6404                        configured_module.health.clone(),
6405                        configured_module.drain_timeout_ms,
6406                    )
6407                    .await
6408                    .map_err(|err| {
6409                        format!(
6410                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6411                        )
6412                    })?;
6413            }
6414            if enabled_changes.contains(module_id) {
6415                // A rescan that starts or stops a module applies an operator's
6416                // edit to the config, so the resulting outage was asked for.
6417                self.route_outages.mark_operator_action(module_id);
6418                module
6419                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6420                    .await
6421                    .map_err(|err| {
6422                        self.route_outages.operator_action_ended_unrefused(module_id);
6423                        format!(
6424                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6425                            configured_module.enabled
6426                        )
6427                    })?;
6428            }
6429        }
6430
6431        for module_id in &added {
6432            let configured_module = configured
6433                .get(module_id)
6434                .expect("added module id came from configured map");
6435            supervisor
6436                .supervise_configured_requested(
6437                    configured_module.module_spec(),
6438                    configured_module.enabled,
6439                    configured_module.health.clone(),
6440                    configured_module.drain_timeout_ms,
6441                    configured_module.restart,
6442                    configured_module.register_timeout,
6443                    Some(requester.clone()),
6444                )
6445                .map_err(|err| {
6446                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6447                })?;
6448        }
6449
6450        Ok((
6451            SupervisorRescanResult {
6452                added,
6453                removed,
6454                changed_pending_reload,
6455                enabled_changes: enabled_changes.iter().cloned().collect(),
6456                unchanged,
6457                preview: false,
6458                // Filled by the caller, which is the only layer that can see the
6459                // previous config to diff against.
6460                restart_required: Vec::new(),
6461                capability_warnings: Vec::new(),
6462                module_actions: Vec::new(),
6463                earlier_pending: Vec::new(),
6464                expect: None,
6465            },
6466            changed,
6467        ))
6468    }
6469
6470    async fn handle_supervisor_set_enabled(
6471        &self,
6472        frame: Frame,
6473        module_id: String,
6474        enabled: bool,
6475        requester: Principal,
6476    ) -> Result<Vec<Frame>, RouterError> {
6477        let operation_lock = self.supervisor.operation_lock();
6478        let _operation_guard = operation_lock.lock().await;
6479        let Some(module) = self.supervisor.get(&module_id) else {
6480            return Ok(vec![control_error_frame(
6481                &frame,
6482                "unknown_module",
6483                format!("module_id '{module_id}' is not supervised"),
6484            )?]);
6485        };
6486
6487        // Enabling counts as well as disabling: a module an operator starts
6488        // is refused until it registers, and that wait was asked for.
6489        self.route_outages.mark_operator_action(&module_id);
6490        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6491            Ok(applied) => applied,
6492            Err(err) => {
6493                self.route_outages
6494                    .operator_action_ended_unrefused(&module_id);
6495                return Ok(vec![control_error_frame(
6496                    &frame,
6497                    "target_unavailable",
6498                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6499                )?]);
6500            }
6501        };
6502        if !applied {
6503            // Already in the requested state: nothing was made unavailable,
6504            // so the mark must not outlive this request.
6505            self.route_outages
6506                .operator_action_ended_unrefused(&module_id);
6507        }
6508
6509        self.capability_evaluator.wake_deadline_loop();
6510        self.refresh_capability_requirements();
6511        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6512        Ok(vec![control_response_body_frame(
6513            &frame,
6514            &response,
6515            "ClientControlResponse::SupervisorAck",
6516        )?])
6517    }
6518
6519    async fn handle_supervisor_health_probe(
6520        &self,
6521        frame: Frame,
6522        module_id: String,
6523    ) -> Result<Vec<Frame>, RouterError> {
6524        self.refresh_capability_requirements();
6525        let Some(registration) = self
6526            .registry
6527            .get_module(&module_id)
6528            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6529        else {
6530            return Ok(vec![control_error_frame(
6531                &frame,
6532                "unknown_module",
6533                format!("module_id '{module_id}' is not registered"),
6534            )?]);
6535        };
6536
6537        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6538        // named for it. Making `module_registration_grants_op` return false
6539        // unconditionally reddens five tests, and every one is named for something
6540        // else -- capability relay, probe/bind demultiplexing, supervision-only
6541        // probing. They exercise a successful advertisement check on the way to their
6542        // own subject.
6543        //
6544        // Real protection, fragile in a specific way: narrowing any of those tests to
6545        // focus on its stated subject would silently remove coverage nobody knows
6546        // they are carrying. Recorded here rather than as a sixth test, because the
6547        // useful fact is WHICH tests hold the guard up -- a new test would add
6548        // coverage without telling the next person what the existing ones quietly do.
6549        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6550        {
6551            return Ok(vec![control_error_frame(
6552                &frame,
6553                "health_not_advertised",
6554                format!("module_id '{module_id}' did not advertise health.check"),
6555            )?]);
6556        }
6557
6558        let deadline = Instant::now() + self.health_probe_timeout;
6559        let pending = match self.forwarding.begin_module_control_rpc_for(
6560            &module_id,
6561            MODULE_CONTROL_OP_HEALTH_CHECK,
6562            deadline,
6563        ) {
6564            Ok(pending) => pending,
6565            Err(err) => {
6566                return Ok(vec![control_error_frame(
6567                    &frame,
6568                    forwarding_error_code(&err),
6569                    err.to_string(),
6570                )?])
6571            }
6572        };
6573
6574        let PendingModuleControlRpc {
6575            endpoint,
6576            module_sink,
6577            negotiated_ver,
6578            corr: probe_corr,
6579            receiver,
6580        } = pending;
6581        let mut guard =
6582            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6583        let probe_body =
6584            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6585                RouterError::backend(
6586                    0,
6587                    frame.header.corr,
6588                    format!("failed to encode health.check request: {err}"),
6589                )
6590            })?;
6591        let probe_frame = Frame::build_with_version(
6592            negotiated_ver,
6593            FrameType::Request,
6594            control_flags(),
6595            0,
6596            0,
6597            probe_corr,
6598            probe_body,
6599        )
6600        .map_err(RouterError::FrameBuild)?;
6601
6602        // Queue admission and the reply share one deadline, so a module that
6603        // stops reading cannot park the requesting connection's reader forever.
6604        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6605            Ok(Ok(())) => {}
6606            Ok(Err(err)) => {
6607                return Ok(vec![control_error_frame(
6608                    &frame,
6609                    "target_unavailable",
6610                    err.to_string(),
6611                )?]);
6612            }
6613            Err(_) => {
6614                return Ok(vec![control_error_frame(
6615                    &frame,
6616                    "module_timeout",
6617                    format!(
6618                        "module_id '{module_id}' did not answer health.check within {:?}",
6619                        self.health_probe_timeout
6620                    ),
6621                )?]);
6622            }
6623        }
6624
6625        match timeout_at(deadline, receiver).await {
6626            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6627                guard.disarm();
6628                let Some(report) = response.health_report() else {
6629                    return Ok(vec![control_error_frame(
6630                        &frame,
6631                        "invalid_control_body",
6632                        "health.check RPC returned a non-health response",
6633                    )?]);
6634                };
6635                // Metrics go out whole here. The supervisor's cached snapshot
6636                // caps this blob (see truncate_health_metrics), and this path
6637                // exists precisely to answer without that cap -- so applying it
6638                // here would leave no way to see what the cached view drops.
6639                let HealthReport {
6640                    status,
6641                    detail,
6642                    metrics,
6643                } = report;
6644                let capability_detail = self
6645                    .capability_evaluator
6646                    .required_problem_detail(&module_id);
6647                let response = ClientControlResponse::SupervisorHealthProbe {
6648                    module_id,
6649                    status,
6650                    detail: append_capability_problem_detail(detail, capability_detail),
6651                    metrics,
6652                };
6653                Ok(vec![control_response_body_frame(
6654                    &frame,
6655                    &response,
6656                    "ClientControlResponse::SupervisorHealthProbe",
6657                )?])
6658            }
6659            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6660                guard.disarm();
6661                Ok(vec![control_error_body_frame(&frame, body)?])
6662            }
6663            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6664                guard.disarm();
6665                Ok(vec![control_error_frame(
6666                    &frame,
6667                    "target_unavailable",
6668                    message,
6669                )?])
6670            }
6671            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6672                guard.disarm();
6673                Ok(vec![control_error_frame(
6674                    &frame,
6675                    "invalid_control_body",
6676                    message,
6677                )?])
6678            }
6679            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6680                guard.disarm();
6681                Ok(vec![control_error_frame(
6682                    &frame,
6683                    "invalid_control_body",
6684                    format!("expected module-control op '{expected}', got '{actual}'"),
6685                )?])
6686            }
6687            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6688                guard.disarm();
6689                Ok(vec![control_error_frame(
6690                    &frame,
6691                    "module_timeout",
6692                    format!(
6693                        "module_id '{module_id}' answered health.check after {:?}",
6694                        self.health_probe_timeout
6695                    ),
6696                )?])
6697            }
6698            Ok(Err(_)) => Ok(vec![control_error_frame(
6699                &frame,
6700                "target_unavailable",
6701                "health.check waiter was canceled before the module responded",
6702            )?]),
6703            Err(_) => Ok(vec![control_error_frame(
6704                &frame,
6705                "module_timeout",
6706                format!(
6707                    "module_id '{module_id}' did not answer health.check within {:?}",
6708                    self.health_probe_timeout
6709                ),
6710            )?]),
6711        }
6712    }
6713
6714    fn supervisor_status(
6715        &self,
6716        module_id: &str,
6717        corr: u64,
6718    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6719        self.supervisor
6720            .get(module_id)
6721            .map(|module| {
6722                let warming = module.is_warming_for_control("status").map_err(|err| {
6723                    RouterError::backend(
6724                        0,
6725                        corr,
6726                        format!(
6727                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6728                        ),
6729                    )
6730                })?;
6731                module.status_for_control("status").map_err(|err| {
6732                    RouterError::backend(
6733                        0,
6734                        corr,
6735                        format!(
6736                            "failed to read supervisor status for module_id '{module_id}': {err}"
6737                        ),
6738                    )
6739                }).map(|status| (status, warming))
6740            })
6741            .transpose()
6742    }
6743
6744    fn guard_module_control_op(
6745        &self,
6746        frame: &Frame,
6747        module_id: &str,
6748        op: &str,
6749    ) -> Result<Option<Frame>, RouterError> {
6750        if self.module_grants_op(module_id, op, frame.header.corr)? {
6751            return Ok(None);
6752        }
6753
6754        Ok(Some(control_error_frame(
6755            frame,
6756            "op_not_allowed",
6757            format!("module_id '{module_id}' did not grant control op '{op}'"),
6758        )?))
6759    }
6760
6761    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6762        let Some(registration) = self
6763            .registry
6764            .get_module(module_id)
6765            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6766        else {
6767            return Ok(false);
6768        };
6769        Ok(module_registration_grants_op(&registration.control_ops, op))
6770    }
6771
6772    fn handle_status_update(
6773        &self,
6774        endpoint: ModuleEndpointId,
6775        frame: Frame,
6776    ) -> Result<Vec<Frame>, RouterError> {
6777        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6778            Ok(update) => update,
6779            Err(err) => {
6780                // Forward-compat: a newer module may push a channel-0 op this subc
6781                // version doesn't know. The control contract says unknown push ops
6782                // are IGNORED, never answered with an error. Only a malformed body
6783                // for an op we DO know is a real error worth surfacing.
6784                if is_known_module_push_op(&frame.body) {
6785                    return Ok(vec![control_error_frame(
6786                        &frame,
6787                        "invalid_control_body",
6788                        format!("malformed module control push body: {err}"),
6789                    )?]);
6790                }
6791                return Ok(Vec::new());
6792            }
6793        };
6794
6795        match update {
6796            ModuleControlPush::RouteStatus {
6797                route_channel,
6798                route_epoch,
6799                status,
6800            } => {
6801                self.forwarding
6802                    .cache_status(endpoint, route_channel, route_epoch, status)
6803                    .map_err(RouterError::Forwarding)?;
6804            }
6805        }
6806        Ok(Vec::new())
6807    }
6808
6809    fn handle_route_poll(
6810        &self,
6811        ctx: &RouteCtx,
6812        frame: Frame,
6813        route_channel: u16,
6814        route_epoch: u32,
6815        kind: PollKind,
6816    ) -> Result<Vec<Frame>, RouterError> {
6817        let snapshot = self
6818            .forwarding
6819            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6820            .map_err(RouterError::Forwarding)?;
6821        let response = match (kind, snapshot) {
6822            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6823                ClientControlResponse::RoutePoll {
6824                    route_channel,
6825                    route_epoch,
6826                    status,
6827                    live: None,
6828                }
6829            }
6830            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6831                route_channel,
6832                route_epoch,
6833                status: None,
6834                live: None,
6835            },
6836            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6837                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6838                // is what makes reporting `true` correct rather than a
6839                // confident guess. `process_live` returns None only when the
6840                // module id has no supervisor snapshot at all -- an
6841                // externally-started module the daemon did not spawn -- and
6842                // for those the supervisor has no opinion to offer, ever. It
6843                // is never None for a supervised module in an unknown state:
6844                // a supervised module always has a snapshot, and the answer
6845                // comes from `state == Running && process_alive`.
6846                //
6847                // The route is Bound, so the module completed a HELLO on a
6848                // live connection; "the process this route points at is
6849                // running" is therefore attested by the binding rather than
6850                // assumed. Reporting `false` for an unsupervised module would
6851                // be the actual lie -- it would tell a client its healthy
6852                // route is dead because the daemon does not manage the
6853                // process.
6854                //
6855                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6856                // module whose liveness is genuinely unknown, e.g. a snapshot
6857                // that has not been populated yet -- THIS DEFAULT BECOMES
6858                // WRONG and must split: unsupervised stays true, unknown
6859                // becomes null so the client can tell the two apart. The
6860                // response field is already `Option<bool>`, so the wire can
6861                // carry that distinction today.
6862                let live = self
6863                    .process_liveness
6864                    .as_ref()
6865                    .and_then(|source| source.process_live(&module_id))
6866                    .unwrap_or(true);
6867                ClientControlResponse::RoutePoll {
6868                    route_channel,
6869                    route_epoch,
6870                    status: None,
6871                    live: Some(live),
6872                }
6873            }
6874            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6875                route_channel,
6876                route_epoch,
6877                status: None,
6878                live: Some(false),
6879            },
6880        };
6881
6882        Ok(vec![control_response_body_frame(
6883            &frame,
6884            &response,
6885            "ClientControlResponse::RoutePoll",
6886        )?])
6887    }
6888
6889    pub(crate) fn observe_module_control_completion(
6890        &self,
6891        completion: ModuleControlRpcCompletion,
6892    ) -> bool {
6893        match completion {
6894            ModuleControlRpcCompletion::Unknown => false,
6895            ModuleControlRpcCompletion::Settled => true,
6896            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6897                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6898                info!(
6899                    module_id = %module_id,
6900                    latency_ms,
6901                    "late health.check answer proves the module is alive"
6902                );
6903                match self
6904                    .supervisor
6905                    .record_late_health_answer(&module_id, latency_ms)
6906                {
6907                    Ok(true) => {}
6908                    Ok(false) => debug!(
6909                        module_id = %module_id,
6910                        latency_ms,
6911                        "late health.check answer has no active supervisor snapshot"
6912                    ),
6913                    Err(err) => warn!(
6914                        module_id = %module_id,
6915                        latency_ms,
6916                        error = %err,
6917                        "failed to record late health.check answer"
6918                    ),
6919                }
6920                true
6921            }
6922        }
6923    }
6924
6925    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6926    /// the module connection whose frame is being handled, or to the client that
6927    /// relay was opened for.
6928    ///
6929    /// This runs on the MODULE connection's frame handler, where returning `Err`
6930    /// ends that connection -- and a module connection carries every client's
6931    /// routes to that module, so ending it costs the whole fleet its tools.
6932    /// `ConnectionClosing` carries the id of the connection that is closing, and
6933    /// when that id is a CLIENT's, the condition is entirely about that one
6934    /// client's route.open. A client-scoped condition has no authority over a
6935    /// shared module connection, so it is logged and the single relay is dropped:
6936    /// the client is going away, and `complete_pending_relay` already removed the
6937    /// relay before failing, so there is nothing left to settle. Anything that
6938    /// relay still reserved is released by that client's own connection teardown,
6939    /// which is already under way -- that is what "closing" means.
6940    ///
6941    /// Every other failure is a statement about THIS connection and stays fatal:
6942    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6943    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6944    /// frames correctly.
6945    fn refuse_to_end_module_connection_for_a_client(
6946        &self,
6947        module_connection_id: ConnectionId,
6948        corr: u64,
6949        err: ForwardingError,
6950    ) -> Result<(), RouterError> {
6951        if let ForwardingError::ConnectionClosing { connection_id } = err {
6952            if connection_id != module_connection_id {
6953                warn!(
6954                    module_connection_id = module_connection_id.get(),
6955                    client_connection_id = connection_id.get(),
6956                    corr,
6957                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6958                );
6959                return Ok(());
6960            }
6961        }
6962        Err(RouterError::Forwarding(err))
6963    }
6964
6965    fn handle_module_relay_response(
6966        &self,
6967        connection_id: ConnectionId,
6968        frame: Frame,
6969    ) -> Result<Vec<Frame>, RouterError> {
6970        let mut secondary_error = None;
6971        let outcome = match frame.header.ty {
6972            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6973                Ok(probe) if probe.op == "route.bind" => {
6974                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6975                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6976                            RouteBindRelayOutcome::Accepted
6977                        }
6978                        Ok(other) => {
6979                            let message =
6980                                format!("route.bind response carried unexpected body: {other:?}");
6981                            secondary_error = Some(control_error_frame(
6982                                &frame,
6983                                "invalid_control_body",
6984                                message.clone(),
6985                            )?);
6986                            RouteBindRelayOutcome::ModuleGone(message)
6987                        }
6988                        Err(err) => {
6989                            let message = format!("malformed route.bind response body: {err}");
6990                            secondary_error = Some(control_error_frame(
6991                                &frame,
6992                                "invalid_control_body",
6993                                message.clone(),
6994                            )?);
6995                            RouteBindRelayOutcome::ModuleGone(message)
6996                        }
6997                    }
6998                }
6999                Ok(probe) => {
7000                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7001                    {
7002                        Ok(response) => ModuleControlRpcOutcome::Response(response),
7003                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7004                            "malformed {} response body: {err}",
7005                            probe.op
7006                        )),
7007                    };
7008                    let completion = self
7009                        .forwarding
7010                        .complete_module_control_rpc(
7011                            connection_id,
7012                            frame.header.corr,
7013                            Some(&probe.op),
7014                            outcome,
7015                        )
7016                        .map_err(RouterError::Forwarding)?;
7017                    if !self.observe_module_control_completion(completion) {
7018                        debug!(
7019                            connection_id = connection_id.get(),
7020                            corr = frame.header.corr,
7021                            op = %probe.op,
7022                            "dropping late or unknown module-control RPC response"
7023                        );
7024                    }
7025                    return Ok(Vec::new());
7026                }
7027                Err(err) => {
7028                    if let Some(expected_op) = self
7029                        .forwarding
7030                        .pending_module_control_op(connection_id, frame.header.corr)
7031                        .map_err(RouterError::Forwarding)?
7032                    {
7033                        let completion = self
7034                            .forwarding
7035                            .complete_module_control_rpc(
7036                                connection_id,
7037                                frame.header.corr,
7038                                None,
7039                                ModuleControlRpcOutcome::MalformedResponse(format!(
7040                                    "malformed {expected_op} response body: {err}"
7041                                )),
7042                            )
7043                            .map_err(RouterError::Forwarding)?;
7044                        if !self.observe_module_control_completion(completion) {
7045                            debug!(
7046                                connection_id = connection_id.get(),
7047                                corr = frame.header.corr,
7048                                "dropping late malformed module-control RPC response"
7049                            );
7050                        }
7051                        return Ok(Vec::new());
7052                    }
7053                    let message = format!("malformed route.bind response body: {err}");
7054                    secondary_error = Some(control_error_frame(
7055                        &frame,
7056                        "invalid_control_body",
7057                        message.clone(),
7058                    )?);
7059                    RouteBindRelayOutcome::ModuleGone(message)
7060                }
7061            },
7062            FrameType::Error => {
7063                if self
7064                    .forwarding
7065                    .pending_module_control_op(connection_id, frame.header.corr)
7066                    .map_err(RouterError::Forwarding)?
7067                    .is_some()
7068                {
7069                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7070                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7071                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7072                            "malformed module-control ERROR body: {err}"
7073                        )),
7074                    };
7075                    let completion = self
7076                        .forwarding
7077                        .complete_module_control_rpc(
7078                            connection_id,
7079                            frame.header.corr,
7080                            None,
7081                            outcome,
7082                        )
7083                        .map_err(RouterError::Forwarding)?;
7084                    if !self.observe_module_control_completion(completion) {
7085                        debug!(
7086                            connection_id = connection_id.get(),
7087                            corr = frame.header.corr,
7088                            "dropping late or unknown module-control RPC error"
7089                        );
7090                    }
7091                    return Ok(Vec::new());
7092                }
7093                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7094                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7095                    Err(err) => {
7096                        let message = format!("malformed route.bind ERROR body: {err}");
7097                        secondary_error = Some(control_error_frame(
7098                            &frame,
7099                            "invalid_control_body",
7100                            message.clone(),
7101                        )?);
7102                        RouteBindRelayOutcome::ModuleGone(message)
7103                    }
7104                }
7105            }
7106            ty => {
7107                return Ok(vec![control_error_frame(
7108                    &frame,
7109                    "unsupported_control_frame",
7110                    format!("unsupported module channel-0 frame {ty:?}"),
7111                )?])
7112            }
7113        };
7114
7115        let settled =
7116            self.forwarding
7117                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7118        let completion = match settled {
7119            Ok(completion) => completion,
7120            Err(err) => {
7121                self.refuse_to_end_module_connection_for_a_client(
7122                    connection_id,
7123                    frame.header.corr,
7124                    err,
7125                )?;
7126                return Ok(secondary_error.into_iter().collect());
7127            }
7128        };
7129        if let Some(target) = completion.abandoned.as_ref() {
7130            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7131        }
7132        if !completion.settled {
7133            debug!(
7134                connection_id = connection_id.get(),
7135                corr = frame.header.corr,
7136                frame_type = ?frame.header.ty,
7137                "dropping late or unknown route.bind relay response"
7138            );
7139        }
7140        Ok(secondary_error.into_iter().collect())
7141    }
7142
7143    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7144        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7145        // GOODBYE ends the connection's logical session even when its socket
7146        // stays open. Use disconnect teardown so verdicts, client notices and
7147        // scope authority are released at the same lifecycle boundary.
7148        self.cleanup_connection_with_end_reason(
7149            connection_id,
7150            RegistrationEndReason::ExplicitGoodbye,
7151        )
7152        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7153        Ok(Vec::new())
7154    }
7155}
7156
7157impl Default for ControlHandler {
7158    fn default() -> Self {
7159        Self::new(Arc::new(Registry::default()))
7160    }
7161}
7162
7163impl crate::supervise::SwapPromotionObserver for ControlHandler {
7164    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7165        self.apply_registration_capabilities(registration);
7166    }
7167}
7168
7169fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7170    CapabilityRequirementStatus {
7171        consumer: status.consumer,
7172        capability: status.capability,
7173        need: match status.need {
7174            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7175            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7176        },
7177        verdict: status.verdict.as_str().to_string(),
7178        episode_seq: status.episode_seq,
7179        config_satisfiable: status.config_satisfiable,
7180        runtime_available: status.runtime_available,
7181        detail: status.detail,
7182    }
7183}
7184
7185fn append_capability_problem_detail(
7186    detail: Option<String>,
7187    capability_detail: Option<String>,
7188) -> Option<String> {
7189    match (detail, capability_detail) {
7190        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7191        (Some(detail), None) => Some(detail),
7192        (None, Some(capability_detail)) => Some(capability_detail),
7193        (None, None) => None,
7194    }
7195}
7196
7197fn subc_ops() -> Vec<String> {
7198    SUBC_CONTROL_OPS
7199        .iter()
7200        .map(|op| (*op).to_string())
7201        .collect()
7202}
7203
7204fn module_subc_ops() -> Vec<String> {
7205    SUBC_CONTROL_OPS
7206        .iter()
7207        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7208        .map(|op| (*op).to_string())
7209        .collect()
7210}
7211
7212#[cfg(test)]
7213fn module_baseline_control_ops() -> Vec<String> {
7214    MODULE_BASELINE_CONTROL_OPS
7215        .iter()
7216        .map(|op| (*op).to_string())
7217        .collect()
7218}
7219
7220fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7221    let mut seen = HashSet::new();
7222    let mut effective = Vec::new();
7223    for op in MODULE_BASELINE_CONTROL_OPS {
7224        if seen.insert((*op).to_string()) {
7225            effective.push((*op).to_string());
7226        }
7227    }
7228    for op in declared.unwrap_or_default() {
7229        if seen.insert(op.clone()) {
7230            effective.push(op);
7231        }
7232    }
7233    effective
7234}
7235
7236fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7237    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7238}
7239
7240fn target_module_id(target: &RouteTarget) -> &str {
7241    match target {
7242        RouteTarget::ToolProvider { module_id }
7243        | RouteTarget::ManagementSurface { module_id }
7244        | RouteTarget::InternalService { module_id, .. } => module_id,
7245    }
7246}
7247
7248fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7249    roles.iter().any(|role| match (target, role) {
7250        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7251        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7252        (
7253            RouteTarget::InternalService { service_id, .. },
7254            ProviderRole::InternalService {
7255                service_id: provided,
7256                ..
7257            },
7258        ) => service_id == provided,
7259        _ => false,
7260    })
7261}
7262
7263fn is_routable_role(role: &ProviderRole) -> bool {
7264    matches!(
7265        role,
7266        ProviderRole::ToolProvider { .. }
7267            | ProviderRole::ManagementSurface { .. }
7268            | ProviderRole::InternalService { .. }
7269    )
7270}
7271
7272#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7273enum ControlRequestBodyError {
7274    UnknownOp,
7275    InvalidBody,
7276}
7277
7278#[derive(Debug, Deserialize)]
7279struct ControlOpProbe {
7280    op: String,
7281}
7282
7283/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7284/// this set is treated as a forward-compat unknown and ignored rather than errored.
7285const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7286
7287fn is_known_module_push_op(body: &[u8]) -> bool {
7288    serde_json::from_slice::<ControlOpProbe>(body)
7289        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7290        .unwrap_or(false)
7291}
7292
7293fn is_known_module_request_op(body: &[u8]) -> bool {
7294    serde_json::from_slice::<ControlOpProbe>(body)
7295        .map(|probe| is_module_to_subc_op(&probe.op))
7296        .unwrap_or(false)
7297}
7298
7299fn is_module_to_subc_op(op: &str) -> bool {
7300    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7301}
7302
7303fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7304    debug!(
7305        op = %op,
7306        connection_id = connection_id.get(),
7307        corr,
7308        "control dispatch"
7309    );
7310}
7311
7312fn log_slow_control_dispatch(
7313    dispatch_started_at: Option<StdInstant>,
7314    op: &'static str,
7315    connection_id: ConnectionId,
7316    corr: u64,
7317    route_open_timings: Option<&RouteOpenTimings>,
7318) {
7319    let Some(dispatch_started_at) = dispatch_started_at else {
7320        return;
7321    };
7322    // Use the same monotonic clock as the phase timers and relay deadline.
7323    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7324    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7325        if let Some(timings) = route_open_timings {
7326            warn!(
7327                op = %op,
7328                connection_id = connection_id.get(),
7329                corr,
7330                elapsed_ms = elapsed.as_millis() as u64,
7331                module_id = ?timings.module_id,
7332                principal = %timings.principal,
7333                admission_ms = timings.admission.as_millis() as u64,
7334                project_root_ms = timings.project_root.as_millis() as u64,
7335                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7336                bind_send_ms = timings.bind_send.as_millis() as u64,
7337                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7338                "slow control dispatch"
7339            );
7340            return;
7341        }
7342        warn!(
7343            op = %op,
7344            connection_id = connection_id.get(),
7345            corr,
7346            elapsed_ms = elapsed.as_millis() as u64,
7347            "slow control dispatch"
7348        );
7349    }
7350}
7351
7352/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7353#[cfg(test)]
7354type TestCallerLookup =
7355    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7356
7357/// Every new client operation needs an explicit privilege decision here.
7358fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7359    match request {
7360        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7361            Some(PrivilegedOperation::Restart { module_id })
7362        }
7363        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7364            Some(PrivilegedOperation::Swap { module_id })
7365        }
7366        ClientControlRequest::SupervisorReload { module_id } => {
7367            Some(PrivilegedOperation::Reload { module_id })
7368        }
7369        ClientControlRequest::SupervisorRescan { preview } => {
7370            if *preview {
7371                None
7372            } else {
7373                Some(PrivilegedOperation::Rescan)
7374            }
7375        }
7376        // Only the well-formed apply asks for approval. A preview changes
7377        // nothing, and any other combination is refused before it touches
7378        // anything, so prompting the operator for it would ask about a request
7379        // that cannot run.
7380        ClientControlRequest::SupervisorRescanChecked {
7381            preview,
7382            candidate,
7383            expect,
7384        } => (!*preview && candidate.is_none() && expect.is_some())
7385            .then_some(PrivilegedOperation::Rescan),
7386        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7387            Some(PrivilegedOperation::SetEnabled {
7388                module_id,
7389                enabled: *enabled,
7390            })
7391        }
7392        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7393            Some(PrivilegedOperation::ReleaseReserved { module_id })
7394        }
7395        ClientControlRequest::ServerDescribe {}
7396        | ClientControlRequest::CatalogList { .. }
7397        | ClientControlRequest::RouteOpen { .. }
7398        | ClientControlRequest::RoutePoll { .. }
7399        | ClientControlRequest::SupervisorList {}
7400        | ClientControlRequest::SupervisorHealth {}
7401        | ClientControlRequest::SupervisorHealthProbe { .. }
7402        | ClientControlRequest::SupervisorRoutes { .. }
7403        | ClientControlRequest::SupervisorProvenance { .. }
7404        | ClientControlRequest::SupervisorStderrTail { .. }
7405        | ClientControlRequest::SupervisorTerminals { .. }
7406        | ClientControlRequest::SupervisorSpawnSnapshot {}
7407        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7408    }
7409}
7410
7411fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7412    match request {
7413        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7414        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7415        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7416        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7417        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7418        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7419        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7420        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7421        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7422        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7423        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7424        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7425        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7426        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7427        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7428        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7429        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7430        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7431        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7432        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7433    }
7434}
7435
7436/// The caller of one `route.open`, as its `route.open refused` line names it.
7437///
7438/// A connection id alone cannot name a caller after the fact: ids restart at
7439/// every daemon restart, so the same number belongs to different modules
7440/// before and after one. These three fields are what the refused line needs
7441/// to be attributable on its own, and they are rendered with the same helpers
7442/// as the `route.open accepted` line so both directions grep alike.
7443struct RouteOpenCaller {
7444    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
7445    /// when the request presented a consumer identity whose launch nonce did
7446    /// not verify. The claimed module id is never printed as a principal.
7447    principal: String,
7448    /// `none` for an unscoped open; the admitted stamp's label once scope
7449    /// admission issued one; before that, the requested selector's label.
7450    scope: String,
7451    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
7452    /// because it is a string the caller chose.
7453    harness: String,
7454}
7455
7456impl RouteOpenCaller {
7457    fn new(
7458        principal: Option<&Principal>,
7459        scope: Option<&ScopeSelector>,
7460        identity: &BindIdentity,
7461    ) -> Self {
7462        Self {
7463            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7464            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7465            harness: bounded_log_text(&identity.harness),
7466        }
7467    }
7468
7469    /// For a refusal that has no parsed request at hand: names whatever the
7470    /// frame still says, and `unknown` where it says nothing parseable.
7471    fn unknown() -> Self {
7472        Self {
7473            principal: "unknown".to_string(),
7474            scope: "unknown".to_string(),
7475            harness: "unknown".to_string(),
7476        }
7477    }
7478}
7479
7480/// Longest client-chosen text (a scope ref, a harness) a route.open log line
7481/// prints before cutting it.
7482const MAX_LOGGED_TEXT_CHARS: usize = 64;
7483
7484/// `value` Debug-quoted, so control characters land escaped and cannot forge
7485/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
7486/// `…` inside the quotes so one request cannot flood the line.
7487fn bounded_log_text(value: &str) -> String {
7488    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7489    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7490        bounded.push('…');
7491    }
7492    format!("{bounded:?}")
7493}
7494
7495/// The `scope=` field of the `route.open accepted` line, from the stamp that
7496/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7497/// so a ref holding control characters cannot forge a log line, and cut to 64
7498/// characters with a trailing `…` to bound the line. The bind proof and the
7499/// scope's attributes never appear.
7500fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7501    use subc_protocol::scope::ScopeKind;
7502    let kind = match stamp.kind {
7503        ScopeKind::Head => "head",
7504        ScopeKind::Worker => "worker",
7505        ScopeKind::Ephemeral => "ephemeral",
7506    };
7507    format!(
7508        "{kind}/{}/{}@{}",
7509        principal_label(&stamp.owner),
7510        bounded_log_text(&stamp.scope_ref),
7511        stamp.scope_epoch
7512    )
7513}
7514
7515/// The `scope=` field of a `route.open refused` line for an open refused
7516/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
7517/// The kind is not known yet, so the literal `requested` stands in its place.
7518/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
7519/// owner it has not been checked against the configured modules: a quoted
7520/// owner means the value was never validated. The bind proof never appears.
7521fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7522    let epoch = selector
7523        .scope_epoch
7524        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7525    format!(
7526        "requested/{}/{}@{epoch}",
7527        bounded_log_text(&principal_label(&selector.owner)),
7528        bounded_log_text(&selector.scope_ref),
7529    )
7530}
7531
7532fn principal_label(principal: &Principal) -> String {
7533    match principal {
7534        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7535        Principal::Direct => "direct".to_string(),
7536        other => format!("{other:?}"),
7537    }
7538}
7539
7540fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7541    match request {
7542        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7543        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7544        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7545        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7546        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7547    }
7548}
7549
7550fn parse_client_control_request(
7551    body: &[u8],
7552) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7553    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7554        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7555            Ok(probe)
7556                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7557                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7558            {
7559                ControlRequestBodyError::InvalidBody
7560            }
7561            Ok(_) => ControlRequestBodyError::UnknownOp,
7562            Err(_) => ControlRequestBodyError::InvalidBody,
7563        };
7564        (err, classification)
7565    })
7566}
7567
7568fn parse_module_control_request_from_module(
7569    body: &[u8],
7570) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7571    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7572        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7573            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7574            Ok(_) => ControlRequestBodyError::UnknownOp,
7575            Err(_) => ControlRequestBodyError::InvalidBody,
7576        };
7577        (err, classification)
7578    })
7579}
7580
7581#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7582enum ProviderRoleKind {
7583    ToolProvider,
7584    PipelineStage,
7585    ManagementSurface,
7586    InternalService,
7587}
7588
7589fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7590    match role {
7591        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7592        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7593        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7594        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7595    }
7596}
7597
7598fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7599    roles.iter().map(provider_role_kind).collect()
7600}
7601
7602/// Most refused scope records named individually in the log per sync; the
7603/// `refused` count on the accepted line is always complete.
7604const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7605
7606/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7607#[derive(Debug, Default, PartialEq, Eq)]
7608struct ScopeOutcomeCounts {
7609    created: usize,
7610    replaced: usize,
7611    updated: usize,
7612    unchanged: usize,
7613    refused: usize,
7614}
7615
7616impl ScopeOutcomeCounts {
7617    fn of(results: &[ScopeRecordResult]) -> Self {
7618        let mut counts = Self::default();
7619        for result in results {
7620            let slot = match result.outcome {
7621                ScopeRecordOutcome::Created => &mut counts.created,
7622                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7623                ScopeRecordOutcome::Updated => &mut counts.updated,
7624                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7625                ScopeRecordOutcome::Refused => &mut counts.refused,
7626            };
7627            *slot += 1;
7628        }
7629        counts
7630    }
7631}
7632
7633#[cfg(test)]
7634mod scope_outcome_count_tests {
7635    use super::*;
7636
7637    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7638        ScopeRecordResult {
7639            scope_ref: "r".to_string(),
7640            scope_epoch: 1,
7641            outcome,
7642            code: None,
7643            message: None,
7644            version: None,
7645            parent_state: None,
7646        }
7647    }
7648
7649    /// Each outcome lands in its own count, so a refused record can never be
7650    /// hidden inside the total the log already printed.
7651    #[test]
7652    fn every_outcome_is_counted_in_its_own_field() {
7653        let results = [
7654            result(ScopeRecordOutcome::Created),
7655            result(ScopeRecordOutcome::Created),
7656            result(ScopeRecordOutcome::Replaced),
7657            result(ScopeRecordOutcome::Updated),
7658            result(ScopeRecordOutcome::Unchanged),
7659            result(ScopeRecordOutcome::Refused),
7660            result(ScopeRecordOutcome::Refused),
7661            result(ScopeRecordOutcome::Refused),
7662        ];
7663        assert_eq!(
7664            ScopeOutcomeCounts::of(&results),
7665            ScopeOutcomeCounts {
7666                created: 2,
7667                replaced: 1,
7668                updated: 1,
7669                unchanged: 1,
7670                refused: 3,
7671            }
7672        );
7673    }
7674}
7675
7676/// Return whether a catalog change can create a newly violating live route.
7677/// Removing an attested claim is intentionally excluded: it makes fewer routes
7678/// forbidden and therefore must leave the existing route census untouched.
7679fn capability_census_trigger(
7680    old: Option<&CapabilityDeclarations>,
7681    new: Option<&CapabilityDeclarations>,
7682) -> bool {
7683    let old_provides = old
7684        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7685        .unwrap_or_default();
7686    let old_denies = old
7687        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7688        .unwrap_or_default();
7689    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7690        provides: Vec::new(),
7691        requires: Vec::new(),
7692        must_never_reach: Vec::new(),
7693    });
7694
7695    new.provides
7696        .iter()
7697        .any(|capability| !old_provides.contains(capability))
7698        || new
7699            .must_never_reach
7700            .iter()
7701            .any(|capability| !old_denies.contains(capability))
7702}
7703
7704/// Find the first capability an attested opener denies that an attested target
7705/// claims. Both manifests are live registry records, never cached or client data.
7706fn denied_capability<'a>(
7707    opening_manifest: &'a ModuleManifest,
7708    target_manifest: &ModuleManifest,
7709) -> Option<&'a str> {
7710    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7711    let target_capabilities = target_manifest.capabilities.as_ref()?;
7712    opening_capabilities
7713        .must_never_reach
7714        .iter()
7715        .find(|denied| {
7716            target_capabilities
7717                .provides
7718                .iter()
7719                .any(|provided| provided == *denied)
7720        })
7721        .map(String::as_str)
7722}
7723
7724fn catalog_update_frozen_field_message(
7725    registered: &ModuleManifest,
7726    provides: &[ProviderRole],
7727) -> Option<String> {
7728    let old_has_provides = !registered.provides.is_empty();
7729    let new_has_provides = !provides.is_empty();
7730    if old_has_provides != new_has_provides {
7731        return Some(format!(
7732            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7733            registered.module_id
7734        ));
7735    }
7736
7737    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7738        return Some(format!(
7739            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7740            registered.module_id
7741        ));
7742    }
7743
7744    let registered_concurrency = manifest_concurrency(registered);
7745    let mut candidate = registered.clone();
7746    candidate.provides = provides.to_vec();
7747    let candidate_concurrency = manifest_concurrency(&candidate);
7748    if candidate_concurrency != registered_concurrency {
7749        return Some(format!(
7750            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7751            registered.module_id, registered_concurrency, candidate_concurrency
7752        ));
7753    }
7754
7755    // control_ops live beside the manifest in the HELLO body, not inside
7756    // ModuleManifest, so a provides-only catalog.update cannot change them.
7757    None
7758}
7759
7760fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7761    manifest.provides.iter().any(is_routable_role)
7762}
7763
7764/// Returns the routable-provider concurrency subc should enforce for this manifest.
7765///
7766/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7767/// InternalService has no role-specific concurrency field, so it retains the
7768/// existing ModuleManaged default for backward compatibility.
7769fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7770    manifest
7771        .provides
7772        .iter()
7773        .find_map(|provider| match provider {
7774            ProviderRole::ToolProvider { concurrency, .. }
7775            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7776            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7777        })
7778        .unwrap_or(Concurrency::ModuleManaged)
7779}
7780
7781/// True when the manifest carries a ManagementSurface role whose concurrency
7782/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7783/// bytes because the typed manifest deliberately erases that distinction: the
7784/// default exists for wire compatibility, and this probe exists so the default
7785/// stays observable. Any parse irregularity returns false -- the caller only
7786/// logs, and a malformed body already failed registration upstream.
7787fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7788    let has_management_surface = manifest
7789        .provides
7790        .iter()
7791        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7792    if !has_management_surface {
7793        return false;
7794    }
7795    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7796        return false;
7797    };
7798    let Some(provides) = raw
7799        .get("manifest")
7800        .and_then(|manifest| manifest.get("provides"))
7801        .and_then(serde_json::Value::as_array)
7802    else {
7803        return false;
7804    };
7805    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7806    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7807    // against the management_surface_manifest_without_concurrency golden, not
7808    // recalled (the externally-tagged guess was this function's first bug).
7809    provides.iter().any(|role| {
7810        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7811            && role.get("concurrency").is_none()
7812    })
7813}
7814
7815fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7816    if peer_version != PROTOCOL_VERSION {
7817        return Err(format!(
7818            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7819        ));
7820    }
7821    Ok(PROTOCOL_VERSION)
7822}
7823
7824fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7825    Frame::build_with_version(
7826        response_version(frame),
7827        FrameType::Pong,
7828        frame.header.flags,
7829        0,
7830        0,
7831        frame.header.corr,
7832        Vec::new(),
7833    )
7834    .map_err(RouterError::FrameBuild)
7835}
7836
7837pub(crate) fn control_error_frame(
7838    frame: &Frame,
7839    code: &'static str,
7840    message: impl Into<String>,
7841) -> Result<Frame, RouterError> {
7842    control_error_body_frame(
7843        frame,
7844        ErrorBody {
7845            code: code.to_string(),
7846            message: message.into(),
7847            detail: None,
7848        },
7849    )
7850}
7851
7852fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7853    let body = serde_json::to_vec(&error).map_err(|err| {
7854        RouterError::backend(
7855            0,
7856            frame.header.corr,
7857            format!("failed to encode control ERROR: {err}"),
7858        )
7859    })?;
7860
7861    Frame::build_with_version(
7862        response_version(frame),
7863        FrameType::Error,
7864        control_flags(),
7865        0,
7866        0,
7867        frame.header.corr,
7868        body,
7869    )
7870    .map_err(RouterError::FrameBuild)
7871}
7872
7873fn control_response_body_frame<T: Serialize>(
7874    frame: &Frame,
7875    reply: &T,
7876    label: &'static str,
7877) -> Result<Frame, RouterError> {
7878    let body = serde_json::to_vec(reply).map_err(|err| {
7879        RouterError::backend(
7880            0,
7881            frame.header.corr,
7882            format!("failed to encode {label}: {err}"),
7883        )
7884    })?;
7885
7886    Frame::build_with_version(
7887        response_version(frame),
7888        FrameType::Response,
7889        control_flags(),
7890        0,
7891        0,
7892        frame.header.corr,
7893        body,
7894    )
7895    .map_err(RouterError::FrameBuild)
7896}
7897
7898/// Map a forwarding failure to the wire code a client sees.
7899///
7900/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7901/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7902/// chosen here decides whether a caller retries or gives up.
7903///
7904/// That makes attribution the load-bearing property, not merely having a code. A
7905/// permanent fault published as a retryable one produces a fleet-wide retry storm
7906/// against something that can never recover; a transient fault published as
7907/// permanent gives up on work that would have succeeded. Both look correct in a
7908/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7909/// the mapping per variant rather than merely asserting that some code exists.
7910///
7911/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7912/// two codes on the same side of the boundary passes it. Measured rather than
7913/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7914/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7915/// a test named for something else that happens to assert the string.
7916///
7917/// That accidental coverage is deliberately left alone rather than promoted to a
7918/// named test, because it guards a property this function does not promise.
7919/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7920/// specific code within a class, so identity is free to change and only the
7921/// partition is a contract. Splitting it out would assert a guarantee nothing
7922/// depends on — and a suite that promises more than the code does is the harder
7923/// thing to correct later, because the next reader cannot tell which assertions
7924/// are load-bearing.
7925///
7926/// Pin identity here the moment a consumer branches on a specific code.
7927fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7928    match err {
7929        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7930        ForwardingError::NoModuleConnection => "target_unavailable",
7931        ForwardingError::ModuleReloading { .. } => "module_reloading",
7932        ForwardingError::ClientRouteChannelExhausted { .. }
7933        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7934        ForwardingError::StaleModuleEndpoint
7935        | ForwardingError::UnknownReservation { .. }
7936        | ForwardingError::ConnectionClosing { .. }
7937        | ForwardingError::ClientEgressClosed { .. }
7938        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7939        // Only a swap candidate's registration can produce this, and it means
7940        // exactly what a second active HELLO for a live id means.
7941        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7942        ForwardingError::RelayCorrelationExhausted
7943        | ForwardingError::RouteOpenBuild(_)
7944        | ForwardingError::Poisoned => "forwarding_error",
7945    }
7946}
7947
7948fn response_version(frame: &Frame) -> u8 {
7949    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7950        frame.header.ver
7951    } else {
7952        PROTOCOL_VERSION
7953    }
7954}
7955
7956fn control_flags() -> Flags {
7957    Flags::new(false, Priority::Passive, false)
7958}
7959
7960/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7961/// channel. The target is the module (a client never saw the route), so this
7962/// takes the module path: delivered late rather than dropped when the module's
7963/// queue is momentarily full, and never closing its connection.
7964fn send_goodbye_target_best_effort(
7965    counters: &DaemonCounters,
7966    target: &GoodbyeTarget,
7967    context: &'static str,
7968) {
7969    let Ok(frame) = Frame::build_with_version(
7970        target.negotiated_ver,
7971        FrameType::Goodbye,
7972        control_flags(),
7973        target.channel,
7974        target.epoch,
7975        0,
7976        Vec::new(),
7977    ) else {
7978        return;
7979    };
7980    crate::forwarding::send_module_route_goodbye(
7981        counters,
7982        &target.sink,
7983        frame,
7984        target.module_id.as_deref(),
7985        context,
7986    );
7987}
7988
7989pub(crate) fn send_route_control_pushes(
7990    forwarding: &ForwardingTable,
7991    routes: Vec<EndpointRoute>,
7992    push: ClientControlPush,
7993) {
7994    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7995    for route in routes {
7996        let target = route.goodbye_target;
7997        if let Some((existing, channels)) = targets
7998            .iter_mut()
7999            .find(|(existing, _)| existing.connection_id == target.connection_id)
8000        {
8001            debug_assert_eq!(
8002                existing.negotiated_ver, target.negotiated_ver,
8003                "one connection cannot negotiate multiple frame versions"
8004            );
8005            if !channels.contains(&target.channel) {
8006                channels.push(target.channel);
8007            }
8008            continue;
8009        }
8010        let channel = target.channel;
8011        targets.push((target, vec![channel]));
8012    }
8013    for (target, mut channels) in targets {
8014        channels.sort_unstable();
8015        let mut push = push.clone();
8016        match &mut push {
8017            ClientControlPush::RouteClosing {
8018                channels: covered, ..
8019            }
8020            | ClientControlPush::RouteClosed {
8021                channels: covered, ..
8022            } => *covered = channels,
8023        }
8024        let body = match serde_json::to_vec(&push) {
8025            Ok(body) => body,
8026            Err(err) => {
8027                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8028                continue;
8029            }
8030        };
8031        let frame = match Frame::build_with_version(
8032            target.negotiated_ver,
8033            FrameType::Push,
8034            control_flags(),
8035            0,
8036            0,
8037            0,
8038            body.clone(),
8039        ) {
8040            Ok(frame) => frame,
8041            Err(err) => {
8042                warn!(
8043                    route_channel = target.channel,
8044                    error = %err,
8045                    "failed to build route lifecycle control PUSH frame"
8046                );
8047                continue;
8048            }
8049        };
8050        if let Err(err) = target.sink.try_send(frame) {
8051            if target.close_on_delivery_failure() {
8052                warn!(
8053                    target_connection_id = target.connection_id.get(),
8054                    route_channel = target.channel,
8055                    error = %err,
8056                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8057                );
8058                let _ = forwarding.escalate_client_delivery_failure(
8059                    target.connection_id,
8060                    target.channel,
8061                    target.epoch,
8062                    CloseReason::new(
8063                        "route_lifecycle_push_delivery_failed",
8064                        format!(
8065                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8066                            target.channel
8067                        ),
8068                    ),
8069                    crate::forwarding::UndeliveredFrame {
8070                        module_id: target.module_id.as_deref(),
8071                        sink: &target.sink,
8072                    },
8073                );
8074            }
8075        }
8076    }
8077}
8078
8079#[cfg(test)]
8080mod tests {
8081    mod activity_read {
8082        include!("activity_read_tests.rs");
8083    }
8084
8085    mod lifecycle_activity {
8086        include!("control_lifecycle_tests.rs");
8087    }
8088    #[cfg(unix)]
8089    #[tokio::test]
8090    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8091        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8092        let path = dir.join("subc.jsonc");
8093        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8094            "program":"/bin/sleep","args":["60"],"protocol":"none",
8095            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8096        }}}).to_string()).unwrap();
8097        let mut configured = crate::daemon_config::load(&path)
8098            .unwrap()
8099            .unwrap()
8100            .modules
8101            .pop()
8102            .unwrap();
8103        let registry = std::sync::Arc::new(crate::Registry::default());
8104        let handle = crate::SupervisorHandle::new();
8105        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8106            .with_handle(handle.clone());
8107        let module = supervisor
8108            .supervise_configured_with_health(
8109                configured.module_spec(),
8110                true,
8111                configured.health.clone(),
8112                None,
8113                configured.restart,
8114            )
8115            .unwrap();
8116        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8117        let before = module.status().unwrap().pid;
8118        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8119        configured.health.cadence = std::time::Duration::from_secs(3600);
8120        let health_only = handler
8121            .reconcile_supervised_modules(
8122                &supervisor,
8123                vec![configured.clone()],
8124                false,
8125                Principal::Direct,
8126            )
8127            .await
8128            .unwrap();
8129        assert!(
8130            health_only.0.changed_pending_reload.is_empty(),
8131            "health policy is already applied live"
8132        );
8133        assert_eq!(module.status().unwrap().pid, before);
8134        assert_eq!(
8135            module.configuration().unwrap().1.http,
8136            configured.health.http
8137        );
8138        configured.args = vec!["61".into()];
8139        let launch = handler
8140            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8141            .await
8142            .unwrap();
8143        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8144        assert_eq!(
8145            module.status().unwrap().pid,
8146            before,
8147            "a launch edit is stored until reload"
8148        );
8149        module.drain().await.unwrap();
8150    }
8151    use cortexkit_test_support::ScratchDir;
8152    use std::{
8153        collections::BTreeMap,
8154        fmt,
8155        path::PathBuf,
8156        sync::{Arc, Mutex},
8157        time::Duration,
8158    };
8159
8160    use serde_json::{json, Value};
8161    use subc_protocol::{
8162        manifest::{
8163            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8164            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8165        },
8166        session::HealthStatus,
8167        FrameType,
8168    };
8169
8170    use super::*;
8171    use crate::{
8172        forwarding::{DataRoute, DataRouteState},
8173        registry::ChannelState,
8174        router::FrameSink,
8175        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8176        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8177        RouteCtx, Router,
8178    };
8179    use tokio::{
8180        sync::mpsc,
8181        time::{sleep, Instant},
8182    };
8183    use tracing::{
8184        field::{Field, Visit},
8185        Event, Subscriber,
8186    };
8187    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8188
8189    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8190    ///
8191    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8192    /// is only populated for `tests/*.rs` integration test binaries -- this file
8193    /// compiles as part of the library target, which gets neither. This test's
8194    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8195    /// and the sibling binary lives two directories up at
8196    /// `<target-dir>/<profile>/fake-aft-stub`.
8197    ///
8198    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8199    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8200    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8201    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8202    /// `NotFound`, which reads as a broken test rather than an unbuilt
8203    /// dependency -- so state the cause and the remedy instead. Deliberately a
8204    /// panic and not a silent skip: a test that quietly passes when it could not
8205    /// run is worse than one that fails, because it reports health it never
8206    /// verified.
8207    fn fake_aft_stub_path() -> PathBuf {
8208        let mut path = std::env::current_exe().expect("current_exe available in tests");
8209        path.pop(); // .../deps/
8210        path.pop(); // .../<profile>/
8211        path.push(if cfg!(windows) {
8212            "fake-aft-stub.exe"
8213        } else {
8214            "fake-aft-stub"
8215        });
8216        assert!(
8217            path.exists(),
8218            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8219             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8220            path.display()
8221        );
8222        path
8223    }
8224
8225    /// Whether clients retry `code` in place: the predicate itself, never a copy
8226    /// of its set. A copied list breaks silently when a code is added to or
8227    /// removed from the real one, and a stale copy here would let exactly the
8228    /// failure this test exists to catch pass.
8229    fn client_retries(code: &str) -> bool {
8230        subc_protocol::error_codes::is_retryable_route_open(code)
8231    }
8232
8233    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8234    /// of failure is worse than publishing none. A permanent fault dressed as
8235    /// retryable makes every client in the fleet retry forever against something
8236    /// that cannot recover; a transient fault dressed as permanent abandons work
8237    /// that would have succeeded.
8238    ///
8239    /// Asserting "a code exists" cannot catch either, because the string is free
8240    /// to say anything. This enumerates every variant and pins which side of the
8241    /// retry boundary it lands on, so a new variant must be classified here
8242    /// deliberately rather than inheriting whichever arm it was appended to.
8243    #[test]
8244    fn retryability_of_forwarding_codes_matches_the_failure() {
8245        // Transient by nature: the target is booting, reloading, or its endpoint
8246        // was swapped mid-flight. Retrying is how these resolve.
8247        let transient = [
8248            ForwardingError::NoModuleConnection,
8249            ForwardingError::ModuleReloading {
8250                module_id: "m".into(),
8251            },
8252            ForwardingError::StaleModuleEndpoint,
8253            ForwardingError::UnknownReservation {
8254                client_channel: 1,
8255                module_channel: 1,
8256            },
8257            ForwardingError::ConnectionClosing {
8258                connection_id: ConnectionId::new(1),
8259            },
8260            ForwardingError::ClientEgressClosed {
8261                connection_id: ConnectionId::new(1),
8262            },
8263            ForwardingError::ModuleEgressUnavailable {
8264                connection_id: ConnectionId::new(1),
8265            },
8266        ];
8267        for err in transient {
8268            let code = forwarding_error_code(&err);
8269            assert!(
8270                client_retries(code),
8271                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8272            );
8273        }
8274
8275        // Not fixed by retrying. Channel and correlation exhaustion need the
8276        // caller to close routes, and a poisoned lock is a daemon that cannot
8277        // recover at all — the worst thing to advertise as retryable, since every
8278        // client would storm a daemon that will never answer.
8279        let permanent = [
8280            ForwardingError::ConnectionRoleConflict {
8281                connection_id: ConnectionId::new(1),
8282            },
8283            ForwardingError::ClientRouteChannelExhausted {
8284                connection_id: ConnectionId::new(1),
8285            },
8286            ForwardingError::ModuleRouteChannelExhausted {
8287                endpoint: ModuleEndpointId {
8288                    connection_id: ConnectionId::new(1),
8289                    generation: 1,
8290                },
8291            },
8292            ForwardingError::RelayCorrelationExhausted,
8293            ForwardingError::RouteOpenBuild("x".into()),
8294            ForwardingError::Poisoned,
8295        ];
8296        for err in permanent {
8297            let code = forwarding_error_code(&err);
8298            assert!(
8299                !client_retries(code),
8300                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8301            );
8302        }
8303    }
8304
8305    /// The principal is the daemon's answer to "who is calling", and modules
8306    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8307    /// plexus gates connector invocation. So a stamp is an authorization input in
8308    /// another process, not a label — and both possible answers SUCCEED, which is
8309    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8310    /// first-party capability to something that never proved it; a supervised one
8311    /// stamped `Direct` silently strips a module of capability it is entitled to.
8312    ///
8313    /// Neither shows up in a test that only checks the bind succeeded. Before this
8314    /// test the only coverage was accidental —
8315    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8316    /// stamped principal on its way past, so narrowing that wire-shape test to its
8317    /// stated subject would have deleted the last assertion on this value. It
8318    /// still asserts the stamp, which is now redundancy rather than the only
8319    /// guard: both fail under the same mutation, and this one names the reason.
8320    /// SCOPE: this handler's supervisor has spawned nothing, so
8321    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8322    /// is unreachable here. Both assertions below are refusals, and a mutant that
8323    /// refuses everything would satisfy them.
8324    ///
8325    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8326    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8327    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8328    /// at source rather than assumed, since a citation is a claim about another
8329    /// file and ages like one. Recorded because a harness that structurally
8330    /// cannot reach an arm reports "none" for that arm identically to one that
8331    /// covers it and found nothing.
8332    #[tokio::test]
8333    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8334        let handler = ControlHandler::default();
8335        let frame =
8336            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8337
8338        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8339        // any process holding the connection file. Nothing was proved, so nothing
8340        // may be granted beyond the unattested floor.
8341        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8342        assert_eq!(
8343            stamped,
8344            Principal::Direct,
8345            "a caller that proved nothing must not be stamped as a supervised module"
8346        );
8347
8348        // A claimed module_id with a nonce no supervised child was given is a
8349        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8350        // quietly demoted to Direct, or an impersonation attempt looks identical
8351        // to an ordinary unattested connection.
8352        let forged = handler
8353            .route_open_principal(
8354                &frame,
8355                Some(ConsumerIdentity {
8356                    module_id: "aft".to_string(),
8357                    launch_nonce: "not-a-real-nonce".to_string(),
8358                }),
8359            )
8360            .unwrap();
8361        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8362        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8363    }
8364
8365    /// The test above hands `route_open_principal` an identity it built itself,
8366    /// which proves the stamping rule and nothing about where the identity comes
8367    /// from. The real producer is a wire body, and the two are joined by a serde
8368    /// field name that nothing else asserts.
8369    ///
8370    /// That join fails quietly in one specific way: an unrecognised key is simply
8371    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8372    /// yields `None` and every supervised module silently drops to `Direct`.
8373    /// Capability-wise that is the safe direction, but it surfaces far from its
8374    /// cause — as a module mysteriously refused bash — and it would pass every
8375    /// test that builds its own input.
8376    ///
8377    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8378    /// would break every client the moment the daemon gains a field, trading a
8379    /// quiet demotion for a hard refusal on additive change. Asserting the join
8380    /// instead means a rename breaks a test here rather than the fleet.
8381    #[test]
8382    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8383        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8384        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8385        let ClientControlRequest::RouteOpen {
8386            consumer_identity, ..
8387        } = parsed
8388        else {
8389            panic!("route.open body must parse as RouteOpen");
8390        };
8391        assert_eq!(
8392            consumer_identity,
8393            Some(ConsumerIdentity {
8394                module_id: "aft".to_string(),
8395                launch_nonce: "n".to_string(),
8396            }),
8397            "the wire field name must reach the value route_open_principal reads"
8398        );
8399    }
8400
8401    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8402        ModuleManifest::builder(module_id, "0.1.0")
8403            .protocol_ver(protocol_ver)
8404            .provides(vec![ProviderRole::ToolProvider {
8405                tools: vec![Tool {
8406                    name: "read".to_string(),
8407                    description: None,
8408                    execution_mode: ExecutionMode::Pure,
8409                    schema: json!({"type": "object"}),
8410                }],
8411                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8412                concurrency: Concurrency::ModuleManaged,
8413                emits_push: true,
8414                sub_supervises: true,
8415            }])
8416            .build()
8417    }
8418
8419    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8420        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8421    }
8422
8423    fn hello_frame_with_control_ops(
8424        module_id: &str,
8425        protocol_ver: u8,
8426        corr: u64,
8427        control_ops: Option<Vec<String>>,
8428    ) -> Frame {
8429        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8430    }
8431
8432    fn hello_frame_with_nonce(
8433        module_id: &str,
8434        protocol_ver: u8,
8435        corr: u64,
8436        launch_nonce: Option<&str>,
8437    ) -> Frame {
8438        hello_frame_full(
8439            module_id,
8440            protocol_ver,
8441            corr,
8442            None,
8443            launch_nonce.map(ToOwned::to_owned),
8444        )
8445    }
8446
8447    fn hello_frame_full(
8448        module_id: &str,
8449        protocol_ver: u8,
8450        corr: u64,
8451        control_ops: Option<Vec<String>>,
8452        launch_nonce: Option<String>,
8453    ) -> Frame {
8454        let body = serde_json::to_vec(&ModuleHelloBody {
8455            manifest: manifest(module_id, protocol_ver),
8456            protocol_ver,
8457            control_ops,
8458            launch_nonce,
8459        })
8460        .unwrap();
8461        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8462    }
8463
8464    fn hello_frame_with_rich_ui(
8465        module_id: &str,
8466        protocol_ver: u8,
8467        corr: u64,
8468        launch_nonce: Option<&str>,
8469        rich_ui: Value,
8470    ) -> Frame {
8471        let mut body = serde_json::to_value(ModuleHelloBody {
8472            manifest: manifest(module_id, protocol_ver),
8473            protocol_ver,
8474            control_ops: None,
8475            launch_nonce: launch_nonce.map(str::to_string),
8476        })
8477        .unwrap();
8478        body["manifest"]["rich_ui"] = rich_ui;
8479        Frame::build(
8480            FrameType::Hello,
8481            control_flags(),
8482            0,
8483            0,
8484            corr,
8485            serde_json::to_vec(&body).unwrap(),
8486        )
8487        .unwrap()
8488    }
8489
8490    fn non_routable_hello_frame_with_control_ops(
8491        module_id: &str,
8492        corr: u64,
8493        control_ops: Option<Vec<String>>,
8494    ) -> Frame {
8495        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8496        manifest.provides.clear();
8497        let body = serde_json::to_vec(&ModuleHelloBody {
8498            manifest,
8499            protocol_ver: PROTOCOL_VERSION,
8500            control_ops,
8501            launch_nonce: None,
8502        })
8503        .unwrap();
8504        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8505    }
8506
8507    fn capability_grammar_hello_frame(
8508        capabilities: Value,
8509        runtime_computed: Option<Value>,
8510        corr: u64,
8511    ) -> Frame {
8512        let mut body = serde_json::to_value(ModuleHelloBody {
8513            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8514            protocol_ver: PROTOCOL_VERSION,
8515            control_ops: None,
8516            launch_nonce: None,
8517        })
8518        .expect("HELLO body serializes");
8519        body["manifest"]["capabilities"] = capabilities;
8520        if let Some(runtime_computed) = runtime_computed {
8521            body["runtime_computed"] = runtime_computed;
8522        }
8523        Frame::build(
8524            FrameType::Hello,
8525            control_flags(),
8526            0,
8527            0,
8528            corr,
8529            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8530        )
8531        .expect("HELLO frame builds")
8532    }
8533
8534    fn channel_request(channel: u16, corr: u64) -> Frame {
8535        Frame::build(
8536            FrameType::Request,
8537            Flags::new(true, Priority::Interactive, false),
8538            channel,
8539            0,
8540            corr,
8541            b"opaque".to_vec(),
8542        )
8543        .unwrap()
8544    }
8545
8546    fn route_ctx(
8547        connection_id: ConnectionId,
8548    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8549        let (tx, rx) = mpsc::channel(8);
8550        (
8551            RouteCtx {
8552                connection_id,
8553                egress: FrameSink::new(tx),
8554            },
8555            rx,
8556        )
8557    }
8558
8559    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8560        serde_json::from_slice(&frame.body).unwrap()
8561    }
8562
8563    /// Register a module over a connection that has a sink and return the
8564    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8565    /// module's own sink rather than returning it as a reply, so the ack is
8566    /// taken off `rx` here and whatever the test reads next is what followed it.
8567    async fn hello_via_sink(
8568        handler: &ControlHandler,
8569        ctx: &RouteCtx,
8570        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8571        hello: Frame,
8572    ) -> Frame {
8573        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8574        assert!(
8575            replies.is_empty(),
8576            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8577        );
8578        let ack = rx
8579            .try_recv()
8580            .expect("HELLO_ACK is queued on the module sink")
8581            .frame;
8582        assert_eq!(ack.header.ty, FrameType::HelloAck);
8583        ack
8584    }
8585
8586    fn parse_error(frame: &Frame) -> Value {
8587        serde_json::from_slice(&frame.body).unwrap()
8588    }
8589
8590    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8591        serde_json::from_slice(&frame.body).unwrap()
8592    }
8593
8594    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8595        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8596            route_channel,
8597            route_epoch: 0,
8598            kind,
8599        })
8600        .unwrap();
8601        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8602    }
8603
8604    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8605        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8606            module_id: module_id.to_string(),
8607        })
8608        .unwrap();
8609        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8610    }
8611
8612    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8613        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8614    }
8615
8616    fn route_open_frame_with_consumer_capabilities(
8617        corr: u64,
8618        module_id: &str,
8619        project_root: ScratchDir,
8620        consumer_capabilities: Option<Vec<String>>,
8621    ) -> Frame {
8622        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8623            target: RouteTarget::ToolProvider {
8624                module_id: module_id.to_string(),
8625            },
8626            identity: BindIdentity::new(
8627                project_root.path().to_path_buf(),
8628                "unit".to_string(),
8629                "session".to_string(),
8630            ),
8631            consumer_identity: None,
8632            consumer_capabilities,
8633            role_versions: None,
8634            admission_facts: None,
8635            scope: None,
8636        })
8637        .unwrap();
8638        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8639    }
8640
8641    fn route_open_frame_with_role_versions(
8642        corr: u64,
8643        module_id: &str,
8644        project_root: ScratchDir,
8645        role_versions: Option<BTreeMap<String, String>>,
8646    ) -> Frame {
8647        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8648            target: RouteTarget::ToolProvider {
8649                module_id: module_id.to_string(),
8650            },
8651            identity: BindIdentity::new(
8652                project_root.path().to_path_buf(),
8653                "unit".to_string(),
8654                format!("session-{corr}"),
8655            ),
8656            consumer_identity: None,
8657            consumer_capabilities: None,
8658            role_versions,
8659            admission_facts: None,
8660            scope: None,
8661        })
8662        .unwrap();
8663        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8664    }
8665
8666    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8667        entries
8668            .iter()
8669            .map(|(role, version)| (role.to_string(), version.to_string()))
8670            .collect()
8671    }
8672
8673    fn route_open_frame_with_admission_facts(
8674        corr: u64,
8675        module_id: &str,
8676        project_root: ScratchDir,
8677        consumer_identity: Option<subc_control::ConsumerIdentity>,
8678        facts: Option<Value>,
8679    ) -> Frame {
8680        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8681            target: RouteTarget::ToolProvider {
8682                module_id: module_id.to_string(),
8683            },
8684            identity: BindIdentity::new(
8685                project_root.path().to_path_buf(),
8686                "unit".to_string(),
8687                format!("session-{corr}"),
8688            ),
8689            consumer_identity,
8690            consumer_capabilities: None,
8691            role_versions: None,
8692            admission_facts: facts,
8693            scope: None,
8694        })
8695        .unwrap();
8696        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8697    }
8698
8699    #[derive(Clone, Default)]
8700    struct EventCapture {
8701        events: Arc<Mutex<Vec<CapturedEvent>>>,
8702    }
8703
8704    #[derive(Clone, Debug)]
8705    struct CapturedEvent {
8706        target: String,
8707        level: tracing::Level,
8708        fields: BTreeMap<String, String>,
8709    }
8710
8711    impl EventCapture {
8712        fn events(&self) -> Vec<CapturedEvent> {
8713            self.events.lock().unwrap().clone()
8714        }
8715    }
8716
8717    impl<S> Layer<S> for EventCapture
8718    where
8719        S: Subscriber,
8720    {
8721        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8722            let mut visitor = EventFieldVisitor::default();
8723            event.record(&mut visitor);
8724            self.events.lock().unwrap().push(CapturedEvent {
8725                target: event.metadata().target().to_string(),
8726                level: *event.metadata().level(),
8727                fields: visitor.fields,
8728            });
8729        }
8730    }
8731
8732    #[derive(Default)]
8733    struct EventFieldVisitor {
8734        fields: BTreeMap<String, String>,
8735    }
8736
8737    impl Visit for EventFieldVisitor {
8738        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8739            self.fields
8740                .insert(field.name().to_string(), format!("{value:?}"));
8741        }
8742    }
8743
8744    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8745        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8746            status,
8747            detail: Some("warming".to_string()),
8748            metrics: Some(json!({"queue_depth": 3})),
8749        })
8750        .unwrap();
8751        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8752    }
8753
8754    fn route_bind_ack(corr: u64) -> Frame {
8755        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8756        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8757    }
8758
8759    fn unique_project_root(label: &str) -> ScratchDir {
8760        ScratchDir::new(label)
8761    }
8762
8763    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8764        match parse_route_poll(frame) {
8765            ClientControlResponse::RoutePoll {
8766                status: None,
8767                live: Some(live),
8768                ..
8769            } => assert_eq!(live, expected_live),
8770            other => panic!("unexpected route.poll response: {other:?}"),
8771        }
8772    }
8773
8774    fn bind_liveness_route(
8775        registry: &Registry,
8776        forwarding: &ForwardingTable,
8777        module_id: &str,
8778    ) -> (RouteCtx, u16, u32) {
8779        let module_connection = ConnectionId::new(101);
8780        let client_connection = ConnectionId::new(202);
8781        let registration = registry
8782            .register_with_control_ops(
8783                manifest(module_id, PROTOCOL_VERSION),
8784                PROTOCOL_VERSION,
8785                module_connection,
8786                module_baseline_control_ops(),
8787            )
8788            .unwrap();
8789        let (module_tx, _module_rx) = mpsc::channel(8);
8790        let endpoint = forwarding
8791            .register_module_connection(
8792                module_connection,
8793                module_id.to_string(),
8794                PROTOCOL_VERSION,
8795                manifest_concurrency(&registration.manifest),
8796                FrameSink::new(module_tx),
8797            )
8798            .unwrap();
8799        let (client_ctx, _client_rx) = route_ctx(client_connection);
8800        let pending = forwarding
8801            .begin_route_bind_relay_for_test(
8802                client_connection,
8803                client_ctx.egress.clone(),
8804                1,
8805                module_id,
8806            )
8807            .unwrap();
8808        assert_eq!(pending.endpoint, endpoint);
8809        let route_channel = pending.client_channel;
8810        let route_epoch = pending.client_epoch;
8811        forwarding
8812            .complete_pending_relay(
8813                module_connection,
8814                pending.corr,
8815                RouteBindRelayOutcome::Accepted,
8816            )
8817            .unwrap();
8818        (client_ctx, route_channel, route_epoch)
8819    }
8820
8821    struct FakeProcessLiveness {
8822        live: Option<bool>,
8823    }
8824
8825    impl ModuleProcessLiveness for FakeProcessLiveness {
8826        fn process_live(&self, _module_id: &str) -> Option<bool> {
8827            self.live
8828        }
8829    }
8830
8831    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8832    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8833    {
8834        let registry = Arc::new(Registry::default());
8835        let supervisor_handle = SupervisorHandle::new();
8836        let supervisor = Supervisor::new_for_test(
8837            Arc::clone(&registry),
8838            RestartPolicy::new(1, Duration::from_millis(10)),
8839        )
8840        .with_handle(supervisor_handle.clone());
8841        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8842        let module = supervisor
8843            .spawn(ModuleSpec {
8844                module_id: "stderr-tail-wire".to_string(),
8845                program: fake_aft_stub_path(),
8846                args: Vec::new(),
8847                env: vec![
8848                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8849                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8850                ],
8851                reserved: false,
8852                reserved_prefixes: Vec::new(),
8853                protocol: ModuleProtocol::Subc,
8854                overlap: Default::default(),
8855            })
8856            .unwrap();
8857
8858        let deadline = Instant::now() + Duration::from_secs(5);
8859        loop {
8860            let tail = module.stderr_tail(None, None);
8861            if tail
8862                .entries
8863                .iter()
8864                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8865                && tail.entries.iter().any(|entry| {
8866                    matches!(
8867                        entry,
8868                        TailEntry::Line {
8869                            truncated: true,
8870                            ..
8871                        }
8872                    )
8873                })
8874            {
8875                break;
8876            }
8877            assert!(
8878                Instant::now() < deadline,
8879                "module did not produce a truncated line and restart boundary: {tail:?}"
8880            );
8881            sleep(Duration::from_millis(10)).await;
8882        }
8883
8884        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8885        let request = ClientControlRequest::SupervisorStderrTail {
8886            module_id: "stderr-tail-wire".to_string(),
8887            max_lines: None,
8888            max_bytes: None,
8889        };
8890        let frame = Frame::build(
8891            FrameType::Request,
8892            control_flags(),
8893            0,
8894            0,
8895            1,
8896            serde_json::to_vec(&request).unwrap(),
8897        )
8898        .unwrap();
8899        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8900        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8901        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8902            serde_json::from_slice(&responses[0].body).unwrap()
8903        else {
8904            panic!("expected supervisor.stderr_tail response");
8905        };
8906
8907        assert!(
8908            tail.entries
8909                .iter()
8910                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8911            "the control response lost the restart boundary"
8912        );
8913        let Some(StderrTailEntry::Line {
8914            text,
8915            truncated,
8916            at_ms,
8917        }) = tail.entries.iter().find(|entry| {
8918            matches!(
8919                entry,
8920                StderrTailEntry::Line {
8921                    truncated: true,
8922                    ..
8923                }
8924            )
8925        })
8926        else {
8927            panic!("the control response lost the truncated line");
8928        };
8929        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8930        assert!(*truncated);
8931        assert!(
8932            at_ms.is_some(),
8933            "the control response lost the line's capture time"
8934        );
8935    }
8936
8937    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8938    /// read done on the worker thread would stall every other task until it
8939    /// finished; the read must run off the worker so this test's own task keeps
8940    /// running while the read is paused.
8941    #[tokio::test(flavor = "current_thread")]
8942    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8943        let dir = ScratchDir::new("terminals-off-worker");
8944        let journal_path = dir.join("terminals.jsonl");
8945        let registry = Arc::new(Registry::default());
8946        let supervisor_handle = SupervisorHandle::new();
8947        let supervisor =
8948            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8949                .with_handle(supervisor_handle.clone())
8950                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8951        let module = supervisor
8952            .spawn(ModuleSpec {
8953                module_id: "terminal-off-worker".to_string(),
8954                program: fake_aft_stub_path(),
8955                args: Vec::new(),
8956                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8957                reserved: false,
8958                reserved_prefixes: Vec::new(),
8959                protocol: ModuleProtocol::Subc,
8960                overlap: Default::default(),
8961            })
8962            .unwrap();
8963        let deadline = Instant::now() + Duration::from_secs(5);
8964        while module.terminal_history().entries.len() != 2 {
8965            assert!(Instant::now() < deadline, "module did not record two exits");
8966            sleep(Duration::from_millis(10)).await;
8967        }
8968
8969        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8970        let handler =
8971            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8972        let frame = Frame::build(
8973            FrameType::Request,
8974            control_flags(),
8975            0,
8976            0,
8977            1,
8978            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8979                module_id: "terminal-off-worker".to_string(),
8980            })
8981            .unwrap(),
8982        )
8983        .unwrap();
8984        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8985        let spawned_at = std::time::Instant::now();
8986        let read = tokio::spawn({
8987            let handler = Arc::clone(&handler);
8988            async move { handler.handle_control_frame(&ctx, frame).await }
8989        });
8990        // Waiting for the pause from a blocking thread keeps this task pending,
8991        // so the runtime's single worker is free to run the read task.
8992        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8993            .await
8994            .unwrap()
8995            .expect("the history read reached its pause");
8996        let elapsed = spawned_at.elapsed();
8997        assert!(
8998            elapsed < Duration::from_secs(2) && !read.is_finished(),
8999            "this task could not run while the history read was paused \
9000             (resumed after {elapsed:?}, read finished: {})",
9001            read.is_finished()
9002        );
9003
9004        drop(release);
9005        let responses = read.await.unwrap().unwrap();
9006        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9007        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9008            panic!("expected supervisor.terminals response");
9009        };
9010        assert_eq!(terminals.entries.len(), 2);
9011        assert_eq!(terminals.journal_skipped_lines, 0);
9012        assert_eq!(terminals.journal_read_errors, 0);
9013    }
9014
9015    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9016    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9017        let registry = Arc::new(Registry::default());
9018        let supervisor_handle = SupervisorHandle::new();
9019        let supervisor =
9020            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9021                .with_handle(supervisor_handle.clone());
9022        let module = supervisor
9023            .spawn(ModuleSpec {
9024                module_id: "terminal-golden".to_string(),
9025                program: fake_aft_stub_path(),
9026                args: Vec::new(),
9027                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9028                reserved: false,
9029                reserved_prefixes: Vec::new(),
9030                protocol: ModuleProtocol::Subc,
9031                overlap: Default::default(),
9032            })
9033            .unwrap();
9034
9035        let deadline = Instant::now() + Duration::from_secs(5);
9036        while module.terminal_history().entries.len() != 2 {
9037            assert!(
9038                Instant::now() < deadline,
9039                "module did not retain two terminal exits: {:?}",
9040                module.terminal_history()
9041            );
9042            sleep(Duration::from_millis(10)).await;
9043        }
9044
9045        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9046        let request = ClientControlRequest::SupervisorTerminals {
9047            module_id: "terminal-golden".to_string(),
9048        };
9049        let frame = Frame::build(
9050            FrameType::Request,
9051            control_flags(),
9052            0,
9053            0,
9054            1,
9055            serde_json::to_vec(&request).unwrap(),
9056        )
9057        .unwrap();
9058        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9059        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9060        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9061        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9062            panic!("expected supervisor.terminals response");
9063        };
9064        assert_eq!(terminals.entries.len(), 2);
9065        assert_eq!(terminals.dropped, 0);
9066
9067        let mut rendered = serde_json::to_value(response).unwrap();
9068        // Wall-clock fields are the observation contract, but not stable fixture
9069        // bytes; normalize only them after the real handler has shaped the response.
9070        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9071        for (index, entry) in rendered["entries"]
9072            .as_array_mut()
9073            .expect("terminal response entries array")
9074            .iter_mut()
9075            .enumerate()
9076        {
9077            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9078        }
9079
9080        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9081            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9082        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9083        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9084            std::fs::write(&golden_path, &serialized).unwrap();
9085        }
9086        let expected: Value =
9087            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9088        assert_eq!(rendered, expected);
9089    }
9090
9091    #[test]
9092    fn hello_registers_manifest_and_returns_ack() {
9093        let registry = Arc::new(Registry::default());
9094        let handler = ControlHandler::new(Arc::clone(&registry));
9095        let conn = ConnectionId::new(1);
9096
9097        let responses = handler
9098            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9099            .unwrap();
9100
9101        assert_eq!(responses.len(), 1);
9102        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9103        assert_eq!(responses[0].header.channel, 0);
9104        assert_eq!(responses[0].header.corr, 7);
9105        let ack = parse_ack(&responses[0]);
9106        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9107        assert!(ack
9108            .subc_capabilities
9109            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9110        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9111        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9112        assert!(ack
9113            .subc_ops
9114            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9115        assert!(ack
9116            .subc_ops
9117            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9118
9119        let registration = registry.get_module("aft").unwrap().unwrap();
9120        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9121        assert_eq!(registration.state, ChannelState::Active);
9122        assert_eq!(registration.connection_id, conn);
9123        assert_eq!(registration.control_ops, module_baseline_control_ops());
9124    }
9125
9126    #[test]
9127    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9128        let invalid_identifiers = [
9129            ("case_change", "credentials-Provider/v1"),
9130            ("leading_zero", "credentials-provider/v01"),
9131            ("trailing_hyphen", "credentials-provider-/v1"),
9132            ("consecutive_hyphens", "credentials--provider/v1"),
9133            ("uppercase", "Credentials-provider/v1"),
9134            ("missing_v", "credentials-provider/1"),
9135            ("whitespace", "credentials provider/v1"),
9136            ("zero_version", "credentials-provider/v0"),
9137            ("out_of_range_version", "credentials-provider/v4294967296"),
9138            (
9139                "overlength_name",
9140                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9141            ),
9142        ];
9143        let mut cases = invalid_identifiers
9144            .into_iter()
9145            .map(|(name, identifier)| {
9146                (
9147                    format!("identifier_{name}"),
9148                    "capabilities.provides[0]".to_string(),
9149                    identifier.to_string(),
9150                    json!({ "provides": [identifier] }),
9151                    None,
9152                )
9153            })
9154            .collect::<Vec<_>>();
9155        cases.extend([
9156            (
9157                "unknown_need".to_string(),
9158                "capabilities.requires[0].need".to_string(),
9159                "deferred".to_string(),
9160                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9161                None,
9162            ),
9163            (
9164                "duplicate_provides".to_string(),
9165                "capabilities.provides[1]".to_string(),
9166                "credentials-provider/v1".to_string(),
9167                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9168                None,
9169            ),
9170            (
9171                "duplicate_must_never_reach".to_string(),
9172                "capabilities.must_never_reach[1]".to_string(),
9173                "credentials-provider/v1".to_string(),
9174                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9175                None,
9176            ),
9177            (
9178                "duplicate_requires_same_need".to_string(),
9179                "capabilities.requires[1]".to_string(),
9180                "credentials-provider/v1".to_string(),
9181                json!({ "requires": [
9182                    { "capability": "credentials-provider/v1", "need": "required" },
9183                    { "capability": "credentials-provider/v1", "need": "required" }
9184                ] }),
9185                None,
9186            ),
9187            (
9188                "duplicate_requires_conflicting_need".to_string(),
9189                "capabilities.requires[1]".to_string(),
9190                "credentials-provider/v1".to_string(),
9191                json!({ "requires": [
9192                    { "capability": "credentials-provider/v1", "need": "required" },
9193                    { "capability": "credentials-provider/v1", "need": "optional" }
9194                ] }),
9195                None,
9196            ),
9197            (
9198                "capabilities_root_pointer".to_string(),
9199                "runtime_computed[0]".to_string(),
9200                "/capabilities".to_string(),
9201                json!({}),
9202                Some(json!(["/capabilities"])),
9203            ),
9204            (
9205                "capabilities_descendant_pointer".to_string(),
9206                "runtime_computed[0]".to_string(),
9207                "/capabilities/provides".to_string(),
9208                json!({}),
9209                Some(json!(["/capabilities/provides"])),
9210            ),
9211            (
9212                "malformed_pointer_without_leading_slash".to_string(),
9213                "runtime_computed[0]".to_string(),
9214                "capabilities".to_string(),
9215                json!({}),
9216                Some(json!(["capabilities"])),
9217            ),
9218            (
9219                "malformed_pointer_escape".to_string(),
9220                "runtime_computed[0]".to_string(),
9221                "/roles/~2/tools".to_string(),
9222                json!({}),
9223                Some(json!(["/roles/~2/tools"])),
9224            ),
9225            (
9226                "unknown_capabilities_field".to_string(),
9227                "capabilities.future".to_string(),
9228                "<array>".to_string(),
9229                json!({ "future": [] }),
9230                None,
9231            ),
9232        ]);
9233
9234        for (index, (name, field, value, capabilities, runtime_computed)) in
9235            cases.into_iter().enumerate()
9236        {
9237            let registry = Arc::new(Registry::default());
9238            let handler = ControlHandler::new(Arc::clone(&registry));
9239            let response = handler
9240                .handle_control(
9241                    ConnectionId::new((index + 1) as u64),
9242                    capability_grammar_hello_frame(
9243                        capabilities,
9244                        runtime_computed,
9245                        index as u64 + 1,
9246                    ),
9247                )
9248                .expect("invalid HELLO returns a refusal");
9249
9250            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9251            let error = parse_error(&response[0]);
9252            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9253            let message = error["message"]
9254                .as_str()
9255                .expect("error message is a string");
9256            assert!(
9257                message.contains(&field),
9258                "{name}: field missing from {message}"
9259            );
9260            assert!(
9261                message.contains(&value),
9262                "{name}: value missing from {message}"
9263            );
9264            assert_eq!(
9265                registry
9266                    .active_registration_count()
9267                    .expect("registry reads"),
9268                0,
9269                "{name}: refused HELLO must not create a catalog entry"
9270            );
9271        }
9272    }
9273
9274    #[test]
9275    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9276        let registry = Arc::new(Registry::default());
9277        let handler = ControlHandler::new(Arc::clone(&registry));
9278        let capabilities = json!({
9279            "provides": ["credentials-provider/v1"],
9280            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9281            "must_never_reach": ["federation-transport/v1"]
9282        });
9283        let response = handler
9284            .handle_control(
9285                ConnectionId::new(99),
9286                capability_grammar_hello_frame(
9287                    capabilities.clone(),
9288                    Some(json!(["/roles/0/tools"])),
9289                    99,
9290                ),
9291            )
9292            .expect("valid HELLO registers");
9293        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9294
9295        let request = Frame::build(
9296            FrameType::Request,
9297            control_flags(),
9298            0,
9299            0,
9300            100,
9301            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9302                .expect("catalog request serializes"),
9303        )
9304        .expect("catalog request frame builds");
9305        let response = handler
9306            .handle_catalog_list(request, None)
9307            .expect("catalog list succeeds");
9308        let ClientControlResponse::CatalogList { modules, .. } =
9309            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9310        else {
9311            panic!("catalog request must return catalog.list");
9312        };
9313        assert_eq!(modules.len(), 1);
9314        assert_eq!(
9315            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9316            capabilities
9317        );
9318    }
9319
9320    #[test]
9321    fn catalog_list_mirrors_management_operation_description() {
9322        let registry = Arc::new(Registry::default());
9323        let handler = ControlHandler::new(Arc::clone(&registry));
9324        let description = "List managed records and return their identifiers and metadata.";
9325        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9326        manifest.provides = vec![ProviderRole::ManagementSurface {
9327            operations: vec![ManagementOperation {
9328                name: "records.list".to_string(),
9329                kind: ManagementOperationKind::Query,
9330                description: Some(description.to_string()),
9331            }],
9332            config_schema: json!({"type": "object"}),
9333            observability: vec![ObservabilitySurface {
9334                name: "records.stats".to_string(),
9335                kind: ObservabilityKind::Snapshot,
9336            }],
9337            identity_scope: vec![IdentityScope::Project],
9338            concurrency: Concurrency::ModuleManaged,
9339        }];
9340        registry
9341            .register_with_control_ops(
9342                manifest,
9343                PROTOCOL_VERSION,
9344                ConnectionId::new(99),
9345                Vec::new(),
9346            )
9347            .expect("described management manifest registers");
9348
9349        let request = Frame::build(
9350            FrameType::Request,
9351            control_flags(),
9352            0,
9353            0,
9354            100,
9355            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9356                .expect("catalog request serializes"),
9357        )
9358        .expect("catalog request frame builds");
9359        let response = handler
9360            .handle_catalog_list(request, None)
9361            .expect("catalog list succeeds");
9362        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9363        assert_eq!(
9364            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9365            "catalog.list must preserve the declared operation description verbatim"
9366        );
9367    }
9368
9369    #[test]
9370    fn catalog_list_mirrors_registered_operation_delivery() {
9371        let registry = Arc::new(Registry::default());
9372        let handler = ControlHandler::new(Arc::clone(&registry));
9373        let delivery =
9374            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9375        let mut body = serde_json::to_value(ModuleHelloBody {
9376            manifest: manifest("watch-management", PROTOCOL_VERSION),
9377            protocol_ver: PROTOCOL_VERSION,
9378            control_ops: None,
9379            launch_nonce: None,
9380        })
9381        .unwrap();
9382        body["manifest"]["provides"] = json!([{
9383            "role": "management_surface",
9384            "operations": [
9385                {"name": "status", "kind": "query"},
9386                {"name": "future", "kind": "query"},
9387                {"name": "reset", "kind": "mutate"}
9388            ],
9389            "config_schema": {}, "observability": [], "identity_scope": []
9390        }]);
9391        body["manifest"]["operation_delivery"] = delivery.clone();
9392        let hello = Frame::build(
9393            FrameType::Hello,
9394            control_flags(),
9395            0,
9396            0,
9397            99,
9398            serde_json::to_vec(&body).unwrap(),
9399        )
9400        .unwrap();
9401        let response = handler
9402            .handle_control(ConnectionId::new(99), hello)
9403            .unwrap();
9404        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9405
9406        let (_, registrations) = registry.list_modules().unwrap();
9407        let registered = &registrations[0].manifest;
9408        assert!(registered.is_watch_operation("status"));
9409        assert!(!registered.is_watch_operation("future"));
9410        assert!(!registered.is_watch_operation("reset"));
9411        assert!(!registered.is_watch_operation("missing"));
9412
9413        let request = Frame::build(
9414            FrameType::Request,
9415            control_flags(),
9416            0,
9417            0,
9418            100,
9419            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9420        )
9421        .unwrap();
9422        let response = handler.handle_catalog_list(request, None).unwrap();
9423        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9424        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9425        let ClientControlResponse::CatalogList { modules, .. } =
9426            serde_json::from_slice(&response[0].body).unwrap()
9427        else {
9428            panic!("expected catalog.list");
9429        };
9430        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9431    }
9432
9433    #[test]
9434    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9435        let registry = Arc::new(Registry::default());
9436        let supervisor = SupervisorHandle::new();
9437        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9438        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9439        let descriptor = json!({
9440            "schema": "ckios.richui.manifest/1",
9441            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9442            "bytes": 4096,
9443            "chunks": 4
9444        });
9445        let connection = ConnectionId::new(201);
9446        let hello = handler
9447            .handle_control(
9448                connection,
9449                hello_frame_with_rich_ui(
9450                    "vault",
9451                    PROTOCOL_VERSION,
9452                    201,
9453                    Some("the-real-nonce"),
9454                    descriptor.clone(),
9455                ),
9456            )
9457            .unwrap();
9458        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9459
9460        let request =
9461            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9462        let provides = registry
9463            .get_module_by_connection(connection)
9464            .unwrap()
9465            .unwrap()
9466            .manifest
9467            .provides;
9468        handler
9469            .handle_catalog_update(connection, request, provides, None, Some(false))
9470            .unwrap();
9471
9472        let request = Frame::build(
9473            FrameType::Request,
9474            control_flags(),
9475            0,
9476            0,
9477            203,
9478            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9479        )
9480        .unwrap();
9481        let response = handler.handle_catalog_list(request, None).unwrap();
9482        let ClientControlResponse::CatalogList { modules, .. } =
9483            serde_json::from_slice(&response[0].body).unwrap()
9484        else {
9485            panic!("expected catalog.list");
9486        };
9487        assert_eq!(
9488            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9489            descriptor
9490        );
9491        assert!(modules[0].nonce_verified);
9492    }
9493
9494    fn hello_with_role_declarations(declarations: Value) -> Frame {
9495        let mut body = serde_json::to_value(ModuleHelloBody {
9496            manifest: manifest("role-test", PROTOCOL_VERSION),
9497            protocol_ver: PROTOCOL_VERSION,
9498            control_ops: None,
9499            launch_nonce: None,
9500        })
9501        .unwrap();
9502        body["manifest"]["role_declarations"] = declarations;
9503        Frame::build(
9504            FrameType::Hello,
9505            control_flags(),
9506            0,
9507            0,
9508            301,
9509            serde_json::to_vec(&body).unwrap(),
9510        )
9511        .unwrap()
9512    }
9513
9514    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9515        let request = Frame::build(
9516            FrameType::Request,
9517            control_flags(),
9518            0,
9519            0,
9520            302,
9521            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9522        )
9523        .unwrap();
9524        let response = handler.handle_catalog_list(request, None).unwrap();
9525        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9526        let ClientControlResponse::CatalogList { mut modules, .. } =
9527            serde_json::from_slice(&response[0].body).unwrap()
9528        else {
9529            panic!("expected catalog.list");
9530        };
9531        (raw["modules"][0].clone(), modules.remove(0))
9532    }
9533
9534    #[test]
9535    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9536        let registry = Arc::new(Registry::default());
9537        let handler = ControlHandler::new(Arc::clone(&registry));
9538        let declarations = json!({
9539            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9540            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9541        });
9542        let connection = ConnectionId::new(301);
9543        let response = handler
9544            .handle_control(
9545                connection,
9546                hello_with_role_declarations(declarations.clone()),
9547            )
9548            .unwrap();
9549        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9550        let registered = registry
9551            .get_module_by_connection(connection)
9552            .unwrap()
9553            .unwrap()
9554            .manifest;
9555        // Compared as JSON values, not bytes: object key order is not part of
9556        // the relay's promise, and workspace builds unify serde_json's
9557        // preserve_order feature, which changes the order a literal keeps.
9558        assert_eq!(
9559            serde_json::to_value(&registered.role_declarations).unwrap(),
9560            declarations
9561        );
9562        let (raw, entry) = role_declarations_catalog(&handler);
9563        assert_eq!(raw["role_declarations"], declarations);
9564        assert_eq!(entry.role_declarations, registered.role_declarations);
9565
9566        let request =
9567            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9568        handler
9569            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9570            .unwrap();
9571        let (raw, entry) = role_declarations_catalog(&handler);
9572        assert_eq!(raw["role_declarations"], declarations);
9573        assert_eq!(entry.role_declarations, registered.role_declarations);
9574    }
9575
9576    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9577        let registry = Arc::new(Registry::default());
9578        let handler = ControlHandler::new(Arc::clone(&registry));
9579        let capture = EventCapture::default();
9580        let _subscriber =
9581            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9582        let response = handler
9583            .handle_control(
9584                ConnectionId::new(301),
9585                hello_with_role_declarations(declarations),
9586            )
9587            .unwrap();
9588        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9589        assert!(registry
9590            .get_module_by_connection(ConnectionId::new(301))
9591            .unwrap()
9592            .unwrap()
9593            .manifest
9594            .role_declarations
9595            .is_empty());
9596        let (raw, entry) = role_declarations_catalog(&handler);
9597        assert!(raw.get("role_declarations").is_none());
9598        assert!(entry.role_declarations.is_empty());
9599        let warnings: Vec<_> = capture
9600            .events()
9601            .into_iter()
9602            .filter(|event| {
9603                event.fields.get("message").map(String::as_str)
9604                    == Some("dropping whole role_declarations map from HELLO")
9605            })
9606            .collect();
9607        assert_eq!(warnings.len(), 1);
9608        assert_eq!(warnings[0].level, tracing::Level::WARN);
9609        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9610        assert_eq!(warnings[0].fields["reason"], reason);
9611    }
9612
9613    #[test]
9614    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9615        for value in [
9616            json!(null),
9617            json!(false),
9618            json!(0),
9619            json!("opaque"),
9620            json!([]),
9621        ] {
9622            assert_hello_drops_role_declarations_with_warning(
9623                json!({"routing/v1": {}, "broken/v1": value}),
9624                "role_declarations values must be objects",
9625            );
9626        }
9627        assert_hello_drops_role_declarations_with_warning(
9628            json!([]),
9629            "role_declarations must be an object",
9630        );
9631    }
9632
9633    #[test]
9634    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9635        assert_hello_drops_role_declarations_with_warning(
9636            json!({"routing/v1": {}, "bad key": {}}),
9637            "role_declarations key must use capability-name grammar",
9638        );
9639    }
9640
9641    #[test]
9642    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9643        assert_hello_drops_role_declarations_with_warning(
9644            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9645            "role_declarations exceeds 16384 compact JSON bytes",
9646        );
9647    }
9648
9649    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
9650        let mut body = serde_json::to_value(ModuleHelloBody {
9651            manifest: manifest("flow-ops", PROTOCOL_VERSION),
9652            protocol_ver: PROTOCOL_VERSION,
9653            control_ops: None,
9654            launch_nonce: None,
9655        })
9656        .unwrap();
9657        body["manifest"]["provides"] = json!([{
9658            "role": "management_surface",
9659            "operations": [
9660                {"name": "diagnostics.findings", "kind": "query"},
9661                {"name": "reset", "kind": "mutate"}
9662            ],
9663            "config_schema": {}, "observability": [], "identity_scope": []
9664        }]);
9665        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
9666        if let Some(flow_operations) = flow_operations {
9667            body["manifest"]["flow_operations"] = flow_operations;
9668        }
9669        Frame::build(
9670            FrameType::Hello,
9671            control_flags(),
9672            0,
9673            0,
9674            401,
9675            serde_json::to_vec(&body).unwrap(),
9676        )
9677        .unwrap()
9678    }
9679
9680    #[test]
9681    fn catalog_list_mirrors_flow_operations_unchanged() {
9682        // Order is kept as declared (not sorted) and an empty list stays an
9683        // affirmative "none", distinct from an absent declaration.
9684        for (capability, flow_operations) in [
9685            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
9686            ("call-modes/v1", json!(["diagnostics.findings"])),
9687            ("flow-scopes/v1", json!([])),
9688        ] {
9689            let registry = Arc::new(Registry::default());
9690            let handler = ControlHandler::new(Arc::clone(&registry));
9691            let connection = ConnectionId::new(401);
9692            let response = handler
9693                .handle_control(
9694                    connection,
9695                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
9696                )
9697                .unwrap();
9698            assert_eq!(response[0].header.ty, FrameType::HelloAck);
9699            let registered = registry
9700                .get_module_by_connection(connection)
9701                .unwrap()
9702                .unwrap()
9703                .manifest;
9704            assert_eq!(
9705                serde_json::to_value(&registered.flow_operations).unwrap(),
9706                flow_operations
9707            );
9708            let (raw, entry) = role_declarations_catalog(&handler);
9709            assert_eq!(raw["flow_operations"], flow_operations);
9710            assert_eq!(entry.flow_operations, registered.flow_operations);
9711        }
9712
9713        let registry = Arc::new(Registry::default());
9714        let handler = ControlHandler::new(Arc::clone(&registry));
9715        let response = handler
9716            .handle_control(
9717                ConnectionId::new(401),
9718                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
9719            )
9720            .unwrap();
9721        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9722        let (raw, entry) = role_declarations_catalog(&handler);
9723        assert!(raw.get("flow_operations").is_none());
9724        assert_eq!(entry.flow_operations, None);
9725    }
9726
9727    fn assert_hello_drops_flow_operations_with_warning(
9728        capabilities: Value,
9729        flow_operations: Value,
9730        reason: &str,
9731    ) {
9732        let registry = Arc::new(Registry::default());
9733        let handler = ControlHandler::new(Arc::clone(&registry));
9734        let capture = EventCapture::default();
9735        let _subscriber =
9736            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9737        let response = handler
9738            .handle_control(
9739                ConnectionId::new(401),
9740                hello_with_flow_operations(capabilities, Some(flow_operations)),
9741            )
9742            .unwrap();
9743        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9744        let registered = registry
9745            .get_module_by_connection(ConnectionId::new(401))
9746            .unwrap()
9747            .expect("the module still registers")
9748            .manifest;
9749        assert_eq!(registered.module_id, "flow-ops");
9750        assert_eq!(registered.flow_operations, None);
9751        let (raw, entry) = role_declarations_catalog(&handler);
9752        assert_eq!(raw["module_id"], "flow-ops");
9753        assert!(raw.get("flow_operations").is_none());
9754        assert_eq!(entry.flow_operations, None);
9755        let warnings: Vec<_> = capture
9756            .events()
9757            .into_iter()
9758            .filter(|event| {
9759                event.fields.get("message").map(String::as_str)
9760                    == Some("dropping flow_operations from HELLO")
9761            })
9762            .collect();
9763        assert_eq!(warnings.len(), 1);
9764        assert_eq!(warnings[0].level, tracing::Level::WARN);
9765        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
9766        assert_eq!(warnings[0].fields["reason"], reason);
9767    }
9768
9769    #[test]
9770    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
9771        assert_hello_drops_flow_operations_with_warning(
9772            json!(["flow-scopes/v1"]),
9773            json!(["diagnostics.findings", "missing"]),
9774            "flow_operations names 'missing', which is not a declared management operation",
9775        );
9776    }
9777
9778    #[test]
9779    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
9780        assert_hello_drops_flow_operations_with_warning(
9781            json!([]),
9782            json!(["diagnostics.findings"]),
9783            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9784        );
9785        // A run-only capability does not admit flows either.
9786        assert_hello_drops_flow_operations_with_warning(
9787            json!(["agent-run-scopes/v1"]),
9788            json!([]),
9789            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9790        );
9791    }
9792
9793    #[test]
9794    fn hello_malformed_flow_operations_is_dropped_with_warning() {
9795        assert_hello_drops_flow_operations_with_warning(
9796            json!(["flow-scopes/v1"]),
9797            json!(["reset", "reset"]),
9798            "flow_operations names 'reset' more than once",
9799        );
9800        assert_hello_drops_flow_operations_with_warning(
9801            json!(["flow-scopes/v1"]),
9802            json!(vec!["reset"; 257]),
9803            "flow_operations lists 257 names, more than 256",
9804        );
9805        assert_hello_drops_flow_operations_with_warning(
9806            json!(["flow-scopes/v1"]),
9807            json!("reset"),
9808            "flow_operations must be an array of operation names",
9809        );
9810    }
9811
9812    #[test]
9813    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9814        let registry = Arc::new(Registry::default());
9815        let handler = ControlHandler::new(registry.clone());
9816        let descriptor = json!({
9817            "schema": "ckios.richui.manifest/1",
9818            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9819            "bytes": 128,
9820            "chunks": 2
9821        });
9822        let malformed = [
9823            ("schema", json!("")),
9824            ("schema", json!("a".repeat(65))),
9825            ("schema", json!("Bad schema")),
9826            (
9827                "manifest_digest",
9828                json!(format!("sha256:{}", "A".repeat(64))),
9829            ),
9830            ("manifest_digest", json!("sha256:abc")),
9831            ("bytes", json!(0)),
9832            ("bytes", json!(16_777_217)),
9833            ("bytes", json!("128")),
9834            ("chunks", json!(0)),
9835            ("chunks", json!(1025)),
9836            ("chunks", json!("2")),
9837        ];
9838        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9839            let mut invalid = descriptor.clone();
9840            invalid[field] = invalid_value;
9841            let response = handler
9842                .handle_control(
9843                    ConnectionId::new(210 + index as u64),
9844                    hello_frame_with_rich_ui(
9845                        &format!("bad-rich-ui-{index}"),
9846                        PROTOCOL_VERSION,
9847                        210 + index as u64,
9848                        None,
9849                        invalid,
9850                    ),
9851                )
9852                .unwrap();
9853            assert_eq!(response[0].header.ty, FrameType::Error);
9854            let error = parse_error(&response[0]);
9855            assert_eq!(error["code"], "invalid_manifest");
9856            assert!(error["message"]
9857                .as_str()
9858                .unwrap()
9859                .contains(&format!("rich_ui.{field}")));
9860            assert!(registry
9861                .get_module(&format!("bad-rich-ui-{index}"))
9862                .unwrap()
9863                .is_none());
9864        }
9865    }
9866
9867    #[test]
9868    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9869        let registry = Arc::new(Registry::default());
9870        let handler = ControlHandler::new(Arc::clone(&registry));
9871        let connection = ConnectionId::new(219);
9872        registry
9873            .register_with_control_ops(
9874                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9875                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9876                        "bad schema",
9877                        format!("sha256:{}", "a".repeat(64)),
9878                        128,
9879                        2,
9880                    )))
9881                    .build(),
9882                PROTOCOL_VERSION,
9883                connection,
9884                Vec::new(),
9885            )
9886            .unwrap();
9887        let request =
9888            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9889        let response = handler
9890            .handle_catalog_update(connection, request, Vec::new(), None, None)
9891            .unwrap();
9892        assert_eq!(response[0].header.ty, FrameType::Error);
9893        let error = parse_error(&response[0]);
9894        assert_eq!(error["code"], "invalid_manifest");
9895        assert!(error["message"]
9896            .as_str()
9897            .unwrap()
9898            .contains("rich_ui.schema"));
9899    }
9900
9901    #[test]
9902    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9903        let registry = Arc::new(Registry::default());
9904        let supervisor = SupervisorHandle::new();
9905        supervisor.open_swap("provider", "candidate-nonce".to_string());
9906        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9907        let connection = ConnectionId::new(218);
9908        let hello = handler
9909            .handle_control(
9910                connection,
9911                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9912            )
9913            .unwrap();
9914        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9915        assert!(registry.list_modules().unwrap().1.is_empty());
9916
9917        registry.promote_candidate("provider").unwrap().unwrap();
9918        let request = Frame::build(
9919            FrameType::Request,
9920            control_flags(),
9921            0,
9922            0,
9923            219,
9924            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9925        )
9926        .unwrap();
9927        let response = handler.handle_catalog_list(request, None).unwrap();
9928        let ClientControlResponse::CatalogList { modules, .. } =
9929            serde_json::from_slice(&response[0].body).unwrap()
9930        else {
9931            panic!("expected catalog.list");
9932        };
9933        assert_eq!(modules.len(), 1);
9934        assert!(modules[0].nonce_verified);
9935    }
9936
9937    #[test]
9938    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9939        let registry = Arc::new(Registry::default());
9940        let handler = ControlHandler::new(Arc::clone(&registry));
9941        let hello = Frame::build(
9942            FrameType::Hello,
9943            control_flags(),
9944            0,
9945            0,
9946            220,
9947            serde_json::to_vec(&ModuleHelloBody {
9948                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9949                protocol_ver: PROTOCOL_VERSION,
9950                control_ops: None,
9951                launch_nonce: Some("unverified".to_string()),
9952            })
9953            .unwrap(),
9954        )
9955        .unwrap();
9956        assert_eq!(
9957            handler
9958                .handle_control(ConnectionId::new(220), hello)
9959                .unwrap()[0]
9960                .header
9961                .ty,
9962            FrameType::HelloAck
9963        );
9964        let request = Frame::build(
9965            FrameType::Request,
9966            control_flags(),
9967            0,
9968            0,
9969            221,
9970            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9971        )
9972        .unwrap();
9973        let response = handler.handle_catalog_list(request, None).unwrap();
9974        let ClientControlResponse::CatalogList { modules, .. } =
9975            serde_json::from_slice(&response[0].body).unwrap()
9976        else {
9977            panic!("expected catalog.list");
9978        };
9979        assert_eq!(
9980            serde_json::to_vec(&modules[0]).unwrap(),
9981            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9982        );
9983        assert!(!modules[0].nonce_verified);
9984    }
9985
9986    #[test]
9987    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9988        let registry = Arc::new(Registry::default());
9989        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9990            [("vault".to_string(), true), ("squatter".to_string(), true)],
9991            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9992        );
9993        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9994        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9995            provides: vec!["credentials-provider/v1".to_string()],
9996            requires: Vec::new(),
9997            must_never_reach: Vec::new(),
9998        });
9999        let frame = Frame::build(
10000            FrameType::Hello,
10001            control_flags(),
10002            0,
10003            0,
10004            77,
10005            serde_json::to_vec(&ModuleHelloBody {
10006                manifest: squatter,
10007                protocol_ver: PROTOCOL_VERSION,
10008                control_ops: None,
10009                launch_nonce: None,
10010            })
10011            .expect("HELLO serializes"),
10012        )
10013        .expect("HELLO frame builds");
10014        let response = handler
10015            .handle_control(ConnectionId::new(77), frame)
10016            .expect("reserved claim receives a typed refusal");
10017        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10018        assert_eq!(
10019            registry
10020                .active_registration_count()
10021                .expect("registry reads"),
10022            0,
10023            "a reserved capability refusal must not leave a catalog entry"
10024        );
10025    }
10026
10027    #[test]
10028    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10029        for settlement in ["timeout", "inconclusive", "drop"] {
10030            let breakers = RouteBindBreakers::default();
10031            let RouteBindAdmission::Admitted {
10032                guard: mut old,
10033                probe: false,
10034            } = breakers.admit("prov")
10035            else {
10036                panic!("ordinary relay admitted")
10037            };
10038            let RouteBindAdmission::Admitted {
10039                guard: mut opener, ..
10040            } = breakers.admit("prov")
10041            else {
10042                panic!("second relay admitted")
10043            };
10044            assert!(
10045                !opener
10046                    .record_timeout(1, Duration::ZERO)
10047                    .unwrap()
10048                    .reopened_after_probe
10049            );
10050            let RouteBindAdmission::Admitted {
10051                guard: mut probe,
10052                probe: true,
10053            } = breakers.admit("prov")
10054            else {
10055                panic!("one cooldown probe admitted")
10056            };
10057            match settlement {
10058                "timeout" => assert!(
10059                    !old.record_timeout(1, Duration::ZERO)
10060                        .unwrap()
10061                        .reopened_after_probe
10062                ),
10063                "inconclusive" => old.record_inconclusive(),
10064                "drop" => drop(old),
10065                _ => unreachable!(),
10066            }
10067            assert!(
10068                matches!(
10069                    breakers.admit("prov"),
10070                    RouteBindAdmission::Refused {
10071                        probe_in_flight: true,
10072                        ..
10073                    }
10074                ),
10075                "{settlement} of a pre-open relay cannot release the real probe"
10076            );
10077            assert!(
10078                probe
10079                    .record_timeout(1, Duration::ZERO)
10080                    .unwrap()
10081                    .reopened_after_probe
10082            );
10083            assert!(matches!(
10084                breakers.admit("prov"),
10085                RouteBindAdmission::Admitted { probe: true, .. }
10086            ));
10087        }
10088        let breakers = RouteBindBreakers::default();
10089        let admit = || match breakers.admit("prov") {
10090            RouteBindAdmission::Admitted { guard, .. } => guard,
10091            _ => panic!("relay admitted"),
10092        };
10093        admit().record_timeout(1, Duration::ZERO);
10094        let mut old_probe = admit();
10095        breakers.reset_for_new_module_connection("prov");
10096        admit().record_timeout(1, Duration::ZERO);
10097        let _new_probe = admit();
10098        old_probe.record_inconclusive();
10099        assert!(matches!(
10100            breakers.admit("prov"),
10101            RouteBindAdmission::Refused {
10102                probe_in_flight: true,
10103                ..
10104            }
10105        ));
10106    }
10107
10108    #[tokio::test]
10109    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10110        for candidate in [false, true] {
10111            let registry = Arc::new(Registry::default());
10112            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10113                [("vault".to_string(), true), ("squatter".to_string(), true)],
10114                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10115            );
10116            let conn = ConnectionId::new(77);
10117            let (ctx, mut rx) = route_ctx(conn);
10118            let initial = capability_manifest("squatter", &[], &[]);
10119            if candidate {
10120                registry
10121                    .register_candidate_with_control_ops(
10122                        initial.clone(),
10123                        PROTOCOL_VERSION,
10124                        conn,
10125                        module_baseline_control_ops(),
10126                    )
10127                    .unwrap();
10128                handler
10129                    .forwarding
10130                    .register_candidate_module_connection(
10131                        conn,
10132                        "squatter".to_string(),
10133                        PROTOCOL_VERSION,
10134                        manifest_concurrency(&initial),
10135                        ctx.egress.clone(),
10136                    )
10137                    .unwrap();
10138            } else {
10139                hello_via_sink(
10140                    &handler,
10141                    &ctx,
10142                    &mut rx,
10143                    hello_frame_with_manifest(initial.clone(), 1),
10144                )
10145                .await;
10146            }
10147            let response = handler
10148                .handle_control_frame(
10149                    &ctx,
10150                    catalog_update_with_capabilities_frame(
10151                        2,
10152                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
10153                            .capabilities
10154                            .unwrap(),
10155                    ),
10156                )
10157                .await
10158                .unwrap();
10159            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10160            assert_eq!(
10161                registry
10162                    .get_module_by_connection(conn)
10163                    .unwrap()
10164                    .unwrap()
10165                    .manifest,
10166                initial
10167            );
10168        }
10169    }
10170
10171    #[test]
10172    fn server_describe_surfaces_required_capability_verdict_fields() {
10173        let registry = Arc::new(Registry::default());
10174        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10175            [
10176                ("consumer".to_string(), true),
10177                ("provider".to_string(), false),
10178            ],
10179            BTreeMap::new(),
10180        );
10181        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10182        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10183            provides: Vec::new(),
10184            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10185                capability: "credentials-provider/v1".to_string(),
10186                need: subc_protocol::manifest::CapabilityNeed::Required,
10187            }],
10188            must_never_reach: Vec::new(),
10189        });
10190        let hello = Frame::build(
10191            FrameType::Hello,
10192            control_flags(),
10193            0,
10194            0,
10195            78,
10196            serde_json::to_vec(&ModuleHelloBody {
10197                manifest: consumer,
10198                protocol_ver: PROTOCOL_VERSION,
10199                control_ops: None,
10200                launch_nonce: None,
10201            })
10202            .expect("HELLO serializes"),
10203        )
10204        .expect("HELLO frame builds");
10205        handler
10206            .handle_control(ConnectionId::new(78), hello)
10207            .expect("consumer registers");
10208        let describe = Frame::build(
10209            FrameType::Request,
10210            control_flags(),
10211            0,
10212            0,
10213            79,
10214            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10215                .expect("request serializes"),
10216        )
10217        .expect("describe frame builds");
10218        let response = handler
10219            .handle_server_describe(describe)
10220            .expect("server.describe succeeds");
10221        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10222        let requirement = &rendered["capability_requirements"][0];
10223        assert_eq!(requirement["consumer"], "consumer");
10224        assert_eq!(requirement["verdict"], "never_provided");
10225        assert_eq!(requirement["episode_seq"], 1);
10226        assert_eq!(requirement["config_satisfiable"], false);
10227        assert_eq!(requirement["runtime_available"], false);
10228        assert!(requirement["detail"]
10229            .as_str()
10230            .expect("detail string")
10231            .contains("credentials-provider/v1"));
10232    }
10233
10234    /// Register a consumer that requires `project-identity/v1` and the provider
10235    /// of it, optionally mark daemon shutdown as begun, then deregister the
10236    /// provider as its connection closes. Returns the `capability.requirement`
10237    /// events that deregistration logged and the verdict the evaluator then
10238    /// holds for the requirement (the one `route.open` refusals read).
10239    fn deregister_required_provider(
10240        daemon_shutdown_begun: bool,
10241    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10242        const CAPABILITY: &str = "project-identity/v1";
10243        let registry = Arc::new(Registry::default());
10244        let daemon_shutdown = DaemonShutdownFlag::default();
10245        let handler = ControlHandler::new(Arc::clone(&registry))
10246            .with_capability_config(
10247                [
10248                    ("consumer".to_string(), true),
10249                    ("provider".to_string(), true),
10250                ],
10251                BTreeMap::new(),
10252            )
10253            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10254        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10255        consumer.capabilities = Some(CapabilityDeclarations {
10256            provides: Vec::new(),
10257            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10258                capability: CAPABILITY.to_string(),
10259                need: subc_protocol::manifest::CapabilityNeed::Required,
10260            }],
10261            must_never_reach: Vec::new(),
10262        });
10263        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10264        let provider_conn = ConnectionId::new(182);
10265        for (conn, manifest, corr) in [
10266            (ConnectionId::new(181), consumer, 181),
10267            (provider_conn, provider, 182),
10268        ] {
10269            let hello = handler
10270                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10271                .expect("module registers");
10272            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10273        }
10274        assert_eq!(
10275            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10276            Some(CapabilityVerdict::Provided),
10277            "the registered provider must satisfy the requirement first"
10278        );
10279        if daemon_shutdown_begun {
10280            daemon_shutdown.set_for_test();
10281        }
10282
10283        let capture = EventCapture::default();
10284        let _subscriber =
10285            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10286        handler
10287            .cleanup_connection(provider_conn)
10288            .expect("provider deregisters");
10289        let events = capture
10290            .events()
10291            .into_iter()
10292            .filter(|event| {
10293                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10294            })
10295            .collect();
10296        (
10297            events,
10298            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10299        )
10300    }
10301
10302    #[test]
10303    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10304        let (events, verdict) = deregister_required_provider(true);
10305        assert_eq!(
10306            verdict,
10307            Some(CapabilityVerdict::NeverProvided),
10308            "shutdown must not change the verdict route.open refusals read"
10309        );
10310        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10311        let event = &events[0];
10312        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10313        assert_eq!(
10314            event.fields.get("cause").map(String::as_str),
10315            Some("\"daemon_shutdown\""),
10316            "{event:?}"
10317        );
10318        assert_eq!(
10319            event.fields.get("verdict").map(String::as_str),
10320            Some("\"never_provided\""),
10321            "{event:?}"
10322        );
10323    }
10324
10325    #[test]
10326    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10327        let (events, verdict) = deregister_required_provider(false);
10328        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10329        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10330        let event = &events[0];
10331        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10332        assert_eq!(event.fields.get("cause"), None, "{event:?}");
10333        assert_eq!(
10334            event.fields.get("verdict").map(String::as_str),
10335            Some("\"never_provided\""),
10336            "{event:?}"
10337        );
10338    }
10339
10340    #[test]
10341    fn catalog_list_omits_capabilities_for_legacy_manifest() {
10342        let registry = Arc::new(Registry::default());
10343        let handler = ControlHandler::new(Arc::clone(&registry));
10344        let hello = handler
10345            .handle_control(
10346                ConnectionId::new(101),
10347                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10348            )
10349            .expect("legacy HELLO registers");
10350        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10351
10352        let request = Frame::build(
10353            FrameType::Request,
10354            control_flags(),
10355            0,
10356            0,
10357            102,
10358            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10359                .expect("catalog request serializes"),
10360        )
10361        .expect("catalog request frame builds");
10362        let response = handler
10363            .handle_catalog_list(request, None)
10364            .expect("catalog list succeeds");
10365        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10366        assert!(
10367            body["modules"][0].get("capabilities").is_none(),
10368            "legacy manifest must retain an absent capabilities field on catalog.list"
10369        );
10370    }
10371
10372    #[test]
10373    fn hello_ack_omits_storage_when_no_storage_config() {
10374        let registry = Arc::new(Registry::default());
10375        let handler = ControlHandler::new(Arc::clone(&registry));
10376        let responses = handler
10377            .handle_control(
10378                ConnectionId::new(1),
10379                hello_frame("aft", PROTOCOL_VERSION, 7),
10380            )
10381            .unwrap();
10382        let ack = parse_ack(&responses[0]);
10383        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10384        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10385    }
10386
10387    #[tokio::test]
10388    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10389        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10390        let registry = Arc::new(Registry::default());
10391        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
10392        let responses = handler
10393            .handle_control(
10394                ConnectionId::new(1),
10395                hello_frame("aft", PROTOCOL_VERSION, 7),
10396            )
10397            .unwrap();
10398        let ack = parse_ack(&responses[0]);
10399        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10400
10401        let described = handler
10402            .handle_control_frame(
10403                &route_ctx(ConnectionId::new(2)).0,
10404                Frame::build(
10405                    FrameType::Request,
10406                    control_flags(),
10407                    0,
10408                    0,
10409                    9,
10410                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10411                )
10412                .unwrap(),
10413            )
10414            .await
10415            .unwrap();
10416        let ClientControlResponse::ServerDescribe { machine_id, .. } =
10417            serde_json::from_slice(&described[0].body).unwrap()
10418        else {
10419            panic!("server.describe answered with another shape");
10420        };
10421        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10422    }
10423
10424    #[test]
10425    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10426        // With a central sqlite storage policy, each registering module gets its
10427        // own resolved descriptor in HELLO_ACK, keyed by its module id.
10428        let registry = Arc::new(Registry::default());
10429        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
10430            crate::daemon_config::StorageConfig::Sqlite {
10431                data_home: std::path::PathBuf::from("/data"),
10432            },
10433        ));
10434
10435        let responses = handler
10436            .handle_control(
10437                ConnectionId::new(1),
10438                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10439            )
10440            .unwrap();
10441        let ack = parse_ack(&responses[0]);
10442        assert_eq!(
10443            ack.storage,
10444            Some(serde_json::json!({
10445                "module_id": "alfonso-routing",
10446                "storage_namespace": "default",
10447                "isolation": { "kind": "module" },
10448                "backend": {
10449                    "backend": "sqlite",
10450                    "path": "/data/cortexkit/alfonso-routing/store.db"
10451                }
10452            })),
10453            "the delivered descriptor is the module's own sqlite store path"
10454        );
10455    }
10456
10457    #[test]
10458    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10459        let registry = Arc::new(Registry::default());
10460        let handler = ControlHandler::new(Arc::clone(&registry));
10461        let conn = ConnectionId::new(1);
10462        let responses = handler
10463            .handle_control(
10464                conn,
10465                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10466            )
10467            .unwrap();
10468        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10469        let registration = registry.get_module("aft").unwrap().unwrap();
10470        assert_eq!(registration.control_ops, module_baseline_control_ops());
10471
10472        let frame =
10473            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10474        assert!(handler
10475            .guard_module_control_op(&frame, "aft", "route.bind")
10476            .unwrap()
10477            .is_none());
10478        let error = handler
10479            .guard_module_control_op(&frame, "aft", "test.synthetic")
10480            .unwrap()
10481            .expect("synthetic ungranted op should be rejected");
10482        assert_eq!(error.header.ty, FrameType::Error);
10483        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10484    }
10485
10486    #[test]
10487    fn hello_control_ops_some_adds_optional_grants() {
10488        let registry = Arc::new(Registry::default());
10489        let handler = ControlHandler::new(Arc::clone(&registry));
10490        handler
10491            .handle_control(
10492                ConnectionId::new(1),
10493                hello_frame_with_control_ops(
10494                    "aft",
10495                    PROTOCOL_VERSION,
10496                    7,
10497                    Some(vec![
10498                        "future.synthetic".to_string(),
10499                        "route.bind".to_string(),
10500                    ]),
10501                ),
10502            )
10503            .unwrap();
10504        let registration = registry.get_module("aft").unwrap().unwrap();
10505        assert_eq!(
10506            registration.control_ops,
10507            vec![
10508                "route.bind".to_string(),
10509                "route.status".to_string(),
10510                "future.synthetic".to_string(),
10511            ]
10512        );
10513        let frame =
10514            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10515        assert!(handler
10516            .guard_module_control_op(&frame, "aft", "future.synthetic")
10517            .unwrap()
10518            .is_none());
10519    }
10520
10521    #[tokio::test]
10522    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10523        let registry = Arc::new(Registry::default());
10524        let forwarding = Arc::new(ForwardingTable::default());
10525        let handler =
10526            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10527        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10528        hello_via_sink(
10529            &handler,
10530            &module_ctx,
10531            &mut module_rx,
10532            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10533        )
10534        .await;
10535
10536        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10537        let responses = handler
10538            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10539            .await
10540            .unwrap();
10541        assert_eq!(responses.len(), 1);
10542        assert_eq!(responses[0].header.ty, FrameType::Error);
10543        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10544        assert!(module_rx.try_recv().is_err());
10545    }
10546
10547    #[tokio::test]
10548    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10549        let registry = Arc::new(Registry::default());
10550        let forwarding = Arc::new(ForwardingTable::default());
10551        let handler =
10552            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10553        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10554        hello_via_sink(
10555            &handler,
10556            &module_ctx,
10557            &mut module_rx,
10558            hello_frame_with_control_ops(
10559                "aft",
10560                PROTOCOL_VERSION,
10561                7,
10562                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10563            ),
10564        )
10565        .await;
10566
10567        let project_root = unique_project_root("demux");
10568        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10569        let route_handler = handler.clone();
10570        let route_task = tokio::spawn(async move {
10571            route_handler
10572                .handle_control_frame(
10573                    &route_client_ctx,
10574                    route_open_frame(100, "aft", project_root),
10575                )
10576                .await
10577                .unwrap()
10578        });
10579        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10580            .await
10581            .unwrap()
10582            .unwrap();
10583        assert!(matches!(
10584            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10585            ModuleControlRequest::RouteBind { .. }
10586        ));
10587
10588        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10589        let health_handler = handler.clone();
10590        let health_task = tokio::spawn(async move {
10591            health_handler
10592                .handle_control_frame(
10593                    &health_client_ctx,
10594                    supervisor_health_probe_frame(101, "aft"),
10595                )
10596                .await
10597                .unwrap()
10598        });
10599        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10600            .await
10601            .unwrap()
10602            .unwrap();
10603        assert_eq!(
10604            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10605            ModuleControlRequest::HealthCheck {}
10606        );
10607
10608        handler
10609            .handle_control_frame(
10610                &module_ctx,
10611                health_response(health_frame.header.corr, HealthStatus::Degraded),
10612            )
10613            .await
10614            .unwrap();
10615        let health_response = health_task.await.unwrap();
10616        assert_eq!(health_response.len(), 1);
10617        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10618            ClientControlResponse::SupervisorHealthProbe {
10619                module_id,
10620                status,
10621                detail,
10622                metrics,
10623            } => {
10624                assert_eq!(module_id, "aft");
10625                assert_eq!(status, HealthStatus::Degraded);
10626                assert_eq!(detail.as_deref(), Some("warming"));
10627                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10628            }
10629            other => panic!("unexpected health response: {other:?}"),
10630        }
10631
10632        handler
10633            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10634            .await
10635            .unwrap();
10636        let route_response = route_task.await.unwrap();
10637        assert!(route_response.is_empty());
10638        let published = route_client_rx.recv().await.unwrap();
10639        assert!(matches!(
10640            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10641            ClientControlResponse::RouteOpen { .. }
10642        ));
10643    }
10644
10645    /// Start one `route.open` on `client_connection` and return its still-running
10646    /// handler task together with the `route.bind` the module received for it.
10647    /// The handler blocks until the module answers, so it has to run as a task
10648    /// while the test drives the module side.
10649    async fn relay_route_open(
10650        handler: &ControlHandler,
10651        client_connection: ConnectionId,
10652        client_egress: &FrameSink,
10653        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10654        corr: u64,
10655        module_id: &str,
10656        project_root_label: &str,
10657    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10658        let ctx = RouteCtx {
10659            connection_id: client_connection,
10660            egress: client_egress.clone(),
10661        };
10662        let handler = handler.clone();
10663        let project_root = unique_project_root(project_root_label);
10664        let module_id = module_id.to_string();
10665        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10666        let task = tokio::spawn(async move {
10667            let _guard = tracing::dispatcher::set_default(&dispatch);
10668            handler
10669                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10670                .await
10671                .unwrap()
10672        });
10673        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10674            .await
10675            .expect("module receives the relayed route.bind")
10676            .expect("module egress is open");
10677        (task, bind.frame)
10678    }
10679
10680    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10681        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10682            ModuleControlRequest::RouteBind {
10683                route_channel,
10684                epoch,
10685                ..
10686            } => (route_channel, epoch),
10687            other => panic!("expected a route.bind request, got {other:?}"),
10688        }
10689    }
10690
10691    fn published_route(frame: &Frame) -> (u16, u32) {
10692        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10693            ClientControlResponse::RouteOpen {
10694                route_channel,
10695                route_epoch,
10696            } => (route_channel, route_epoch),
10697            other => panic!("expected a route.open response, got {other:?}"),
10698        }
10699    }
10700
10701    /// Reproduction of a production outage. A client had `route.open`s in
10702    /// flight to a module and was already marked closing -- its egress had refused a
10703    /// module frame, so the daemon asked its connection to end -- while its sink
10704    /// was still open. When the module acked those binds, the daemon refused to
10705    /// commit a route for a closing client, and that refusal was returned from
10706    /// the MODULE connection's frame handler, where a router error that has no
10707    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
10708    /// the supervisor correctly did not respawn a clean exit, and every seat lost
10709    /// its tools for hours -- one client's teardown took down a connection
10710    /// carrying ~170 other routes.
10711    ///
10712    /// The window is opened here by calling the production path that opens it
10713    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
10714    /// state that matters is "in `closing_connections`, sink still open, relay
10715    /// still pending", and it lasts only from the close request until the
10716    /// connection loop reacts to it; a socket-level test can flood a client into
10717    /// that escalation but cannot pin the module's ack inside the window. Closing
10718    /// the socket instead takes the other path entirely -- connection teardown
10719    /// removes the pending relay under the same lock, so the ack finds nothing.
10720    #[tokio::test]
10721    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10722        let registry = Arc::new(Registry::default());
10723        let forwarding = Arc::new(ForwardingTable::default());
10724        let handler =
10725            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10726
10727        let module_connection = ConnectionId::new(30);
10728        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10729        hello_via_sink(
10730            &handler,
10731            &module_ctx,
10732            &mut module_rx,
10733            hello_frame("aft", PROTOCOL_VERSION, 7),
10734        )
10735        .await;
10736
10737        let dying_client = ConnectionId::new(31);
10738        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10739
10740        // A published route on the dying client. The escalation below only marks
10741        // a connection closing for a route it has already published.
10742        let (first_task, first_bind) = relay_route_open(
10743            &handler,
10744            dying_client,
10745            &dying_ctx.egress,
10746            &mut module_rx,
10747            100,
10748            "aft",
10749            "closing-first",
10750        )
10751        .await;
10752        handler
10753            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10754            .await
10755            .unwrap();
10756        assert!(first_task.await.unwrap().is_empty());
10757        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10758
10759        // A second route.open from the same client, relayed and awaiting its ack.
10760        let (second_task, second_bind) = relay_route_open(
10761            &handler,
10762            dying_client,
10763            &dying_ctx.egress,
10764            &mut module_rx,
10765            101,
10766            "aft",
10767            "closing-second",
10768        )
10769        .await;
10770        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10771
10772        // The window: the client is closing, its sink is still open, and its
10773        // second bind is still pending.
10774        assert!(forwarding
10775            .escalate_client_delivery_failure(
10776                dying_client,
10777                first_channel,
10778                first_epoch,
10779                CloseReason::new(
10780                    "module_to_client_delivery_failed",
10781                    "client egress refused a module frame",
10782                ),
10783                crate::forwarding::UndeliveredFrame {
10784                    module_id: None,
10785                    sink: &dying_ctx.egress,
10786                },
10787            )
10788            .unwrap());
10789        assert!(!dying_ctx.egress.is_closed());
10790
10791        // The frame that used to end the module connection.
10792        let ack = handler
10793            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10794            .await;
10795        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10796        if module_loop_error.is_some() {
10797            // What the server's connection loop does with a router error that has
10798            // no ERROR-frame translation: end the connection, which releases the
10799            // module's registration and every route on it.
10800            handler.cleanup_connection(module_connection).unwrap();
10801        }
10802        // Read the module's next frame before opening the co-tenant's route, so
10803        // the GOODBYE assertion below is about THIS ack and not about later
10804        // traffic. `None` means the module was told nothing.
10805        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10806            .await
10807            .ok()
10808            .flatten();
10809
10810        // 1. The module connection is still registered.
10811        assert!(
10812            registry
10813                .get_module_by_connection(module_connection)
10814                .unwrap()
10815                .is_some(),
10816            "one client's closing connection ended the shared module connection: \
10817             {module_loop_error:?}"
10818        );
10819        // ...and still serving: another client can open and use a route on it.
10820        let cotenant = ConnectionId::new(32);
10821        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10822        let (cotenant_task, cotenant_bind) = relay_route_open(
10823            &handler,
10824            cotenant,
10825            &cotenant_ctx.egress,
10826            &mut module_rx,
10827            102,
10828            "aft",
10829            "closing-cotenant",
10830        )
10831        .await;
10832        handler
10833            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10834            .await
10835            .unwrap();
10836        assert!(cotenant_task.await.unwrap().is_empty());
10837        let (cotenant_channel, cotenant_epoch) =
10838            published_route(&cotenant_rx.recv().await.unwrap());
10839        assert!(matches!(
10840            forwarding
10841                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10842                .unwrap(),
10843            DataRoute::Client(DataRouteState::Bound(_))
10844        ));
10845
10846        // 2. The module was told to drop the binding it created for the route
10847        //    that will never be published.
10848        let goodbye = post_ack_module_frame
10849            .expect("module receives a GOODBYE for the abandoned route channel");
10850        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10851        assert_eq!(goodbye.header.channel, abandoned_channel);
10852        assert_eq!(goodbye.header.epoch, abandoned_epoch);
10853
10854        // 3. The dying client received nothing: no route was ever published to
10855        //    it. Its route.open is answered as unavailable, which the connection
10856        //    loop would write to a socket that is already going away.
10857        assert!(dying_rx.try_recv().is_err());
10858        let second_response = second_task.await.unwrap();
10859        assert_eq!(second_response.len(), 1);
10860        assert_eq!(
10861            parse_error(&second_response[0])["code"],
10862            "target_unavailable"
10863        );
10864    }
10865
10866    /// One `catalog.list` as a client sees it: the generation and the listed
10867    /// module ids with their readiness.
10868    async fn catalog_snapshot(
10869        handler: &ControlHandler,
10870        client_ctx: &RouteCtx,
10871        corr: u64,
10872    ) -> (u64, Vec<(String, bool)>) {
10873        let body =
10874            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10875        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10876        let reply = handler
10877            .handle_control_frame(client_ctx, request)
10878            .await
10879            .unwrap()
10880            .pop()
10881            .unwrap();
10882        let ClientControlResponse::CatalogList {
10883            generation,
10884            modules,
10885            ..
10886        } = serde_json::from_slice(&reply.body).unwrap()
10887        else {
10888            panic!("not a catalog.list reply");
10889        };
10890        (
10891            generation,
10892            modules
10893                .into_iter()
10894                .map(|entry| (entry.module_id, entry.ready))
10895                .collect(),
10896        )
10897    }
10898
10899    /// A handler whose HELLO for module "aft" on connection 40 is parked
10900    /// between reserving its registry entry and installing its forwarding
10901    /// endpoint. Returns the handler, the two-party barrier (wait once to know
10902    /// the HELLO is parked, once more to release it), the HELLO's thread, and
10903    /// the module's context and receiver.
10904    fn hello_held_before_forwarding() -> (
10905        ControlHandler,
10906        Arc<std::sync::Barrier>,
10907        std::thread::JoinHandle<Vec<Frame>>,
10908        RouteCtx,
10909        mpsc::Receiver<crate::router::OutboundFrame>,
10910    ) {
10911        let hold = Arc::new(std::sync::Barrier::new(2));
10912        let handler = ControlHandler::with_forwarding(
10913            Arc::new(Registry::default()),
10914            Arc::new(ForwardingTable::default()),
10915        )
10916        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10917        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10918        let hello = {
10919            let handler = handler.clone();
10920            let module_ctx = module_ctx.clone();
10921            std::thread::spawn(move || {
10922                handler
10923                    .handle_hello(
10924                        module_ctx.connection_id,
10925                        Some(module_ctx.egress.clone()),
10926                        hello_frame("aft", PROTOCOL_VERSION, 7),
10927                    )
10928                    .unwrap()
10929            })
10930        };
10931        (handler, hold, hello, module_ctx, module_rx)
10932    }
10933
10934    /// While a HELLO has reserved its registry entry but its forwarding
10935    /// endpoint is not installed yet, `route.open` to the module can only be
10936    /// refused. `catalog.list` must not list the module in that window: a
10937    /// client that opens a route as soon as it sees a module listed and ready
10938    /// would be refused for a module the daemon had just advertised. The
10939    /// refusal in the window stays the retryable `target_unavailable`, never
10940    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10941    /// Once the HELLO completes, the module is listed and a route to it opens.
10942    #[tokio::test]
10943    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10944        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10945        let client = ConnectionId::new(41);
10946        let (client_ctx, mut client_rx) = route_ctx(client);
10947
10948        hold.wait();
10949        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10950        assert!(handler.registry.get_module("aft").unwrap().is_some());
10951        assert!(!handler
10952            .forwarding
10953            .has_live_module_connection("aft")
10954            .unwrap());
10955        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10956        assert!(
10957            listed.is_empty(),
10958            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10959        );
10960        let refused = handler
10961            .handle_control_frame(
10962                &client_ctx,
10963                route_open_frame(301, "aft", unique_project_root("listed-window")),
10964            )
10965            .await
10966            .unwrap();
10967        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10968        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10969        let duplicate = handler
10970            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10971            .await
10972            .unwrap();
10973        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10974
10975        hold.wait();
10976        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10977        let ack = module_rx.try_recv().unwrap().frame;
10978        assert_eq!(ack.header.ty, FrameType::HelloAck);
10979        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10980        assert_eq!(listed, vec![("aft".to_string(), true)]);
10981
10982        let (open_task, bind) = relay_route_open(
10983            &handler,
10984            client,
10985            &client_ctx.egress,
10986            &mut module_rx,
10987            303,
10988            "aft",
10989            "listed-after",
10990        )
10991        .await;
10992        assert!(handler
10993            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10994            .await
10995            .unwrap()
10996            .is_empty());
10997        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10998        published_route(&client_rx.try_recv().unwrap().frame);
10999    }
11000
11001    /// A catalog watcher keys on the generation: when it changes, it re-reads
11002    /// the listing. So the generation must advance exactly when the module
11003    /// appears. If it advanced while the module was still hidden, a watcher
11004    /// would read the new generation without the module, and the module would
11005    /// later appear under that same generation with nothing telling the
11006    /// watcher to look again.
11007    #[tokio::test]
11008    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11009        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11010        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11011        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11012
11013        hold.wait();
11014        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11015        assert!(listed.is_empty());
11016        assert_eq!(
11017            during, before,
11018            "the generation moved while the module was not listed"
11019        );
11020
11021        hold.wait();
11022        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11023        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11024        assert_eq!(listed, vec![("aft".to_string(), true)]);
11025        assert_eq!(
11026            after,
11027            before + 1,
11028            "the module appeared without exactly one generation bump"
11029        );
11030    }
11031
11032    /// A HELLO whose forwarding endpoint cannot be installed (here the
11033    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
11034    /// refused and leaves nothing behind: no registry entry, no catalog entry,
11035    /// and the catalog generation where it was.
11036    #[tokio::test]
11037    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11038        let registry = Arc::new(Registry::default());
11039        let handler = ControlHandler::with_forwarding(
11040            Arc::clone(&registry),
11041            Arc::new(ForwardingTable::default()),
11042        );
11043        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11044        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11045        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11046        drop(module_rx);
11047
11048        let replies = handler
11049            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11050            .await
11051            .unwrap();
11052        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11053        assert!(registry.get_module("aft").unwrap().is_none());
11054        assert!(registry
11055            .get_module_by_connection(module_ctx.connection_id)
11056            .unwrap()
11057            .is_none());
11058        assert_eq!(
11059            catalog_snapshot(&handler, &client_ctx, 301).await,
11060            (before, Vec::new())
11061        );
11062    }
11063
11064    /// A handler whose `route.open` parks after queueing the route.bind and
11065    /// before marking it enqueued, with module "aft" registered on connection
11066    /// 40. Returns the two-party barrier that parks and releases it.
11067    async fn route_open_held_after_bind_send() -> (
11068        ControlHandler,
11069        Arc<tokio::sync::Barrier>,
11070        RouteCtx,
11071        mpsc::Receiver<crate::router::OutboundFrame>,
11072    ) {
11073        let hold = Arc::new(tokio::sync::Barrier::new(2));
11074        let handler = ControlHandler::with_forwarding(
11075            Arc::new(Registry::default()),
11076            Arc::new(ForwardingTable::default()),
11077        )
11078        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11079        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11080        hello_via_sink(
11081            &handler,
11082            &module_ctx,
11083            &mut module_rx,
11084            hello_frame("aft", PROTOCOL_VERSION, 7),
11085        )
11086        .await;
11087        (handler, hold, module_ctx, module_rx)
11088    }
11089
11090    /// A fast module acks the route.bind, and the reader commits the route,
11091    /// before `route.open` has marked the relay enqueued. The daemon used to
11092    /// read the missing relay as an abandoned bind and send the module a
11093    /// GOODBYE for the route it had just committed: the client was told its
11094    /// route was open, the module unbound the channel, and the client's first
11095    /// request was dropped on the module side as unbound.
11096    #[tokio::test]
11097    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11098        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11099        let client = ConnectionId::new(41);
11100        let (client_ctx, mut client_rx) = route_ctx(client);
11101
11102        let (open_task, bind) = relay_route_open(
11103            &handler,
11104            client,
11105            &client_ctx.egress,
11106            &mut module_rx,
11107            200,
11108            "aft",
11109            "acked-before-mark",
11110        )
11111        .await;
11112        let (module_channel, module_epoch) = route_bind_channel(&bind);
11113        // route.open is now parked between queueing the bind and marking it.
11114        hold.wait().await;
11115
11116        // The module answers inside that window, and the route commits.
11117        assert!(handler
11118            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11119            .await
11120            .unwrap()
11121            .is_empty());
11122        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11123
11124        hold.wait().await;
11125        assert!(
11126            open_task.await.unwrap().is_empty(),
11127            "route.open answers accepted"
11128        );
11129
11130        // Every frame route.open could have sent is already queued, because it
11131        // has returned; the module must have been sent nothing.
11132        match module_rx.try_recv() {
11133            Err(mpsc::error::TryRecvError::Empty) => {}
11134            Ok(outbound) => panic!(
11135                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11136                outbound.frame.header.ty,
11137                outbound.frame.header.channel,
11138                outbound.frame.header.epoch
11139            ),
11140            Err(err) => panic!("module egress closed: {err}"),
11141        }
11142
11143        // The client's first request on the route is the next thing the module
11144        // reads, on the binding it just created.
11145        let router = Router::with_control_handler(Arc::new(handler.clone()));
11146        let request = Frame::build(
11147            FrameType::Request,
11148            Flags::new(true, Priority::Interactive, false),
11149            client_channel,
11150            client_epoch,
11151            201,
11152            b"first request".to_vec(),
11153        )
11154        .unwrap();
11155        router
11156            .route_for_connection(&client_ctx, request)
11157            .await
11158            .unwrap();
11159        let delivered = module_rx.try_recv().unwrap().frame;
11160        assert_eq!(delivered.header.ty, FrameType::Request);
11161        assert_eq!(
11162            (delivered.header.channel, delivered.header.epoch),
11163            (module_channel, module_epoch)
11164        );
11165        assert_eq!(delivered.body, b"first request");
11166    }
11167
11168    /// The client goes away while `route.open` is still queueing the bind, so
11169    /// the relay is dropped with no module reply. The teardown cannot send the
11170    /// GOODBYE itself (it could reach the module ahead of the bind), so
11171    /// `route.open` must send it once the bind is queued; otherwise the module
11172    /// keeps a binding the daemon will never commit.
11173    #[tokio::test]
11174    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11175        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11176        let client = ConnectionId::new(42);
11177        let (client_ctx, _client_rx) = route_ctx(client);
11178
11179        let (open_task, bind) = relay_route_open(
11180            &handler,
11181            client,
11182            &client_ctx.egress,
11183            &mut module_rx,
11184            300,
11185            "aft",
11186            "released-before-mark",
11187        )
11188        .await;
11189        let (module_channel, module_epoch) = route_bind_channel(&bind);
11190        hold.wait().await;
11191
11192        handler.cleanup_connection(client).unwrap();
11193        assert!(
11194            module_rx.try_recv().is_err(),
11195            "client teardown must not GOODBYE a bind route.open is still queueing"
11196        );
11197
11198        hold.wait().await;
11199        let response = open_task.await.unwrap();
11200        assert_eq!(response.len(), 1);
11201        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11202
11203        let goodbye = module_rx
11204            .try_recv()
11205            .expect("module is told to drop the binding the daemon abandoned")
11206            .frame;
11207        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11208        assert_eq!(
11209            (goodbye.header.channel, goodbye.header.epoch),
11210            (module_channel, module_epoch)
11211        );
11212        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11213    }
11214
11215    /// The module acks inside the same window, but the client's egress closed
11216    /// first, so the reader refuses to commit. The ack proves the module holds
11217    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
11218    /// send a second one.
11219    #[tokio::test]
11220    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11221        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11222        let client = ConnectionId::new(43);
11223        let (client_ctx, client_rx) = route_ctx(client);
11224
11225        let (open_task, bind) = relay_route_open(
11226            &handler,
11227            client,
11228            &client_ctx.egress,
11229            &mut module_rx,
11230            400,
11231            "aft",
11232            "gone-client-before-mark",
11233        )
11234        .await;
11235        let (module_channel, module_epoch) = route_bind_channel(&bind);
11236        hold.wait().await;
11237
11238        drop(client_rx);
11239        assert!(client_ctx.egress.is_closed());
11240        handler
11241            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11242            .await
11243            .unwrap();
11244
11245        hold.wait().await;
11246        let response = open_task.await.unwrap();
11247        assert_eq!(response.len(), 1);
11248        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11249
11250        let goodbye = module_rx
11251            .try_recv()
11252            .expect("module is told to drop the binding it acked")
11253            .frame;
11254        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11255        assert_eq!(
11256            (goodbye.header.channel, goodbye.header.epoch),
11257            (module_channel, module_epoch)
11258        );
11259        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11260        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11261    }
11262
11263    /// The fence at the module-loop boundary, stated as its own contract: which
11264    /// forwarding failures are allowed to end the module connection that is being
11265    /// served. A `ConnectionClosing` naming some client is about that client, and
11266    /// a module connection is shared; the same error naming the module's own
11267    /// connection is about this connection and must stay fatal, as must failures
11268    /// that are about the forwarding table itself.
11269    #[test]
11270    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11271        let handler = ControlHandler::default();
11272        let module_connection = ConnectionId::new(30);
11273        let client_connection = ConnectionId::new(31);
11274
11275        handler
11276            .refuse_to_end_module_connection_for_a_client(
11277                module_connection,
11278                77,
11279                ForwardingError::ConnectionClosing {
11280                    connection_id: client_connection,
11281                },
11282            )
11283            .expect("a closing client must never end the module connection");
11284
11285        assert!(matches!(
11286            handler.refuse_to_end_module_connection_for_a_client(
11287                module_connection,
11288                78,
11289                ForwardingError::ConnectionClosing {
11290                    connection_id: module_connection,
11291                },
11292            ),
11293            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11294                connection_id
11295            })) if connection_id == module_connection
11296        ));
11297        assert!(matches!(
11298            handler.refuse_to_end_module_connection_for_a_client(
11299                module_connection,
11300                79,
11301                ForwardingError::Poisoned,
11302            ),
11303            Err(RouterError::Forwarding(ForwardingError::Poisoned))
11304        ));
11305        assert!(matches!(
11306            handler.refuse_to_end_module_connection_for_a_client(
11307                module_connection,
11308                80,
11309                ForwardingError::StaleModuleEndpoint,
11310            ),
11311            Err(RouterError::Forwarding(
11312                ForwardingError::StaleModuleEndpoint
11313            ))
11314        ));
11315    }
11316
11317    /// The spawn-attestation guard is what stops a connected module from claiming
11318    /// another module's identity and being stamped `Reserved` for it. Every other
11319    /// test that supplies a consumer_identity supplies a CORRECT one, because a
11320    /// correct one is what the rest of the flow needs -- so the guard's rejection
11321    /// branch was never the subject of an assertion, only its acceptance branch.
11322    ///
11323    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
11324    /// whole subc-core library suite green; only the forwarding integration tests
11325    /// notice, and they notice for unrelated reasons. This test exists so the
11326    /// refusal itself is asserted where the guard lives: it fails if the identity
11327    /// check stops refusing, which is the direction that matters, since a guard
11328    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
11329    #[tokio::test]
11330    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11331        let registry = Arc::new(Registry::default());
11332        let forwarding = Arc::new(ForwardingTable::default());
11333        let supervisor = SupervisorHandle::new();
11334        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11335        let handler =
11336            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11337                .with_supervisor(supervisor);
11338
11339        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11340        hello_via_sink(
11341            &handler,
11342            &target_ctx,
11343            &mut target_rx,
11344            hello_frame("target", PROTOCOL_VERSION, 1),
11345        )
11346        .await;
11347
11348        // A real supervised module id presenting the wrong nonce. This is the
11349        // impersonation case: the attacker knows a privileged module_id, which is
11350        // public, and guesses at the nonce, which is not.
11351        let wrong_nonce = handler
11352            .handle_control_frame(
11353                &route_ctx(ConnectionId::new(91)).0,
11354                route_open_frame_with_admission_facts(
11355                    20,
11356                    "target",
11357                    unique_project_root("admission-facts"),
11358                    Some(subc_control::ConsumerIdentity {
11359                        module_id: "fed".to_string(),
11360                        launch_nonce: "not-the-real-nonce".to_string(),
11361                    }),
11362                    None,
11363                ),
11364            )
11365            .await
11366            .unwrap();
11367        assert_eq!(
11368            parse_error(&wrong_nonce[0])["code"],
11369            "bad_consumer_identity",
11370            "a mismatched launch nonce must be refused, not stamped Reserved"
11371        );
11372
11373        // A module id the supervisor never spawned at all, so no nonce exists to
11374        // compare against. An implementation that treats "no record" as "nothing
11375        // to check" fails open here while passing the case above.
11376        let never_spawned = handler
11377            .handle_control_frame(
11378                &route_ctx(ConnectionId::new(92)).0,
11379                route_open_frame_with_admission_facts(
11380                    21,
11381                    "target",
11382                    unique_project_root("admission-facts"),
11383                    Some(subc_control::ConsumerIdentity {
11384                        module_id: "never-spawned".to_string(),
11385                        launch_nonce: "any-nonce".to_string(),
11386                    }),
11387                    None,
11388                ),
11389            )
11390            .await
11391            .unwrap();
11392        assert_eq!(
11393            parse_error(&never_spawned[0])["code"],
11394            "bad_consumer_identity",
11395            "an unspawned module_id must be refused rather than accepted for lack of a record"
11396        );
11397    }
11398
11399    /// The refusal test above proves the guard says NO. Nothing proved it can say
11400    /// YES, and the difference is not academic: replacing the whole authorization
11401    /// with `false` -- admitting no consumer identity at all, revoking Reserved
11402    /// standing for every supervised module in the fleet -- leaves 110 of the 111
11403    /// library tests GREEN. The one that notices does so by HANGING, because it
11404    /// waits for a bind that can no longer happen.
11405    ///
11406    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
11407    /// or flaky test, invites a RETRY rather than an investigation, and the retry
11408    /// hangs too and gets blamed on the runner. So a total revocation of the
11409    /// daemon's trust grant would have shipped behind a symptom nobody attributes
11410    /// to code.
11411    ///
11412    /// The bias is structural rather than accidental. A REFUSAL looks like a
11413    /// failure someone writes a test for; a GRANT looks like the happy path. Every
11414    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
11415    /// suite for that reason, and this one is the purest case in the daemon.
11416    ///
11417    /// This test asserts the EFFECT rather than the absence of an error: the module
11418    /// receives a RouteBind and it carries `Reserved` naming the attested module.
11419    /// A guard that admitted nobody would produce no bind at all; one that admitted
11420    /// everybody would stamp the wrong principal, which the refusal test catches.
11421    #[tokio::test]
11422    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11423        let registry = Arc::new(Registry::default());
11424        let forwarding = Arc::new(ForwardingTable::default());
11425        let supervisor = SupervisorHandle::new();
11426        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11427        let handler =
11428            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11429                .with_supervisor(supervisor);
11430
11431        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11432        hello_via_sink(
11433            &handler,
11434            &target_ctx,
11435            &mut target_rx,
11436            hello_frame("target", PROTOCOL_VERSION, 1),
11437        )
11438        .await;
11439
11440        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11441        let route_handler = handler.clone();
11442        let route_task = tokio::spawn(async move {
11443            route_handler
11444                .handle_control_frame(
11445                    &client_ctx,
11446                    route_open_frame_with_admission_facts(
11447                        30,
11448                        "target",
11449                        unique_project_root("admission-facts"),
11450                        Some(subc_control::ConsumerIdentity {
11451                            module_id: "fed".to_string(),
11452                            launch_nonce: "fed-nonce".to_string(),
11453                        }),
11454                        None,
11455                    ),
11456                )
11457                .await
11458                .unwrap()
11459        });
11460
11461        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
11462        // the very mutation it exists to catch -- a guard that admits nobody -- no
11463        // bind is ever sent, so it HUNG rather than failing. That reproduces the
11464        // exact defect being fixed: a total revocation detected only as a stalled
11465        // suite, which reads as flakiness and invites a retry. An acceptance test
11466        // that waits for an effect must bound the wait, or a red becomes a hang.
11467        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11468            .await
11469            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11470            .expect("module control channel closed before route.bind");
11471        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11472        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11473            panic!("expected route.bind")
11474        };
11475        assert_eq!(
11476            principal,
11477            Some(Principal::Reserved {
11478                module_id: "fed".to_string()
11479            }),
11480            "a correctly attested consumer must be stamped Reserved for its own id"
11481        );
11482
11483        handler
11484            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11485            .await
11486            .unwrap();
11487        assert!(route_task.await.unwrap().is_empty());
11488        assert!(
11489            matches!(
11490                serde_json::from_slice::<ClientControlResponse>(
11491                    &client_rx.recv().await.unwrap().body
11492                )
11493                .unwrap(),
11494                ClientControlResponse::RouteOpen { .. }
11495            ),
11496            "the route must actually open, not merely avoid an error"
11497        );
11498    }
11499
11500    #[tokio::test(start_paused = true)]
11501    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11502        let registry = Arc::new(Registry::default());
11503        let forwarding = Arc::new(ForwardingTable::default());
11504        let supervisor = SupervisorHandle::new();
11505        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11506        let handler =
11507            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11508                .with_supervisor(supervisor);
11509
11510        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11511        hello_via_sink(
11512            &handler,
11513            &target_ctx,
11514            &mut target_rx,
11515            hello_frame("target", PROTOCOL_VERSION, 1),
11516        )
11517        .await;
11518
11519        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11520        let direct_handler = handler.clone();
11521        let direct_open = tokio::spawn(async move {
11522            direct_handler
11523                .handle_control_frame(
11524                    &direct_ctx,
11525                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
11526                )
11527                .await
11528                .unwrap()
11529        });
11530        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11531            .await
11532            .expect("no direct route.bind within 5s")
11533            .expect("target control channel closed before direct route.bind");
11534        handler
11535            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11536            .await
11537            .unwrap();
11538        assert!(direct_open.await.unwrap().is_empty());
11539        let _ = direct_rx.recv().await.unwrap();
11540
11541        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11542        let reserved_handler = handler.clone();
11543        let reserved_open = tokio::spawn(async move {
11544            reserved_handler
11545                .handle_control_frame(
11546                    &reserved_ctx,
11547                    route_open_frame_with_admission_facts(
11548                        3,
11549                        "target",
11550                        unique_project_root("admission-facts"),
11551                        Some(ConsumerIdentity {
11552                            module_id: "fed".to_string(),
11553                            launch_nonce: "fed-nonce".to_string(),
11554                        }),
11555                        None,
11556                    ),
11557                )
11558                .await
11559                .unwrap()
11560        });
11561        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11562            .await
11563            .expect("no reserved route.bind within 5s")
11564            .expect("target control channel closed before reserved route.bind");
11565        handler
11566            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11567            .await
11568            .unwrap();
11569        assert!(reserved_open.await.unwrap().is_empty());
11570        let _ = reserved_rx.recv().await.unwrap();
11571
11572        forwarding
11573            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11574            .unwrap();
11575        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11576        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11577            module_id: Some("target".to_string()),
11578        })
11579        .unwrap();
11580        let census_frame =
11581            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11582        let response = handler
11583            .handle_control_frame(&census_ctx, census_frame)
11584            .await
11585            .unwrap()
11586            .pop()
11587            .unwrap();
11588        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11589        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11590        assert!(matches!(
11591            decoded,
11592            ClientControlResponse::SupervisorRoutes { .. }
11593        ));
11594        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11595        assert_eq!(routes.len(), 2);
11596        assert!(routes.iter().all(|route| route["draining"] == true));
11597        // The census carries WHY: the reason the drain was begun with, in the
11598        // route.closing vocabulary, on every draining route this drain marked.
11599        assert!(
11600            routes.iter().all(|route| route["drain_reason"] == "reload"),
11601            "draining routes must name the drain's reason: {routes:?}"
11602        );
11603        assert!(routes.iter().any(|route| {
11604            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11605        }));
11606        assert!(routes.iter().any(|route| {
11607            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11608        }));
11609
11610        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11611            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11612        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11613            std::fs::write(
11614                &golden_path,
11615                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11616            )
11617            .unwrap();
11618        }
11619        let expected: Value =
11620            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11621        assert_eq!(actual, expected);
11622    }
11623
11624    async fn query_live_roots(
11625        handler: &ControlHandler,
11626        module_ctx: &RouteCtx,
11627    ) -> ModuleControlResponseToModule {
11628        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11629        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11630        let response = handler
11631            .handle_control_frame(module_ctx, frame)
11632            .await
11633            .unwrap()
11634            .pop()
11635            .unwrap();
11636        serde_json::from_slice(&response.body).unwrap()
11637    }
11638
11639    #[tokio::test(start_paused = true)]
11640    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11641        let registry = Arc::new(Registry::default());
11642        let forwarding = Arc::new(ForwardingTable::default());
11643        let handler = ControlHandler::with_forwarding(registry, forwarding);
11644        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11645        hello_via_sink(
11646            &handler,
11647            &target_ctx,
11648            &mut target_rx,
11649            hello_frame("target", PROTOCOL_VERSION, 1),
11650        )
11651        .await;
11652        let root = unique_project_root("live-roots-known");
11653        let path = ProjectRootId::from_path_allowing_missing(root.path())
11654            .unwrap()
11655            .as_path()
11656            .to_path_buf();
11657        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11658        let open_handler = handler.clone();
11659        let opened = tokio::spawn(async move {
11660            open_handler
11661                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11662                .await
11663                .unwrap()
11664        });
11665        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11666            .await
11667            .unwrap()
11668            .unwrap();
11669        handler
11670            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11671            .await
11672            .unwrap();
11673        assert!(opened.await.unwrap().is_empty());
11674        let _ = client_rx.recv().await.unwrap();
11675
11676        let root = unique_project_root("live-roots-pending");
11677        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11678            .unwrap()
11679            .as_path()
11680            .to_path_buf();
11681        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11682        let open_handler = handler.clone();
11683        let pending = tokio::spawn(async move {
11684            open_handler
11685                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11686                .await
11687                .unwrap()
11688        });
11689        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11690            .await
11691            .unwrap()
11692            .unwrap();
11693        let actual = query_live_roots(&handler, &target_ctx).await;
11694        let ModuleControlResponseToModule::LiveRoots {
11695            roots,
11696            unknown_root_bindings,
11697            total_bindings,
11698        } = actual
11699        else {
11700            panic!("expected live roots")
11701        };
11702        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11703        assert_eq!(unknown_root_bindings, 0);
11704        assert_eq!(
11705            roots.len(),
11706            2,
11707            "root-known arm must retain each canonical root"
11708        );
11709        assert_eq!(
11710            total_bindings,
11711            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11712        );
11713        let counts = roots
11714            .iter()
11715            .map(|root| (root.project_root.clone(), root.bound, root.pending))
11716            .collect::<Vec<_>>();
11717        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11718        expected.sort_by(|a, b| a.0.cmp(&b.0));
11719        assert_eq!(
11720            counts, expected,
11721            "roots must sort by path and count pending separately"
11722        );
11723        handler
11724            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11725            .await
11726            .unwrap();
11727        assert!(pending.await.unwrap().is_empty());
11728    }
11729
11730    #[tokio::test(start_paused = true)]
11731    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11732        let forwarding = Arc::new(ForwardingTable::default());
11733        let handler =
11734            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11735        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11736        hello_via_sink(
11737            &handler,
11738            &target_ctx,
11739            &mut target_rx,
11740            hello_frame("target", PROTOCOL_VERSION, 1),
11741        )
11742        .await;
11743        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11744        let pending = forwarding
11745            .begin_route_bind_relay_for_test(
11746                client_ctx.connection_id,
11747                client_ctx.egress.clone(),
11748                2,
11749                "target",
11750            )
11751            .unwrap();
11752        forwarding
11753            .complete_pending_relay(
11754                target_ctx.connection_id,
11755                pending.corr,
11756                RouteBindRelayOutcome::Accepted,
11757            )
11758            .unwrap();
11759        let actual = query_live_roots(&handler, &target_ctx).await;
11760        let ModuleControlResponseToModule::LiveRoots {
11761            roots,
11762            unknown_root_bindings,
11763            total_bindings,
11764        } = actual
11765        else {
11766            panic!("expected live roots")
11767        };
11768        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11769        assert_eq!(
11770            unknown_root_bindings, 1,
11771            "unknown-root arm must not read as no bindings"
11772        );
11773        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11774        assert_eq!(
11775            total_bindings,
11776            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11777        );
11778    }
11779
11780    /// A module reads HELLO_ACK as its first frame and exits on anything else,
11781    /// so the ack has to be on its outbound queue before the module is
11782    /// routable. The connection loop writes a handler's replies only after the
11783    /// handler returns; this test stops in exactly that gap, runs a real
11784    /// route.open from another connection, and only then writes whatever the
11785    /// HELLO handler returned, the way the loop would. If the ack were still a
11786    /// reply, the route.bind request would reach the module first.
11787    #[tokio::test(start_paused = true)]
11788    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11789        let forwarding = Arc::new(ForwardingTable::default());
11790        let handler =
11791            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11792        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11793        let replies = handler
11794            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11795            .await
11796            .unwrap();
11797        let queued_by_hello = module_rx.len();
11798
11799        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11800        let open_handler = handler.clone();
11801        let open = tokio::spawn(async move {
11802            open_handler
11803                .handle_control_frame(
11804                    &client_ctx,
11805                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11806                )
11807                .await
11808                .unwrap()
11809        });
11810        // Let the route.open run until its route.bind is on the module's queue.
11811        let mut spins = 0;
11812        while module_rx.len() == queued_by_hello {
11813            spins += 1;
11814            assert!(spins < 10_000, "route.open never queued a route.bind");
11815            tokio::task::yield_now().await;
11816        }
11817
11818        // Now the connection loop's half: write the HELLO handler's replies.
11819        for reply in replies {
11820            module_ctx.egress.send(reply).await.unwrap();
11821        }
11822
11823        let first = module_rx.recv().await.unwrap().frame;
11824        assert_eq!(
11825            first.header.ty,
11826            FrameType::HelloAck,
11827            "the first frame a registering module reads must be its HELLO_ACK"
11828        );
11829        assert_eq!(first.header.corr, 7);
11830        let second = module_rx.recv().await.unwrap().frame;
11831        assert_eq!(second.header.ty, FrameType::Request);
11832        assert!(
11833            matches!(
11834                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11835                ModuleControlRequest::RouteBind { .. }
11836            ),
11837            "the route.bind follows the ack"
11838        );
11839        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11840
11841        handler
11842            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11843            .await
11844            .unwrap();
11845        assert!(open.await.unwrap().is_empty());
11846        let _ = client_rx.recv().await.unwrap();
11847    }
11848
11849    #[tokio::test(start_paused = true)]
11850    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11851        let handler = ControlHandler::with_forwarding(
11852            Arc::new(Registry::default()),
11853            Arc::new(ForwardingTable::default()),
11854        );
11855        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11856        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11857        hello_via_sink(
11858            &handler,
11859            &first_ctx,
11860            &mut first_rx,
11861            hello_frame("first", PROTOCOL_VERSION, 1),
11862        )
11863        .await;
11864        hello_via_sink(
11865            &handler,
11866            &second_ctx,
11867            &mut second_rx,
11868            hello_frame("second", PROTOCOL_VERSION, 2),
11869        )
11870        .await;
11871        let root = unique_project_root("second-only");
11872        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11873        let cloned = handler.clone();
11874        let open = tokio::spawn(async move {
11875            cloned
11876                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11877                .await
11878                .unwrap()
11879        });
11880        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11881            .await
11882            .unwrap()
11883            .unwrap();
11884        let first = query_live_roots(&handler, &first_ctx).await;
11885        let second = query_live_roots(&handler, &second_ctx).await;
11886        assert!(
11887            matches!(
11888                first,
11889                ModuleControlResponseToModule::LiveRoots {
11890                    total_bindings: 0,
11891                    ..
11892                }
11893            ),
11894            "cross-module scope must not expose another module's roots"
11895        );
11896        assert!(
11897            matches!(
11898                second,
11899                ModuleControlResponseToModule::LiveRoots {
11900                    total_bindings: 1,
11901                    ..
11902                }
11903            ),
11904            "second module must see its pending route"
11905        );
11906        handler
11907            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11908            .await
11909            .unwrap();
11910        assert!(open.await.unwrap().is_empty());
11911    }
11912
11913    #[tokio::test(start_paused = true)]
11914    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11915        let handler = ControlHandler::with_forwarding(
11916            Arc::new(Registry::default()),
11917            Arc::new(ForwardingTable::default()),
11918        );
11919        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11920        hello_via_sink(
11921            &handler,
11922            &target_ctx,
11923            &mut target_rx,
11924            hello_frame("target", PROTOCOL_VERSION, 1),
11925        )
11926        .await;
11927        let actual = query_live_roots(&handler, &target_ctx).await;
11928        let ModuleControlResponseToModule::LiveRoots {
11929            roots,
11930            unknown_root_bindings,
11931            total_bindings,
11932        } = actual
11933        else {
11934            panic!("expected live roots")
11935        };
11936        assert!(roots.is_empty());
11937        assert_eq!(unknown_root_bindings, 0);
11938        assert_eq!(total_bindings, 0);
11939        assert_eq!(
11940            total_bindings,
11941            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11942        );
11943    }
11944
11945    /// Read the vendored fed corpus rather than hand-building a package.
11946    ///
11947    /// A hand-built object encodes what the test author believed the carrier
11948    /// emits. These vectors are what it actually emits, and one of them exists
11949    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11950    /// ignores additive unknown fields at the traversal emit terminus."
11951    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11952        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11953            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11954        let text = std::fs::read_to_string(&path)
11955            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11956        let vectors: Vec<(String, Value)> = text
11957            .lines()
11958            .filter(|line| !line.trim().is_empty())
11959            .map(|line| {
11960                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11961                let id = entry["corpus_id"]
11962                    .as_str()
11963                    .expect("every vector carries a corpus_id")
11964                    .to_string();
11965                (id, entry["package"].clone())
11966            })
11967            .collect();
11968        // Pin the count: a corpus that silently shrinks would take its coverage
11969        // with it, and a suite reading N-1 vectors reports the same clean pass
11970        // as one reading N.
11971        assert_eq!(
11972            vectors.len(),
11973            3,
11974            "vendored fed corpus changed size; re-sync from subc-federation"
11975        );
11976
11977        // Pin what makes the corpus DISCRIMINATING, not just present.
11978        //
11979        // The relay test below takes its expected value from the corpus, so the
11980        // corpus supplies the test's power to detect a lossy relay rather than
11981        // its correctness. A relay that dropped unrecognised fields would still
11982        // be caught -- but only by a package carrying fields it does not know.
11983        // Shrink every package to the handful of keys any implementation would
11984        // recognise and the test keeps passing over an input that can no longer
11985        // fail, which is the same clean green as a corpus that shrank away.
11986        //
11987        // So assert the precondition rather than duplicating the packages here:
11988        // at least one vector must carry a field beyond the small common set.
11989        // That is one claim to maintain instead of nine, and it fails loudly if
11990        // a re-sync ever flattens the corpus.
11991        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11992        let richest = vectors
11993            .iter()
11994            .filter_map(|(_, package)| package.as_object())
11995            .map(|object| {
11996                object
11997                    .keys()
11998                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11999                    .count()
12000            })
12001            .max()
12002            .unwrap_or(0);
12003        assert!(
12004            richest >= 2,
12005            "vendored corpus no longer carries a package with unmodelled fields, \
12006             so the relay test can no longer distinguish a verbatim relay from a lossy one"
12007        );
12008
12009        vectors
12010    }
12011
12012    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
12013    ///
12014    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
12015    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
12016    /// three-key object, so a relay that quietly dropped fields it did not
12017    /// recognise would satisfy it. These vectors carry nine keys including ones
12018    /// this crate has no type for, so a typed relay fails here and only here.
12019    #[tokio::test]
12020    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12021        for (corpus_id, package) in fed_admission_facts_vectors() {
12022            let registry = Arc::new(Registry::default());
12023            let forwarding = Arc::new(ForwardingTable::default());
12024            let supervisor = SupervisorHandle::new();
12025            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12026            let handler =
12027                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12028                    .with_supervisor(supervisor)
12029                    .with_admission_facts_config(
12030                        Some("fed".to_string()),
12031                        Some(vec!["target".to_string()]),
12032                    );
12033
12034            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12035            hello_via_sink(
12036                &handler,
12037                &target_ctx,
12038                &mut target_rx,
12039                hello_frame("target", PROTOCOL_VERSION, 1),
12040            )
12041            .await;
12042
12043            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12044            let route_handler = handler.clone();
12045            let expected = package.clone();
12046            let route_task = tokio::spawn(async move {
12047                route_handler
12048                    .handle_control_frame(
12049                        &client_ctx,
12050                        route_open_frame_with_admission_facts(
12051                            20,
12052                            "target",
12053                            unique_project_root("admission-facts"),
12054                            Some(subc_control::ConsumerIdentity {
12055                                module_id: "fed".to_string(),
12056                                launch_nonce: "fed-nonce".to_string(),
12057                            }),
12058                            Some(package),
12059                        ),
12060                    )
12061                    .await
12062                    .unwrap()
12063            });
12064
12065            let bind_frame = target_rx.recv().await.unwrap();
12066            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12067            let ModuleControlRequest::RouteBind {
12068                admission_facts, ..
12069            } = bind
12070            else {
12071                panic!("{corpus_id}: expected route.bind")
12072            };
12073            assert_eq!(
12074                admission_facts,
12075                Some(expected),
12076                "{corpus_id}: relay must not add, drop or reshape any field"
12077            );
12078
12079            handler
12080                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12081                .await
12082                .unwrap();
12083            route_task.await.unwrap();
12084        }
12085    }
12086
12087    #[tokio::test]
12088    async fn route_bind_never_relays_the_openers_build_class() {
12089        let registry = Arc::new(Registry::default());
12090        let forwarding = Arc::new(ForwardingTable::default());
12091        let handler =
12092            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12093        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12094        hello_via_sink(
12095            &handler,
12096            &target_ctx,
12097            &mut target_rx,
12098            hello_frame("target", PROTOCOL_VERSION, 1),
12099        )
12100        .await;
12101
12102        let root = unique_project_root("build-class");
12103        let mut identity = BindIdentity::new(
12104            root.path().to_path_buf(),
12105            "unit".to_string(),
12106            "s".to_string(),
12107        );
12108        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12109        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12110            target: RouteTarget::ToolProvider {
12111                module_id: "target".to_string(),
12112            },
12113            identity,
12114            consumer_identity: None,
12115            consumer_capabilities: None,
12116            role_versions: None,
12117            admission_facts: None,
12118            scope: None,
12119        })
12120        .unwrap();
12121        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12122
12123        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12124        let route_handler = handler.clone();
12125        let route_task = tokio::spawn(async move {
12126            route_handler
12127                .handle_control_frame(&client_ctx, open)
12128                .await
12129                .unwrap()
12130        });
12131        let bind_frame = target_rx.recv().await.unwrap();
12132        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12133        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12134            panic!("expected route.bind")
12135        };
12136        assert_eq!(
12137            identity.build_class, None,
12138            "route.bind carried the build class"
12139        );
12140        assert!(
12141            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12142            "the route.bind bytes still name build_class"
12143        );
12144        handler
12145            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12146            .await
12147            .unwrap();
12148        route_task.await.unwrap();
12149    }
12150
12151    #[tokio::test]
12152    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12153        let registry = Arc::new(Registry::default());
12154        let forwarding = Arc::new(ForwardingTable::default());
12155        let supervisor = SupervisorHandle::new();
12156        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12157        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12158        let handler =
12159            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12160                .with_supervisor(supervisor)
12161                .with_admission_facts_config(
12162                    Some("fed".to_string()),
12163                    Some(vec!["target".to_string()]),
12164                );
12165
12166        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12167        hello_via_sink(
12168            &handler,
12169            &target_ctx,
12170            &mut target_rx,
12171            hello_frame("target", PROTOCOL_VERSION, 1),
12172        )
12173        .await;
12174        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12175        hello_via_sink(
12176            &handler,
12177            &other_ctx,
12178            &mut other_rx,
12179            hello_frame("other", PROTOCOL_VERSION, 2),
12180        )
12181        .await;
12182
12183        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12184        let expected_facts = facts.clone();
12185        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12186        let route_handler = handler.clone();
12187        let route_task = tokio::spawn(async move {
12188            route_handler
12189                .handle_control_frame(
12190                    &client_ctx,
12191                    route_open_frame_with_admission_facts(
12192                        10,
12193                        "target",
12194                        unique_project_root("admission-facts"),
12195                        Some(subc_control::ConsumerIdentity {
12196                            module_id: "fed".to_string(),
12197                            launch_nonce: "fed-nonce".to_string(),
12198                        }),
12199                        Some(facts.clone()),
12200                    ),
12201                )
12202                .await
12203                .unwrap()
12204        });
12205        let bind_frame = target_rx.recv().await.unwrap();
12206        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12207        let ModuleControlRequest::RouteBind {
12208            admission_facts, ..
12209        } = bind
12210        else {
12211            panic!("expected route.bind")
12212        };
12213        assert_eq!(admission_facts, Some(expected_facts));
12214        handler
12215            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12216            .await
12217            .unwrap();
12218        assert!(route_task.await.unwrap().is_empty());
12219        assert!(matches!(
12220            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12221                .unwrap(),
12222            ClientControlResponse::RouteOpen { .. }
12223        ));
12224
12225        let direct = handler
12226            .handle_control_frame(
12227                &route_ctx(ConnectionId::new(73)).0,
12228                route_open_frame_with_admission_facts(
12229                    11,
12230                    "target",
12231                    unique_project_root("admission-facts"),
12232                    None,
12233                    Some(json!({"x": 1})),
12234                ),
12235            )
12236            .await
12237            .unwrap();
12238        assert_eq!(
12239            parse_error(&direct[0])["code"],
12240            "admission_facts_not_permitted"
12241        );
12242
12243        let different_reserved = handler
12244            .handle_control_frame(
12245                &route_ctx(ConnectionId::new(77)).0,
12246                route_open_frame_with_admission_facts(
12247                    15,
12248                    "target",
12249                    unique_project_root("admission-facts"),
12250                    Some(subc_control::ConsumerIdentity {
12251                        module_id: "other".to_string(),
12252                        launch_nonce: "other-nonce".to_string(),
12253                    }),
12254                    Some(json!({"x": 1})),
12255                ),
12256            )
12257            .await
12258            .unwrap();
12259        assert_eq!(
12260            parse_error(&different_reserved[0])["code"],
12261            "admission_facts_not_permitted"
12262        );
12263
12264        let other_target = handler
12265            .handle_control_frame(
12266                &route_ctx(ConnectionId::new(74)).0,
12267                route_open_frame_with_admission_facts(
12268                    12,
12269                    "other",
12270                    unique_project_root("admission-facts"),
12271                    Some(subc_control::ConsumerIdentity {
12272                        module_id: "fed".to_string(),
12273                        launch_nonce: "fed-nonce".to_string(),
12274                    }),
12275                    Some(json!({"x": 1})),
12276                ),
12277            )
12278            .await
12279            .unwrap();
12280        assert_eq!(
12281            parse_error(&other_target[0])["code"],
12282            "admission_facts_target_not_allowed"
12283        );
12284
12285        let nonexistent = handler
12286            .handle_control_frame(
12287                &route_ctx(ConnectionId::new(75)).0,
12288                route_open_frame_with_admission_facts(
12289                    13,
12290                    "missing",
12291                    unique_project_root("admission-facts"),
12292                    None,
12293                    Some(json!({"x": 1})),
12294                ),
12295            )
12296            .await
12297            .unwrap();
12298        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12299
12300        let described = handler
12301            .handle_control_frame(
12302                &route_ctx(ConnectionId::new(76)).0,
12303                Frame::build(
12304                    FrameType::Request,
12305                    control_flags(),
12306                    0,
12307                    0,
12308                    14,
12309                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12310                )
12311                .unwrap(),
12312            )
12313            .await
12314            .unwrap();
12315        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12316            serde_json::from_slice(&described[0].body).unwrap()
12317        else {
12318            panic!("expected server.describe response")
12319        };
12320        assert!(capabilities
12321            .iter()
12322            .any(|cap| cap == "admission_facts_relay_v1"));
12323    }
12324
12325    #[tokio::test]
12326    async fn admission_facts_without_configured_carrier_are_rejected() {
12327        let registry = Arc::new(Registry::default());
12328        let forwarding = Arc::new(ForwardingTable::default());
12329        let handler = ControlHandler::with_forwarding(registry, forwarding);
12330        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12331        hello_via_sink(
12332            &handler,
12333            &target_ctx,
12334            &mut target_rx,
12335            hello_frame("target", PROTOCOL_VERSION, 1),
12336        )
12337        .await;
12338
12339        let responses = handler
12340            .handle_control_frame(
12341                &route_ctx(ConnectionId::new(79)).0,
12342                route_open_frame_with_admission_facts(
12343                    16,
12344                    "target",
12345                    unique_project_root("admission-facts"),
12346                    None,
12347                    Some(json!({"x": 1})),
12348                ),
12349            )
12350            .await
12351            .unwrap();
12352        assert_eq!(
12353            parse_error(&responses[0])["code"],
12354            "admission_facts_not_permitted"
12355        );
12356    }
12357
12358    #[tokio::test]
12359    async fn route_open_relays_consumer_capabilities_verbatim() {
12360        let registry = Arc::new(Registry::default());
12361        let forwarding = Arc::new(ForwardingTable::default());
12362        let handler =
12363            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12364        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12365        hello_via_sink(
12366            &handler,
12367            &module_ctx,
12368            &mut module_rx,
12369            hello_frame("aft", PROTOCOL_VERSION, 7),
12370        )
12371        .await;
12372
12373        let expected = vec!["elicitation".to_string(), "roots".to_string()];
12374        let expected_for_request = expected.clone();
12375        let project_root = unique_project_root("consumer-capabilities-present");
12376        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12377        let route_handler = handler.clone();
12378        let route_task = tokio::spawn(async move {
12379            route_handler
12380                .handle_control_frame(
12381                    &client_ctx,
12382                    route_open_frame_with_consumer_capabilities(
12383                        401,
12384                        "aft",
12385                        project_root,
12386                        Some(expected_for_request),
12387                    ),
12388                )
12389                .await
12390                .unwrap()
12391        });
12392        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12393            .await
12394            .unwrap()
12395            .unwrap();
12396        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12397        let ModuleControlRequest::RouteBind {
12398            consumer_capabilities,
12399            ..
12400        } = bind
12401        else {
12402            panic!("expected route.bind request, got {bind:?}");
12403        };
12404        assert_eq!(consumer_capabilities, Some(expected.clone()));
12405
12406        handler
12407            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12408            .await
12409            .unwrap();
12410        let route_response = route_task.await.unwrap();
12411        assert!(route_response.is_empty());
12412        let published = client_rx.recv().await.unwrap();
12413        assert!(matches!(
12414            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12415            ClientControlResponse::RouteOpen { .. }
12416        ));
12417    }
12418
12419    #[tokio::test]
12420    async fn route_open_without_consumer_capabilities_relays_none() {
12421        let registry = Arc::new(Registry::default());
12422        let forwarding = Arc::new(ForwardingTable::default());
12423        let handler =
12424            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12425        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12426        hello_via_sink(
12427            &handler,
12428            &module_ctx,
12429            &mut module_rx,
12430            hello_frame("aft", PROTOCOL_VERSION, 7),
12431        )
12432        .await;
12433
12434        let project_root = unique_project_root("consumer-capabilities-absent");
12435        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12436        let route_handler = handler.clone();
12437        let route_task = tokio::spawn(async move {
12438            route_handler
12439                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12440                .await
12441                .unwrap()
12442        });
12443        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12444            .await
12445            .unwrap()
12446            .unwrap();
12447        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12448        let ModuleControlRequest::RouteBind {
12449            consumer_capabilities,
12450            ..
12451        } = bind
12452        else {
12453            panic!("expected route.bind request, got {bind:?}");
12454        };
12455        assert_eq!(consumer_capabilities, None);
12456
12457        handler
12458            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12459            .await
12460            .unwrap();
12461        let route_response = route_task.await.unwrap();
12462        assert!(route_response.is_empty());
12463        let published = client_rx.recv().await.unwrap();
12464        assert!(matches!(
12465            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12466            ClientControlResponse::RouteOpen { .. }
12467        ));
12468    }
12469
12470    /// Opens a route to a freshly registered `aft` with `sent` as the
12471    /// route.open's role_versions, acks the bind, and returns the role_versions
12472    /// the module's bind carried.
12473    async fn bind_role_versions_for(
12474        sent: Option<BTreeMap<String, String>>,
12475        connection: u64,
12476    ) -> Option<BTreeMap<String, String>> {
12477        let registry = Arc::new(Registry::default());
12478        let forwarding = Arc::new(ForwardingTable::default());
12479        let handler =
12480            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12481        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12482        hello_via_sink(
12483            &handler,
12484            &module_ctx,
12485            &mut module_rx,
12486            hello_frame("aft", PROTOCOL_VERSION, 7),
12487        )
12488        .await;
12489        let project_root = unique_project_root("role-versions");
12490        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12491        let route_handler = handler.clone();
12492        let route_task = tokio::spawn(async move {
12493            route_handler
12494                .handle_control_frame(
12495                    &client_ctx,
12496                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
12497                )
12498                .await
12499                .unwrap()
12500        });
12501        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12502            .await
12503            .expect("a well-formed route.open reaches the module as a bind")
12504            .unwrap();
12505        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12506        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12507            panic!("expected route.bind request, got {bind:?}");
12508        };
12509        handler
12510            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12511            .await
12512            .unwrap();
12513        assert!(route_task.await.unwrap().is_empty());
12514        let published = client_rx.recv().await.unwrap();
12515        assert!(matches!(
12516            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12517            ClientControlResponse::RouteOpen { .. }
12518        ));
12519        role_versions
12520    }
12521
12522    #[tokio::test]
12523    async fn route_open_relays_role_versions_verbatim() {
12524        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12525        assert_eq!(
12526            bind_role_versions_for(Some(sent.clone()), 141).await,
12527            Some(sent)
12528        );
12529    }
12530
12531    /// An empty map declares nothing, so the provider sees no field rather
12532    /// than an empty object it would have to treat as a second "none".
12533    #[tokio::test]
12534    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12535        assert_eq!(bind_role_versions_for(None, 143).await, None);
12536        assert_eq!(
12537            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12538            None
12539        );
12540    }
12541
12542    #[tokio::test]
12543    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12544        let registry = Arc::new(Registry::default());
12545        let forwarding = Arc::new(ForwardingTable::default());
12546        let handler =
12547            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12548        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12549        hello_via_sink(
12550            &handler,
12551            &module_ctx,
12552            &mut module_rx,
12553            hello_frame("aft", PROTOCOL_VERSION, 7),
12554        )
12555        .await;
12556
12557        let nine: BTreeMap<String, String> = (0..9)
12558            .map(|index| (format!("role-{index}"), "v1".to_string()))
12559            .collect();
12560        for (label, malformed) in [
12561            (
12562                "invalid role name",
12563                role_versions(&[("Tool_Provider", "v1")]),
12564            ),
12565            ("invalid version", role_versions(&[("tool-provider", "v0")])),
12566            ("nine entries", nine),
12567        ] {
12568            // A refused open answers at once; one that reached the module
12569            // would wait for its bind ack and trip this timeout.
12570            let responses = tokio::time::timeout(
12571                Duration::from_secs(1),
12572                handler.handle_control_frame(
12573                    &route_ctx(ConnectionId::new(148)).0,
12574                    route_open_frame_with_role_versions(
12575                        404,
12576                        "aft",
12577                        unique_project_root("role-versions-malformed"),
12578                        Some(malformed),
12579                    ),
12580                ),
12581            )
12582            .await
12583            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12584            .unwrap();
12585            assert_eq!(responses.len(), 1, "{label}");
12586            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12587            let error = parse_error(&responses[0]);
12588            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12589            assert_eq!(
12590                error["detail"]["field"], "role_versions",
12591                "{label}: {error}"
12592            );
12593            assert!(
12594                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12595                "{label}: a malformed declaration is terminal"
12596            );
12597            assert!(
12598                module_rx.try_recv().is_err(),
12599                "{label}: the module must never see a bind"
12600            );
12601        }
12602    }
12603
12604    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
12605    /// consumer can tell this daemon forwards the field from one that would
12606    /// drop it.
12607    #[tokio::test]
12608    async fn route_role_versions_capability_is_advertised() {
12609        let handler = ControlHandler::new(Arc::new(Registry::default()));
12610        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12611        let ack = hello_via_sink(
12612            &handler,
12613            &ctx,
12614            &mut rx,
12615            hello_frame("m", PROTOCOL_VERSION, 1),
12616        )
12617        .await;
12618        let ack = parse_ack(&ack);
12619        assert!(
12620            ack.subc_capabilities
12621                .iter()
12622                .any(|c| c == "route-role-versions/v1"),
12623            "{:?}",
12624            ack.subc_capabilities
12625        );
12626
12627        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12628        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12629        let reply = handler
12630            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12631            .await
12632            .unwrap()
12633            .pop()
12634            .unwrap();
12635        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12636            serde_json::from_slice(&reply.body).unwrap()
12637        else {
12638            panic!("not a server.describe reply");
12639        };
12640        assert!(
12641            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12642            "{capabilities:?}"
12643        );
12644    }
12645
12646    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12647    /// catalog entry without `events` (the module declares none) from a daemon
12648    /// that never reports declared events.
12649    #[tokio::test]
12650    async fn catalog_events_capability_is_advertised() {
12651        let handler = ControlHandler::new(Arc::new(Registry::default()));
12652        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12653        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12654        let reply = handler
12655            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12656            .await
12657            .unwrap()
12658            .pop()
12659            .unwrap();
12660        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12661            serde_json::from_slice(&reply.body).unwrap()
12662        else {
12663            panic!("not a server.describe reply");
12664        };
12665        assert!(
12666            capabilities
12667                .iter()
12668                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12669            "{capabilities:?}"
12670        );
12671    }
12672
12673    #[tokio::test]
12674    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12675        let handler = ControlHandler::new(Arc::new(Registry::default()));
12676        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12677        let ack = hello_via_sink(
12678            &handler,
12679            &module,
12680            &mut module_rx,
12681            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12682        )
12683        .await;
12684        let ack = parse_ack(&ack);
12685        assert!(
12686            ack.subc_capabilities
12687                .iter()
12688                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12689            "{:?}",
12690            ack.subc_capabilities
12691        );
12692        assert!(ack
12693            .subc_capabilities
12694            .iter()
12695            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12696
12697        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12698        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12699        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12700        let reply = handler
12701            .handle_control_frame(&client, frame)
12702            .await
12703            .unwrap()
12704            .pop()
12705            .unwrap();
12706        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12707            serde_json::from_slice(&reply.body).unwrap()
12708        else {
12709            panic!("not a server.describe reply");
12710        };
12711        assert!(
12712            capabilities
12713                .iter()
12714                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12715            "{capabilities:?}"
12716        );
12717        assert!(capabilities
12718            .iter()
12719            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12720    }
12721
12722    #[tokio::test]
12723    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12724        let handler = ControlHandler::new(Arc::new(Registry::default()));
12725        let (module, mut rx) = route_ctx(ConnectionId::new(156));
12726        let ack = hello_via_sink(
12727            &handler,
12728            &module,
12729            &mut rx,
12730            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12731        )
12732        .await;
12733        assert!(parse_ack(&ack)
12734            .subc_capabilities
12735            .iter()
12736            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12737        assert!(parse_ack(&ack)
12738            .subc_capabilities
12739            .iter()
12740            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12741        let (client, _rx) = route_ctx(ConnectionId::new(157));
12742        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12743        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12744        let reply = handler
12745            .handle_control_frame(&client, frame)
12746            .await
12747            .unwrap()
12748            .pop()
12749            .unwrap();
12750        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12751            serde_json::from_slice(&reply.body).unwrap()
12752        else {
12753            panic!("server.describe reply")
12754        };
12755        assert!(capabilities
12756            .iter()
12757            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12758        assert!(capabilities
12759            .iter()
12760            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12761    }
12762
12763    #[tokio::test]
12764    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12765        let handler = ControlHandler::new(Arc::new(Registry::default()));
12766        let (module, mut rx) = route_ctx(ConnectionId::new(158));
12767        let ack = hello_via_sink(
12768            &handler,
12769            &module,
12770            &mut rx,
12771            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12772        )
12773        .await;
12774        assert!(parse_ack(&ack)
12775            .subc_capabilities
12776            .iter()
12777            .any(|cap| cap == "scope-workdir/v1"));
12778        let (client, _rx) = route_ctx(ConnectionId::new(159));
12779        let frame = Frame::build(
12780            FrameType::Request,
12781            control_flags(),
12782            0,
12783            0,
12784            5,
12785            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12786        )
12787        .unwrap();
12788        let reply = handler
12789            .handle_control_frame(&client, frame)
12790            .await
12791            .unwrap()
12792            .remove(0);
12793        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12794            serde_json::from_slice(&reply.body).unwrap()
12795        else {
12796            panic!("server.describe reply")
12797        };
12798        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12799    }
12800
12801    #[tokio::test]
12802    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
12803        let handler = ControlHandler::new(Arc::new(Registry::default()));
12804        let (module, mut rx) = route_ctx(ConnectionId::new(160));
12805        let ack = hello_via_sink(
12806            &handler,
12807            &module,
12808            &mut rx,
12809            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
12810        )
12811        .await;
12812        assert!(parse_ack(&ack)
12813            .subc_capabilities
12814            .iter()
12815            .any(|cap| cap == "worker-runs/v1"));
12816        let (client, _rx) = route_ctx(ConnectionId::new(161));
12817        let frame = Frame::build(
12818            FrameType::Request,
12819            control_flags(),
12820            0,
12821            0,
12822            5,
12823            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12824        )
12825        .unwrap();
12826        let reply = handler
12827            .handle_control_frame(&client, frame)
12828            .await
12829            .unwrap()
12830            .remove(0);
12831        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12832            serde_json::from_slice(&reply.body).unwrap()
12833        else {
12834            panic!("server.describe reply")
12835        };
12836        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
12837    }
12838
12839    #[tokio::test]
12840    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12841        let handler = ControlHandler::new(Arc::new(Registry::default()));
12842        let (module, mut rx) = route_ctx(ConnectionId::new(154));
12843        let ack = hello_via_sink(
12844            &handler,
12845            &module,
12846            &mut rx,
12847            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12848        )
12849        .await;
12850        assert!(parse_ack(&ack)
12851            .subc_capabilities
12852            .iter()
12853            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12854        let (client, _rx) = route_ctx(ConnectionId::new(155));
12855        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12856        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12857        let reply = handler
12858            .handle_control_frame(&client, frame)
12859            .await
12860            .unwrap()
12861            .pop()
12862            .unwrap();
12863        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12864            serde_json::from_slice(&reply.body).unwrap()
12865        else {
12866            panic!("not a server.describe reply")
12867        };
12868        assert!(capabilities
12869            .iter()
12870            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12871    }
12872
12873    #[tokio::test]
12874    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12875        let registry = Arc::new(Registry::default());
12876        let forwarding = Arc::new(ForwardingTable::default());
12877        let handler =
12878            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12879                .with_health_probe_timeout(Duration::from_secs(5));
12880        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12881        hello_via_sink(
12882            &handler,
12883            &module_ctx,
12884            &mut module_rx,
12885            non_routable_hello_frame_with_control_ops(
12886                "mcp",
12887                300,
12888                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12889            ),
12890        )
12891        .await;
12892        assert!(registry
12893            .get_module("mcp")
12894            .unwrap()
12895            .unwrap()
12896            .manifest
12897            .provides
12898            .is_empty());
12899
12900        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12901        let route_response = handler
12902            .handle_control_frame(
12903                &route_client_ctx,
12904                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12905            )
12906            .await
12907            .unwrap();
12908        assert_eq!(route_response[0].header.ty, FrameType::Error);
12909        assert_eq!(
12910            parse_error(&route_response[0])["code"],
12911            "target_unavailable"
12912        );
12913        assert!(parse_error(&route_response[0])["message"]
12914            .as_str()
12915            .unwrap()
12916            .contains("does not provide the requested target"));
12917        assert!(module_rx.try_recv().is_err());
12918
12919        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12920        let health_handler = handler.clone();
12921        let health_task = tokio::spawn(async move {
12922            health_handler
12923                .handle_control_frame(
12924                    &health_client_ctx,
12925                    supervisor_health_probe_frame(302, "mcp"),
12926                )
12927                .await
12928                .unwrap()
12929        });
12930        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12931            .await
12932            .unwrap()
12933            .unwrap();
12934        assert_eq!(
12935            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12936            ModuleControlRequest::HealthCheck {}
12937        );
12938        handler
12939            .handle_control_frame(
12940                &module_ctx,
12941                health_response(health_frame.header.corr, HealthStatus::Ok),
12942            )
12943            .await
12944            .unwrap();
12945        let health_response = health_task.await.unwrap();
12946        assert_eq!(health_response[0].header.ty, FrameType::Response);
12947        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12948            ClientControlResponse::SupervisorHealthProbe {
12949                module_id, status, ..
12950            } => {
12951                assert_eq!(module_id, "mcp");
12952                assert_eq!(status, HealthStatus::Ok);
12953            }
12954            other => panic!("unexpected health response: {other:?}"),
12955        }
12956
12957        // Exercise the forwarding cleanup path directly while leaving the registry
12958        // advertisement in place. If cleanup leaves a stale control sink behind,
12959        // the next probe will enqueue onto it and wait for the long probe timeout
12960        // instead of returning an immediate no-connection error.
12961        forwarding
12962            .cleanup_connection(module_ctx.connection_id)
12963            .unwrap();
12964        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12965        let cleanup_response = tokio::time::timeout(
12966            Duration::from_millis(200),
12967            handler.handle_control_frame(
12968                &cleanup_probe_ctx,
12969                supervisor_health_probe_frame(303, "mcp"),
12970            ),
12971        )
12972        .await
12973        .expect("probe should fail immediately when the control lane is gone")
12974        .unwrap();
12975        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12976        assert_eq!(
12977            parse_error(&cleanup_response[0])["code"],
12978            "target_unavailable"
12979        );
12980        assert!(parse_error(&cleanup_response[0])["message"]
12981            .as_str()
12982            .unwrap()
12983            .contains("no module connection"));
12984
12985        handler
12986            .cleanup_connection(module_ctx.connection_id)
12987            .unwrap();
12988    }
12989
12990    #[tokio::test]
12991    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12992        let registry = Arc::new(Registry::default());
12993        let supervisor_handle = SupervisorHandle::new();
12994        let supervisor =
12995            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12996                .with_handle(supervisor_handle.clone())
12997                .with_connection_file_path(
12998                    std::env::temp_dir()
12999                        .join(format!("subc-route-open-warming-{}", std::process::id())),
13000                );
13001        let module = supervisor
13002            .supervise_configured(
13003                ModuleSpec {
13004                    module_id: "warming".to_string(),
13005                    program: fake_aft_stub_path(),
13006                    args: Vec::new(),
13007                    env: Vec::new(),
13008                    reserved: false,
13009                    reserved_prefixes: Vec::new(),
13010                    protocol: ModuleProtocol::Subc,
13011                    overlap: Default::default(),
13012                },
13013                true,
13014            )
13015            .unwrap();
13016        assert_eq!(module.state().unwrap(), ModuleState::Running);
13017
13018        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13019        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13020        let response = handler
13021            .handle_control_frame(
13022                &ctx,
13023                route_open_frame(304, "warming", unique_project_root("warming")),
13024            )
13025            .await
13026            .unwrap();
13027        module.stop().await.unwrap();
13028
13029        assert_eq!(response[0].header.ty, FrameType::Error);
13030        let error = parse_error(&response[0]);
13031        assert_eq!(error["code"], "module_warming");
13032        assert!(error["message"]
13033            .as_str()
13034            .unwrap()
13035            .contains("state=running, enabled=true, live=false"));
13036    }
13037
13038    #[test]
13039    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13040        let handler = ControlHandler::new(Arc::new(Registry::default()));
13041        let capture = EventCapture::default();
13042        let _subscriber =
13043            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13044        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13045        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13046        let pending = (0..limit).collect::<Vec<_>>();
13047        let response = handler
13048            .route_open_capacity_refusal(
13049                &ctx,
13050                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13051                "busy",
13052                pending.len(),
13053                limit,
13054            )
13055            .unwrap();
13056        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13057        let event = capture
13058            .events()
13059            .into_iter()
13060            .find(|event| {
13061                event.target == "control"
13062                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13063            })
13064            .expect("connection admission refusal event");
13065        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13066        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13067        // Refused before `handle_route_open` parses the request, so the caller
13068        // fields are read from the frame itself.
13069        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13070        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13071    }
13072
13073    #[test]
13074    fn route_open_target_cap_logs_admission_reason_and_capacity() {
13075        let handler = ControlHandler::new(Arc::new(Registry::default()));
13076        let capture = EventCapture::default();
13077        let _subscriber =
13078            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13079        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13080        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13081        let guards = (0..limit)
13082            .map(|_| {
13083                handler
13084                    .route_bind_concurrency
13085                    .try_admit("busy", limit)
13086                    .unwrap()
13087            })
13088            .collect::<Vec<_>>();
13089        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13090            Err(in_flight) => in_flight,
13091            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13092        };
13093        let response = handler
13094            .route_open_target_capacity_refusal(
13095                &ctx,
13096                &RouteOpenCaller::unknown(),
13097                &route_open_frame(397, "busy", unique_project_root("target-cap")),
13098                "busy",
13099                in_flight,
13100            )
13101            .unwrap();
13102        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13103        let event = capture
13104            .events()
13105            .into_iter()
13106            .find(|event| {
13107                event.target == "control"
13108                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13109            })
13110            .expect("target admission refusal event");
13111        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13112        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13113        drop(guards);
13114    }
13115
13116    #[cfg(unix)]
13117    #[tokio::test(start_paused = true)]
13118    async fn supervisor_list_shows_a_failed_modules_next_retry() {
13119        let registry = Arc::new(Registry::default());
13120        let handle = SupervisorHandle::new();
13121        let supervisor =
13122            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13123                .with_handle(handle.clone());
13124        let module = supervisor
13125            .spawn(crate::ModuleSpec {
13126                module_id: "retry-list".into(),
13127                program: PathBuf::from("/bin/sh"),
13128                args: vec!["-c".into(), "exit 17".into()],
13129                env: vec![],
13130                reserved: false,
13131                reserved_prefixes: vec![],
13132                protocol: ModuleProtocol::None,
13133                overlap: Default::default(),
13134            })
13135            .unwrap();
13136        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13137        while module.state().unwrap() != crate::ModuleState::Failed {
13138            assert!(
13139                std::time::Instant::now() < deadline,
13140                "waiting for budget failure"
13141            );
13142            tokio::task::yield_now().await;
13143        }
13144        let handler = ControlHandler::new(registry).with_supervisor(handle);
13145        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13146        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13147        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13148        assert_eq!(body["modules"][0]["state"], "failed");
13149        let retry = body["modules"][0]["next_retry_at_ms"]
13150            .as_u64()
13151            .expect("scheduled retry on the wire");
13152        let now = std::time::SystemTime::now()
13153            .duration_since(std::time::UNIX_EPOCH)
13154            .unwrap()
13155            .as_millis() as u64;
13156        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13157        module.set_enabled(false).await.unwrap();
13158        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13159        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13160        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13161        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13162    }
13163
13164    #[tokio::test]
13165    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13166        let registry = Arc::new(Registry::default());
13167        let handle = SupervisorHandle::new();
13168        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13169            .with_handle(handle.clone());
13170        supervisor
13171            .supervise_configured(
13172                crate::ModuleSpec {
13173                    module_id: "drift".into(),
13174                    program: PathBuf::from("disabled-module"),
13175                    args: Vec::new(),
13176                    env: Vec::new(),
13177                    reserved: false,
13178                    reserved_prefixes: Vec::new(),
13179                    protocol: ModuleProtocol::Subc,
13180                    overlap: Default::default(),
13181                },
13182                false,
13183            )
13184            .unwrap();
13185        let handler = ControlHandler::new(registry).with_supervisor(handle);
13186        for configured in [None, Some(false), Some(true)] {
13187            handler.capability_evaluator.configure(
13188                configured.map(|enabled| ("drift".to_string(), enabled)),
13189                BTreeMap::new(),
13190            );
13191            let frame =
13192                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13193            let replies = handler
13194                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13195                .await
13196                .unwrap();
13197            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13198            assert!(body.get("configured_enabled").is_none());
13199            assert_eq!(body["op"], "supervisor.ack");
13200            let frame =
13201                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13202            let replies = handler.handle_supervisor_list(frame).await.unwrap();
13203            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13204            assert_eq!(
13205                body["modules"][0].get("configured_enabled"),
13206                configured.map(Value::Bool).as_ref()
13207            );
13208        }
13209    }
13210
13211    #[tokio::test(flavor = "current_thread")]
13212    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13213        let handler = ControlHandler::new(Arc::new(Registry::default()));
13214        let capture = EventCapture::default();
13215        let _subscriber =
13216            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13217        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13218        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13219
13220        let _ = handler
13221            .handle_client_control_request(
13222                &ctx,
13223                frame,
13224                ClientControlRequest::SupervisorSetEnabled {
13225                    module_id: "broca".to_string(),
13226                    enabled: false,
13227                },
13228                &mut None,
13229            )
13230            .await
13231            .unwrap();
13232
13233        let events = capture
13234            .events()
13235            .into_iter()
13236            .filter(|event| {
13237                event.target == "control"
13238                    && event.fields.get("message").map(String::as_str)
13239                        == Some("supervisor request received")
13240            })
13241            .collect::<Vec<_>>();
13242        assert_eq!(events.len(), 1, "one supervisor request log line");
13243        let event = &events[0];
13244        assert_eq!(event.level, tracing::Level::INFO);
13245        assert_eq!(
13246            event.fields.get("op"),
13247            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13248        );
13249        assert_eq!(
13250            event.fields.get("module_id"),
13251            Some(&"\"broca\"".to_string())
13252        );
13253        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13254        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13255        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13256    }
13257
13258    #[tokio::test(flavor = "current_thread")]
13259    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13260        let registry = Arc::new(Registry::default());
13261        let module_connection = ConnectionId::new(303);
13262        registry
13263            .register_with_control_ops(
13264                manifest("aft", PROTOCOL_VERSION),
13265                PROTOCOL_VERSION,
13266                module_connection,
13267                module_baseline_control_ops(),
13268            )
13269            .unwrap();
13270        let handler = ControlHandler::new(registry);
13271        let capture = EventCapture::default();
13272        let _subscriber =
13273            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13274        let (ctx, _rx) = route_ctx(module_connection);
13275        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13276
13277        let _ = handler
13278            .handle_client_control_request(
13279                &ctx,
13280                frame,
13281                ClientControlRequest::SupervisorSetEnabled {
13282                    module_id: "broca".to_string(),
13283                    enabled: false,
13284                },
13285                &mut None,
13286            )
13287            .await
13288            .unwrap();
13289
13290        let callers = capture
13291            .events()
13292            .into_iter()
13293            .filter(|event| {
13294                event.fields.get("message").map(String::as_str)
13295                    == Some("supervisor request received")
13296            })
13297            .map(|event| event.fields.get("caller").cloned())
13298            .collect::<Vec<_>>();
13299        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13300    }
13301
13302    #[tokio::test(flavor = "current_thread")]
13303    async fn supervisor_list_does_not_log_request_received() {
13304        let handler = ControlHandler::new(Arc::new(Registry::default()));
13305        let capture = EventCapture::default();
13306        let _subscriber =
13307            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13308        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13309        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13310
13311        handler
13312            .handle_client_control_request(
13313                &ctx,
13314                frame,
13315                ClientControlRequest::SupervisorList {},
13316                &mut None,
13317            )
13318            .await
13319            .unwrap();
13320
13321        assert!(capture.events().into_iter().all(|event| {
13322            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13323        }));
13324    }
13325
13326    #[tokio::test(flavor = "current_thread")]
13327    async fn supervisor_rescan_preview_does_not_log_request_received() {
13328        let handler = ControlHandler::new(Arc::new(Registry::default()));
13329        let capture = EventCapture::default();
13330        let _subscriber =
13331            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13332        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13333        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13334
13335        handler
13336            .handle_client_control_request(
13337                &ctx,
13338                frame,
13339                ClientControlRequest::SupervisorRescan { preview: true },
13340                &mut None,
13341            )
13342            .await
13343            .unwrap();
13344
13345        assert!(capture.events().into_iter().all(|event| {
13346            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13347        }));
13348    }
13349
13350    struct ApproveOperator;
13351    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13352        fn prompt(
13353            &self,
13354            _: &str,
13355            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13356        ) -> crate::operator_confirm::ProviderResult {
13357            crate::operator_confirm::ProviderResult::Approved
13358        }
13359    }
13360
13361    #[tokio::test]
13362    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13363        assert_eq!(
13364            ControlHandler::default().privileged_operator_prompt,
13365            PrivilegedOperatorPrompt::Log
13366        );
13367        for (initial, changed, running) in [
13368            ("log", "enforce", PrivilegedOperatorPrompt::Log),
13369            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13370        ] {
13371            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13372            let path = dir.join("subc.jsonc");
13373            std::fs::write(
13374                &path,
13375                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13376            )
13377            .unwrap();
13378            let config = crate::daemon_config::load(&path).unwrap().unwrap();
13379            let registry = Arc::new(Registry::default());
13380            let handle = SupervisorHandle::new();
13381            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13382                .with_handle(handle.clone());
13383            let handler = ControlHandler::new(registry)
13384                .with_supervisor(handle)
13385                .with_privileged_operator_prompt(config.privileged_operator_prompt)
13386                .with_supervisor_rescan(supervisor, &path, config.port);
13387            handler.forwarding.operator_confirms().configure(
13388                Arc::new(ApproveOperator),
13389                crate::operator_confirm::OperatorLimits::default(),
13390            );
13391            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13392
13393            for (disk_mode, needs_restart) in [
13394                (Some(initial), false),
13395                (Some(changed), true),
13396                (Some(changed), true),
13397                (None, running == PrivilegedOperatorPrompt::Enforce),
13398                (Some(initial), false),
13399            ] {
13400                let mut doc = json!({"version":1});
13401                if let Some(mode) = disk_mode {
13402                    doc["privileged_operator_prompt"] = json!(mode);
13403                }
13404                std::fs::write(&path, doc.to_string()).unwrap();
13405                for preview in [true, false] {
13406                    let request = ClientControlRequest::SupervisorRescan { preview };
13407                    let frame = Frame::build(
13408                        FrameType::Request,
13409                        control_flags(),
13410                        0,
13411                        0,
13412                        1,
13413                        serde_json::to_vec(&request).unwrap(),
13414                    )
13415                    .unwrap();
13416                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13417                    assert_eq!(frames.len(), 1);
13418                    let ClientControlResponse::SupervisorRescan { result } =
13419                        serde_json::from_slice(&frames[0].body).unwrap()
13420                    else {
13421                        panic!("expected a rescan response");
13422                    };
13423                    let expected = if needs_restart {
13424                        vec!["privileged_operator_prompt"]
13425                    } else {
13426                        Vec::new()
13427                    };
13428                    assert_eq!(result.restart_required, expected);
13429                    assert_eq!(handler.privileged_operator_prompt, running);
13430                    assert_eq!(
13431                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13432                        running
13433                    );
13434                }
13435            }
13436        }
13437    }
13438
13439    #[tokio::test]
13440    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13441        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13442        let path = dir.join("subc.jsonc");
13443        let registry = Arc::new(Registry::default());
13444        let handle = SupervisorHandle::new();
13445        let supervisor =
13446            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13447        let handler = ControlHandler::new(registry)
13448            .with_supervisor(handle)
13449            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13450            .with_supervisor_rescan(supervisor, &path, None);
13451        handler.forwarding.operator_confirms().configure(
13452            Arc::new(ApproveOperator),
13453            crate::operator_confirm::OperatorLimits::default(),
13454        );
13455        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13456        for preview in [true, false] {
13457            std::fs::write(
13458                &path,
13459                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13460            )
13461            .unwrap();
13462            let request = ClientControlRequest::SupervisorRescan { preview };
13463            let frame = Frame::build(
13464                FrameType::Request,
13465                control_flags(),
13466                0,
13467                0,
13468                1,
13469                serde_json::to_vec(&request).unwrap(),
13470            )
13471            .unwrap();
13472            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13473            assert_eq!(frames.len(), 1);
13474            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13475            assert_eq!(
13476                handler.privileged_operator_prompt,
13477                PrivilegedOperatorPrompt::Enforce
13478            );
13479        }
13480    }
13481
13482    /// One wire code has several senders, so the refusal line names the check
13483    /// that refused. This drives the shared refusal path for ordinary refusals
13484    /// with an unregistered
13485    /// target and requires the branch label on the event.
13486    #[tokio::test]
13487    async fn route_open_refusal_names_the_check_that_refused() {
13488        let handler = ControlHandler::new(Arc::new(Registry::default()));
13489        let capture = EventCapture::default();
13490        let _subscriber =
13491            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13492        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13493        let response = handler
13494            .handle_control_frame(
13495                &ctx,
13496                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13497            )
13498            .await
13499            .unwrap();
13500
13501        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13502        let event = capture
13503            .events()
13504            .into_iter()
13505            .find(|event| {
13506                event.target == "control"
13507                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13508            })
13509            .expect("route.open refusal event");
13510        assert_eq!(
13511            event.fields.get("reason"),
13512            Some(&"\"not_registered\"".to_string())
13513        );
13514    }
13515
13516    #[tokio::test]
13517    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13518        let registry = Arc::new(Registry::default());
13519        let supervisor_handle = SupervisorHandle::new();
13520        let supervisor =
13521            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13522                .with_handle(supervisor_handle.clone())
13523                .with_connection_file_path(std::env::temp_dir().join(format!(
13524                    "subc-route-open-refusal-info-{}",
13525                    std::process::id()
13526                )));
13527        let module = supervisor
13528            .supervise_configured(
13529                ModuleSpec {
13530                    module_id: "warming".to_string(),
13531                    program: fake_aft_stub_path(),
13532                    args: Vec::new(),
13533                    env: Vec::new(),
13534                    reserved: false,
13535                    reserved_prefixes: Vec::new(),
13536                    protocol: ModuleProtocol::Subc,
13537                    overlap: Default::default(),
13538                },
13539                true,
13540            )
13541            .unwrap();
13542        assert_eq!(module.state().unwrap(), ModuleState::Running);
13543
13544        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13545        assert!(handler
13546            .counters()
13547            .snapshot()
13548            .get("route_open_refused_by_code")
13549            .is_none());
13550        let capture = EventCapture::default();
13551        let _subscriber =
13552            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13553        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13554        let response = handler
13555            .handle_control_frame(
13556                &ctx,
13557                route_open_frame(394, "warming", unique_project_root("refusal-info")),
13558            )
13559            .await
13560            .unwrap();
13561        module.stop().await.unwrap();
13562
13563        assert_eq!(parse_error(&response[0])["code"], "module_warming");
13564        let event = capture
13565            .events()
13566            .into_iter()
13567            .find(|event| {
13568                event.target == "control"
13569                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13570            })
13571            .expect("route.open refusal event");
13572        assert_eq!(
13573            event.fields.get("module_id"),
13574            Some(&"\"warming\"".to_string())
13575        );
13576        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13577        assert_eq!(
13578            event.fields.get("reason"),
13579            Some(&"\"supervised_not_registered\"".to_string())
13580        );
13581        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13582        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13583        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13584        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13585        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13586        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13587        assert_eq!(
13588            handler.counters().snapshot()["route_open_refused_by_code"],
13589            json!({ "module_warming": 1 })
13590        );
13591    }
13592
13593    const OUTAGE_START: &str = "route.open refusing module: not serving";
13594    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13595
13596    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13597        capture
13598            .events()
13599            .into_iter()
13600            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13601            .collect()
13602    }
13603
13604    fn supervise_stub(
13605        registry: &Arc<Registry>,
13606        module_id: &str,
13607        enabled: bool,
13608    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13609        let supervisor_handle = SupervisorHandle::new();
13610        let supervisor =
13611            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13612                .with_handle(supervisor_handle.clone())
13613                .with_connection_file_path(std::env::temp_dir().join(format!(
13614                    "subc-route-outage-{module_id}-{}",
13615                    std::process::id()
13616                )));
13617        let module = supervisor
13618            .supervise_configured(
13619                ModuleSpec {
13620                    module_id: module_id.to_string(),
13621                    program: fake_aft_stub_path(),
13622                    args: Vec::new(),
13623                    env: Vec::new(),
13624                    reserved: false,
13625                    reserved_prefixes: Vec::new(),
13626                    protocol: ModuleProtocol::Subc,
13627                    overlap: Default::default(),
13628                },
13629                enabled,
13630            )
13631            .unwrap();
13632        (supervisor_handle, module)
13633    }
13634
13635    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13636        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13637            module_id: module_id.to_string(),
13638            drain_timeout_ms: Some(50),
13639        })
13640        .unwrap();
13641        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13642    }
13643
13644    /// Two handlers built over one forwarding table must share one outage
13645    /// tracker; separate trackers would each log their own opening line for
13646    /// the same outage.
13647    #[test]
13648    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13649        let registry = Arc::new(Registry::default());
13650        let forwarding = Arc::new(ForwardingTable::default());
13651        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13652        let second = ControlHandler::with_forwarding(registry, forwarding);
13653        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13654    }
13655
13656    /// A client can name any module id it likes. Refusing an unknown one,
13657    /// however often, must not create outage state or outage lines, or the
13658    /// tracker would be a memory sink any client could fill.
13659    #[tokio::test(flavor = "current_thread")]
13660    async fn route_open_unknown_module_refusals_add_no_outage_state() {
13661        let handler = ControlHandler::new(Arc::new(Registry::default()));
13662        let capture = EventCapture::default();
13663        let _subscriber =
13664            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13665        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13666        for corr in 0..8 {
13667            let response = handler
13668                .handle_control_frame(
13669                    &ctx,
13670                    route_open_frame(
13671                        380 + corr,
13672                        &format!("nobody-{corr}"),
13673                        unique_project_root("outage-unknown"),
13674                    ),
13675                )
13676                .await
13677                .unwrap();
13678            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13679        }
13680
13681        assert_eq!(handler.route_outages.tracked_module_count(), 0);
13682        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13683        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13684    }
13685
13686    /// Drives the refusal path end to end: a supervised module that served
13687    /// before and stopped being registered with no instruction to stop is a
13688    /// WARN, and the same module refused after an operator `supervisor.restart`
13689    /// is an INFO.
13690    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13691    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13692        let registry = Arc::new(Registry::default());
13693        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
13694        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13695        let capture = EventCapture::default();
13696        let _subscriber =
13697            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13698        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13699        // The stub never registers, so pretend it served once: otherwise every
13700        // refusal would fall in its startup window.
13701        handler.route_outages.record_accepted("outage-restart");
13702
13703        let response = handler
13704            .handle_control_frame(
13705                &ctx,
13706                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13707            )
13708            .await
13709            .unwrap();
13710        assert_eq!(response[0].header.ty, FrameType::Error);
13711        let starts = outage_lines(&capture, OUTAGE_START);
13712        assert_eq!(starts.len(), 1, "{starts:?}");
13713        assert_eq!(starts[0].level, tracing::Level::WARN);
13714        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13715        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13716        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13717        handler.route_outages.record_accepted("outage-restart");
13718        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13719
13720        let restart = handler
13721            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13722            .await
13723            .unwrap();
13724        assert_eq!(
13725            restart[0].header.ty,
13726            FrameType::Response,
13727            "{:?}",
13728            parse_error(&restart[0])
13729        );
13730        handler
13731            .handle_control_frame(
13732                &ctx,
13733                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13734            )
13735            .await
13736            .unwrap();
13737        module.stop().await.unwrap();
13738
13739        let starts = outage_lines(&capture, OUTAGE_START);
13740        assert_eq!(starts.len(), 2, "{starts:?}");
13741        assert_eq!(starts[1].level, tracing::Level::INFO);
13742        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13743    }
13744
13745    /// A restart refused before it touched the module (here: the module is
13746    /// disabled) must clear its operator mark, so the next real outage is
13747    /// still reported as a warning.
13748    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13749    async fn failed_operator_restart_leaves_no_operator_mark() {
13750        let registry = Arc::new(Registry::default());
13751        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
13752        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13753        let capture = EventCapture::default();
13754        let _subscriber =
13755            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13756        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13757        handler.route_outages.record_accepted("outage-disabled");
13758
13759        let restart = handler
13760            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13761            .await
13762            .unwrap();
13763        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13764        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13765
13766        handler
13767            .handle_control_frame(
13768                &ctx,
13769                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13770            )
13771            .await
13772            .unwrap();
13773        let starts = outage_lines(&capture, OUTAGE_START);
13774        assert_eq!(starts.len(), 1, "{starts:?}");
13775        assert_eq!(starts[0].level, tracing::Level::WARN);
13776    }
13777
13778    #[tokio::test(flavor = "current_thread")]
13779    async fn route_open_unknown_module_escapes_target_module_id() {
13780        let handler = ControlHandler::new(Arc::new(Registry::default()));
13781        let capture = EventCapture::default();
13782        let _subscriber =
13783            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13784        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13785        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13786        let response = handler
13787            .handle_control_frame(
13788                &ctx,
13789                route_open_frame(
13790                    395,
13791                    hostile_module_id,
13792                    unique_project_root("hostile-target-module-id"),
13793                ),
13794            )
13795            .await
13796            .unwrap();
13797
13798        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13799        let event = capture
13800            .events()
13801            .into_iter()
13802            .find(|event| {
13803                event.target == "control"
13804                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13805            })
13806            .expect("route.open unknown-module refusal event");
13807        let logged = event.fields.get("module_id").expect("module_id field");
13808        assert!(!logged.bytes().any(|byte| byte < 0x20));
13809        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13810    }
13811
13812    #[tokio::test(flavor = "current_thread")]
13813    async fn route_open_module_rejection_uses_daemon_counter_key() {
13814        let registry = Arc::new(Registry::default());
13815        let forwarding = Arc::new(ForwardingTable::default());
13816        let handler =
13817            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13818        let module_connection = ConnectionId::new(95);
13819        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13820        hello_via_sink(
13821            &handler,
13822            &module_ctx,
13823            &mut module_rx,
13824            hello_frame("aft", PROTOCOL_VERSION, 395),
13825        )
13826        .await;
13827
13828        let client_connection = ConnectionId::new(96);
13829        let (client_ctx, _client_rx) = route_ctx(client_connection);
13830        let capture = EventCapture::default();
13831        let _subscriber =
13832            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13833        let (route_task, bind) = relay_route_open(
13834            &handler,
13835            client_connection,
13836            &client_ctx.egress,
13837            &mut module_rx,
13838            396,
13839            "aft",
13840            "hostile-module-code",
13841        )
13842        .await;
13843        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13844        let rejection = Frame::build(
13845            FrameType::Error,
13846            control_flags(),
13847            0,
13848            0,
13849            bind.header.corr,
13850            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13851        )
13852        .unwrap();
13853        handler
13854            .handle_control_frame(&module_ctx, rejection)
13855            .await
13856            .unwrap();
13857
13858        let response = route_task.await.unwrap();
13859        assert_eq!(parse_error(&response[0])["code"], hostile_code);
13860        let counters = handler.counters().snapshot();
13861        assert_eq!(
13862            counters["route_open_refused_by_code"],
13863            json!({ "module_rejected": 1 })
13864        );
13865        assert!(counters["route_open_refused_by_code"]
13866            .get(hostile_code)
13867            .is_none());
13868
13869        let event = capture
13870            .events()
13871            .into_iter()
13872            .find(|event| {
13873                event.target == "control"
13874                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13875            })
13876            .expect("route.open module-rejection refusal event");
13877        let logged = event.fields.get("module_code").expect("module_code field");
13878        assert!(!logged.bytes().any(|byte| byte < 0x20));
13879        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13880    }
13881
13882    #[tokio::test]
13883    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13884        let registry = Arc::new(Registry::default());
13885        let supervisor_handle = SupervisorHandle::new();
13886        let missing_program = std::env::temp_dir().join(format!(
13887            "subc-route-open-missing-program-{}",
13888            std::process::id()
13889        ));
13890        let supervisor =
13891            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13892                .with_handle(supervisor_handle.clone());
13893        let module = supervisor
13894            .supervise_configured(
13895                ModuleSpec {
13896                    module_id: "failed".to_string(),
13897                    program: missing_program,
13898                    args: Vec::new(),
13899                    env: Vec::new(),
13900                    reserved: false,
13901                    reserved_prefixes: Vec::new(),
13902                    protocol: ModuleProtocol::Subc,
13903                    overlap: Default::default(),
13904                },
13905                true,
13906            )
13907            .unwrap();
13908        assert_eq!(module.state().unwrap(), ModuleState::Failed);
13909
13910        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13911        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13912        let response = handler
13913            .handle_control_frame(
13914                &ctx,
13915                route_open_frame(305, "failed", unique_project_root("failed")),
13916            )
13917            .await
13918            .unwrap();
13919
13920        assert_eq!(response[0].header.ty, FrameType::Error);
13921        let error = parse_error(&response[0]);
13922        assert_eq!(error["code"], "target_unavailable");
13923        assert!(error["message"]
13924            .as_str()
13925            .unwrap()
13926            .contains("state=failed, enabled=true, live=false"));
13927    }
13928
13929    #[tokio::test]
13930    async fn route_open_role_mismatch_remains_target_unavailable() {
13931        let registry = Arc::new(Registry::default());
13932        let handler = ControlHandler::new(Arc::clone(&registry));
13933        handler
13934            .handle_control(
13935                ConnectionId::new(41),
13936                non_routable_hello_frame_with_control_ops("health-only", 306, None),
13937            )
13938            .unwrap();
13939
13940        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13941        let response = handler
13942            .handle_control_frame(
13943                &ctx,
13944                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13945            )
13946            .await
13947            .unwrap();
13948
13949        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13950        assert!(parse_error(&response[0])["message"]
13951            .as_str()
13952            .unwrap()
13953            .contains("does not provide the requested target"));
13954    }
13955
13956    #[tokio::test]
13957    async fn route_open_inactive_registration_remains_target_unavailable() {
13958        let registry = Arc::new(Registry::default());
13959        let handler = ControlHandler::new(Arc::clone(&registry));
13960        handler
13961            .handle_control(
13962                ConnectionId::new(43),
13963                hello_frame("inactive", PROTOCOL_VERSION, 308),
13964            )
13965            .unwrap();
13966        assert!(registry
13967            .set_module_state_for_test("inactive", ChannelState::Closed)
13968            .unwrap());
13969
13970        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13971        let response = handler
13972            .handle_control_frame(
13973                &ctx,
13974                route_open_frame(309, "inactive", unique_project_root("inactive")),
13975            )
13976            .await
13977            .unwrap();
13978
13979        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13980        assert!(parse_error(&response[0])["message"]
13981            .as_str()
13982            .unwrap()
13983            .contains("is not active"));
13984    }
13985
13986    #[tokio::test]
13987    async fn late_health_reply_is_recorded_through_the_module_response_path() {
13988        let registry = Arc::new(Registry::default());
13989        let forwarding = Arc::new(ForwardingTable::default());
13990        let supervisor_handle = SupervisorHandle::new();
13991        let supervisor =
13992            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
13993                .with_forwarding(Arc::clone(&forwarding))
13994                .with_handle(supervisor_handle.clone());
13995        let module = supervisor
13996            .supervise_configured(
13997                crate::ModuleSpec {
13998                    module_id: "late-health-response".to_string(),
13999                    program: PathBuf::from("disabled-module"),
14000                    args: Vec::new(),
14001                    env: Vec::new(),
14002                    reserved: false,
14003                    reserved_prefixes: Vec::new(),
14004                    protocol: ModuleProtocol::Subc,
14005                    overlap: Default::default(),
14006                },
14007                false,
14008            )
14009            .unwrap();
14010        let handler =
14011            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14012                .with_supervisor(supervisor_handle);
14013        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14014        handler
14015            .handle_control_frame(
14016                &module_ctx,
14017                hello_frame_with_control_ops(
14018                    "late-health-response",
14019                    PROTOCOL_VERSION,
14020                    7,
14021                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14022                ),
14023            )
14024            .await
14025            .unwrap();
14026        let probe_started_at = Instant::now() - Duration::from_millis(80);
14027        let pending = forwarding
14028            .begin_health_probe_rpc_for(
14029                "late-health-response",
14030                MODULE_CONTROL_OP_HEALTH_CHECK,
14031                probe_started_at,
14032                Instant::now() - Duration::from_millis(1),
14033            )
14034            .unwrap();
14035        assert!(forwarding
14036            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
14037            .unwrap());
14038
14039        let responses = handler
14040            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
14041            .await
14042            .unwrap();
14043
14044        assert!(responses.is_empty());
14045        let health = module.status().unwrap().health;
14046        assert_eq!(health.late_answer_count, 1);
14047        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
14048    }
14049
14050    #[tokio::test]
14051    async fn health_probe_timeout_and_module_death_are_typed() {
14052        let registry = Arc::new(Registry::default());
14053        let forwarding = Arc::new(ForwardingTable::default());
14054        let handler =
14055            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14056                .with_health_probe_timeout(Duration::from_millis(50));
14057        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14058        hello_via_sink(
14059            &handler,
14060            &module_ctx,
14061            &mut module_rx,
14062            hello_frame_with_control_ops(
14063                "aft",
14064                PROTOCOL_VERSION,
14065                7,
14066                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14067            ),
14068        )
14069        .await;
14070
14071        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14072        let responses = handler
14073            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14074            .await
14075            .unwrap();
14076        assert_eq!(responses[0].header.ty, FrameType::Error);
14077        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14078        let _ = module_rx.try_recv();
14079
14080        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14081        let health_handler = handler.clone();
14082        let death_task = tokio::spawn(async move {
14083            health_handler
14084                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14085                .await
14086                .unwrap()
14087        });
14088        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14089            .await
14090            .unwrap()
14091            .unwrap();
14092        handler
14093            .cleanup_connection(module_ctx.connection_id)
14094            .unwrap();
14095        let responses = death_task.await.unwrap();
14096        assert_eq!(responses[0].header.ty, FrameType::Error);
14097        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14098    }
14099
14100    #[test]
14101    fn hello_requires_exact_protocol_version() {
14102        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14103            let registry = Arc::new(Registry::default());
14104            let handler = ControlHandler::new(Arc::clone(&registry));
14105            let responses = handler
14106                .handle_control(
14107                    ConnectionId::new(connection),
14108                    hello_frame("aft", offered, 9),
14109                )
14110                .unwrap();
14111
14112            assert_eq!(responses.len(), 1);
14113            assert_eq!(responses[0].header.ty, FrameType::Error);
14114            let error = parse_error(&responses[0]);
14115            assert_eq!(error["code"], "version_unsupported");
14116            assert!(registry.get_module("aft").unwrap().is_none());
14117            assert_eq!(registry.active_registration_count().unwrap(), 0);
14118        }
14119    }
14120
14121    #[test]
14122    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14123        let registry = Arc::new(Registry::default());
14124        let forwarding = Arc::new(ForwardingTable::default());
14125        let handler =
14126            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14127        let module_connection = ConnectionId::new(301);
14128        let registration = registry
14129            .register_with_control_ops(
14130                manifest("aft-push", PROTOCOL_VERSION),
14131                PROTOCOL_VERSION,
14132                module_connection,
14133                module_baseline_control_ops(),
14134            )
14135            .unwrap();
14136        let (module_tx, _module_rx) = mpsc::channel(8);
14137        let endpoint = forwarding
14138            .register_module_connection(
14139                module_connection,
14140                "aft-push".to_string(),
14141                PROTOCOL_VERSION,
14142                manifest_concurrency(&registration.manifest),
14143                FrameSink::new(module_tx),
14144            )
14145            .unwrap();
14146
14147        // A push op this version does not know is ignored (forward-compat), not errored.
14148        let unknown = Frame::build(
14149            FrameType::Push,
14150            control_flags(),
14151            0,
14152            0,
14153            5,
14154            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14155        )
14156        .unwrap();
14157        let out = handler.handle_status_update(endpoint, unknown).unwrap();
14158        assert!(
14159            out.is_empty(),
14160            "unknown push op must be ignored, got {out:?}"
14161        );
14162
14163        // A malformed body for a KNOWN op is a real error worth surfacing.
14164        let malformed = Frame::build(
14165            FrameType::Push,
14166            control_flags(),
14167            0,
14168            0,
14169            6,
14170            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14171        )
14172        .unwrap();
14173        let out = handler.handle_status_update(endpoint, malformed).unwrap();
14174        assert_eq!(out.len(), 1);
14175        assert_eq!(out[0].header.ty, FrameType::Error);
14176        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14177    }
14178
14179    #[test]
14180    fn hello_rejected_when_connection_already_owns_client_routes() {
14181        let registry = Arc::new(Registry::default());
14182        let forwarding = Arc::new(ForwardingTable::default());
14183        let handler =
14184            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14185        // Commits a client route on connection 202 (bound to a module on conn 101).
14186        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
14187        let client_connection = ConnectionId::new(202);
14188
14189        // That same connection now tries to register as a module: rejected, so one
14190        // connection never holds both client-route and module-endpoint state.
14191        let responses = handler
14192            .handle_control(
14193                client_connection,
14194                hello_frame("aft-second", PROTOCOL_VERSION, 9),
14195            )
14196            .unwrap();
14197        assert_eq!(responses[0].header.ty, FrameType::Error);
14198        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14199        assert!(registry.get_module("aft-second").unwrap().is_none());
14200    }
14201
14202    #[tokio::test]
14203    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14204        let registry = Arc::new(Registry::default());
14205        let forwarding = Arc::new(ForwardingTable::default());
14206        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14207        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14208        hello_via_sink(
14209            &handler,
14210            &module_ctx,
14211            &mut module_rx,
14212            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14213        )
14214        .await;
14215        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14216        let pending = forwarding
14217            .begin_route_bind_relay_for_test(
14218                client_ctx.connection_id,
14219                client_ctx.egress.clone(),
14220                2,
14221                "alpha",
14222            )
14223            .unwrap();
14224        forwarding
14225            .complete_pending_relay(
14226                module_ctx.connection_id,
14227                pending.corr,
14228                RouteBindRelayOutcome::Accepted,
14229            )
14230            .unwrap();
14231        client_rx.try_recv().unwrap();
14232        for module_id in ["beta", "alpha"] {
14233            let replies = handler
14234                .handle_control_frame(
14235                    &module_ctx,
14236                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14237                )
14238                .await
14239                .unwrap();
14240            assert_eq!(replies.len(), 1, "second HELLO must be refused");
14241            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14242        }
14243        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14244        assert!(registry.get_module("beta").unwrap().is_none());
14245        assert!(matches!(
14246            forwarding
14247                .lookup_data_route(
14248                    client_ctx.connection_id,
14249                    pending.client_channel,
14250                    pending.client_epoch,
14251                )
14252                .unwrap(),
14253            DataRoute::Client(DataRouteState::Bound(_))
14254        ));
14255        assert!(handler
14256            .hello_launch_nonces
14257            .lock()
14258            .unwrap()
14259            .presented(module_ctx.connection_id, Some("alpha-nonce")));
14260        assert!(module_rx.try_recv().is_err());
14261    }
14262
14263    #[test]
14264    fn reserved_module_hello_requires_matching_launch_nonce() {
14265        let registry = Arc::new(Registry::default());
14266        let supervisor = SupervisorHandle::new();
14267        // The supervisor recorded the nonce it injected when it spawned the reserved
14268        // module; the HELLO verifier checks against the same shared handle.
14269        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14270        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14271
14272        // A HELLO with NO nonce is rejected.
14273        let no_nonce = handler
14274            .handle_control(
14275                ConnectionId::new(1),
14276                hello_frame("vault", PROTOCOL_VERSION, 1),
14277            )
14278            .unwrap();
14279        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14280        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14281        assert!(registry.get_module("vault").unwrap().is_none());
14282
14283        // A HELLO with the WRONG nonce is rejected.
14284        let wrong = handler
14285            .handle_control(
14286                ConnectionId::new(2),
14287                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14288            )
14289            .unwrap();
14290        assert_eq!(wrong[0].header.ty, FrameType::Error);
14291        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14292        assert!(registry.get_module("vault").unwrap().is_none());
14293
14294        // A HELLO with the CORRECT nonce registers.
14295        let ok = handler
14296            .handle_control(
14297                ConnectionId::new(3),
14298                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14299            )
14300            .unwrap();
14301        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14302        assert!(registry.get_module("vault").unwrap().is_some());
14303    }
14304
14305    #[test]
14306    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14307        let registry = Arc::new(Registry::default());
14308        let supervisor = SupervisorHandle::new();
14309        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14310        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14311        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14312
14313        let squat = handler
14314            .handle_control(
14315                ConnectionId::new(1),
14316                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14317            )
14318            .unwrap();
14319        assert_eq!(squat[0].header.ty, FrameType::Error);
14320        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14321        assert!(parse_error(&squat[0])["message"]
14322            .as_str()
14323            .unwrap()
14324            .contains("fed:"));
14325
14326        let accepted_peer = handler
14327            .handle_control(
14328                ConnectionId::new(2),
14329                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14330            )
14331            .unwrap();
14332        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14333
14334        let accepted_short = handler
14335            .handle_control(
14336                ConnectionId::new(3),
14337                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14338            )
14339            .unwrap();
14340        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14341
14342        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14343            let response = handler
14344                .handle_control(
14345                    ConnectionId::new(conn),
14346                    hello_frame(module_id, PROTOCOL_VERSION, conn),
14347                )
14348                .unwrap();
14349            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14350        }
14351    }
14352
14353    #[test]
14354    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14355        let registry = Arc::new(Registry::default());
14356        let supervisor = SupervisorHandle::new();
14357        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14358        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14359        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14360        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14361
14362        let owner_nonce = handler
14363            .handle_control(
14364                ConnectionId::new(1),
14365                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14366            )
14367            .unwrap();
14368        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14369        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14370        assert!(registry.get_module("fed:special").unwrap().is_none());
14371
14372        let exact_nonce = handler
14373            .handle_control(
14374                ConnectionId::new(2),
14375                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14376            )
14377            .unwrap();
14378        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14379        assert!(registry.get_module("fed:special").unwrap().is_some());
14380    }
14381
14382    #[test]
14383    fn non_reserved_module_ignores_launch_nonce() {
14384        let registry = Arc::new(Registry::default());
14385        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
14386        // registration succeeds whether a spawned process echoes a nonce or not.
14387        let handler = ControlHandler::new(Arc::clone(&registry));
14388        let no_nonce = handler
14389            .handle_control(
14390                ConnectionId::new(1),
14391                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14392            )
14393            .unwrap();
14394        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14395        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14396
14397        let echoed_nonce = handler
14398            .handle_control(
14399                ConnectionId::new(2),
14400                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14401            )
14402            .unwrap();
14403        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14404        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14405    }
14406
14407    #[test]
14408    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14409        let handler = ControlHandler::default();
14410        let conn = ConnectionId::new(1);
14411        let malformed = Frame::build(
14412            FrameType::Hello,
14413            control_flags(),
14414            0,
14415            0,
14416            3,
14417            b"{not json".to_vec(),
14418        )
14419        .unwrap();
14420
14421        let error = handler.handle_control(conn, malformed).unwrap();
14422        assert_eq!(error[0].header.ty, FrameType::Error);
14423        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14424
14425        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14426        let pong = handler.handle_control(conn, ping).unwrap();
14427        assert_eq!(pong[0].header.ty, FrameType::Pong);
14428        assert_eq!(pong[0].header.corr, 4);
14429    }
14430
14431    #[test]
14432    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14433        let registry = Arc::new(Registry::default());
14434        let handler = ControlHandler::new(Arc::clone(&registry));
14435
14436        handler
14437            .handle_control(
14438                ConnectionId::new(1),
14439                hello_frame("aft", PROTOCOL_VERSION, 1),
14440            )
14441            .unwrap();
14442        let duplicate = handler
14443            .handle_control(
14444                ConnectionId::new(2),
14445                hello_frame("aft", PROTOCOL_VERSION, 2),
14446            )
14447            .unwrap();
14448
14449        assert_eq!(duplicate[0].header.ty, FrameType::Error);
14450        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14451        let registration = registry.get_module("aft").unwrap().unwrap();
14452        assert_eq!(registration.connection_id, ConnectionId::new(1));
14453    }
14454
14455    #[test]
14456    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14457        let registry = Arc::new(Registry::default());
14458        let forwarding = Arc::new(ForwardingTable::default());
14459        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14460        let handler =
14461            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14462                .with_process_liveness(process_liveness);
14463        let (ctx, route_channel, route_epoch) =
14464            bind_liveness_route(&registry, &forwarding, "aft-dead");
14465        let responses = handler
14466            .handle_route_poll(
14467                &ctx,
14468                route_poll_frame(41, PollKind::Liveness, route_channel),
14469                route_channel,
14470                route_epoch,
14471                PollKind::Liveness,
14472            )
14473            .unwrap();
14474
14475        assert_eq!(responses.len(), 1);
14476        assert_eq!(responses[0].header.ty, FrameType::Response);
14477        assert_route_poll_liveness(&responses[0], false);
14478    }
14479
14480    #[test]
14481    fn liveness_poll_without_process_source_uses_bound_route() {
14482        let registry = Arc::new(Registry::default());
14483        let forwarding = Arc::new(ForwardingTable::default());
14484        let handler =
14485            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14486        let (ctx, route_channel, route_epoch) =
14487            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
14488        let responses = handler
14489            .handle_route_poll(
14490                &ctx,
14491                route_poll_frame(42, PollKind::Liveness, route_channel),
14492                route_channel,
14493                route_epoch,
14494                PollKind::Liveness,
14495            )
14496            .unwrap();
14497
14498        assert_route_poll_liveness(&responses[0], true);
14499    }
14500
14501    #[test]
14502    fn liveness_poll_untracked_process_source_uses_bound_route() {
14503        let registry = Arc::new(Registry::default());
14504        let forwarding = Arc::new(ForwardingTable::default());
14505        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14506        let handler =
14507            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14508                .with_process_liveness(process_liveness);
14509        let (ctx, route_channel, route_epoch) =
14510            bind_liveness_route(&registry, &forwarding, "aft-untracked");
14511        let responses = handler
14512            .handle_route_poll(
14513                &ctx,
14514                route_poll_frame(43, PollKind::Liveness, route_channel),
14515                route_channel,
14516                route_epoch,
14517                PollKind::Liveness,
14518            )
14519            .unwrap();
14520
14521        assert_route_poll_liveness(&responses[0], true);
14522    }
14523
14524    #[tokio::test]
14525    async fn unknown_op_returns_unknown_control_op() {
14526        let handler = ControlHandler::default();
14527        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14528        let request = Frame::build(
14529            FrameType::Request,
14530            control_flags(),
14531            0,
14532            0,
14533            55,
14534            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14535        )
14536        .unwrap();
14537
14538        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14539
14540        assert_eq!(response.len(), 1);
14541        assert_eq!(response[0].header.ty, FrameType::Error);
14542        assert_eq!(response[0].header.corr, 55);
14543        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14544    }
14545
14546    #[tokio::test]
14547    async fn supervisor_provenance_rejects_unknown_exact_module() {
14548        let handler = ControlHandler::default();
14549        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14550        let request = Frame::build(
14551            FrameType::Request,
14552            control_flags(),
14553            0,
14554            0,
14555            57,
14556            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14557        )
14558        .unwrap();
14559
14560        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14561
14562        assert_eq!(response.len(), 1);
14563        assert_eq!(response[0].header.ty, FrameType::Error);
14564        assert_eq!(response[0].header.corr, 57);
14565        let error = parse_error(&response[0]);
14566        assert_eq!(error["code"], "unknown_module");
14567        assert_eq!(error["message"], "module_id 'missing' is not supervised");
14568    }
14569
14570    #[test]
14571    fn provenance_probe_override_keeps_handler_tests_deterministic() {
14572        let expected = subc_control::RunningImageAgreement::Unavailable {
14573            reason: subc_control::RunningImageUnavailableReason::HashFailed,
14574        };
14575        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14576        assert_eq!(handler.provenance_probe_override, Some(expected));
14577    }
14578
14579    #[test]
14580    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14581        let verdict = reload_verdict(
14582            std::path::Path::new("/bin/new"),
14583            Some(std::path::Path::new("/bin/old")),
14584            subc_control::RunningImageAgreement::Unavailable {
14585                reason: subc_control::RunningImageUnavailableReason::HashFailed,
14586            },
14587        );
14588        assert!(matches!(
14589            verdict.path,
14590            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14591                if configured == std::path::Path::new("/bin/new")
14592                    && spawned_from == std::path::Path::new("/bin/old")
14593        ));
14594    }
14595
14596    #[test]
14597    fn launch_spec_without_running_digest_is_unavailable_not_match() {
14598        use subc_control::{
14599            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14600        };
14601        let configured = crate::supervise::ModuleSpec {
14602            module_id: "legacy".into(),
14603            program: "/module".into(),
14604            args: Vec::new(),
14605            env: Vec::new(),
14606            reserved: false,
14607            reserved_prefixes: Vec::new(),
14608            protocol: subc_control::ModuleProtocol::Subc,
14609            overlap: Default::default(),
14610        };
14611        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14612        assert_eq!(
14613            launch_spec_agreement(&configured, false, Some(&digest)),
14614            Agreement::Unavailable {
14615                reason: Reason::NotRunning
14616            }
14617        );
14618        assert_eq!(
14619            launch_spec_agreement(&configured, true, None),
14620            Agreement::Unavailable {
14621                reason: Reason::SpawnedSpecUnavailable
14622            }
14623        );
14624        assert_eq!(
14625            launch_spec_agreement(&configured, true, Some(&digest)),
14626            Agreement::Match
14627        );
14628    }
14629
14630    #[cfg(windows)]
14631    #[test]
14632    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14633        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14634        let configured = dir.join("module.exe");
14635        std::fs::write(&configured, b"executable path fixture").unwrap();
14636        let spawned = std::fs::canonicalize(&configured).unwrap();
14637        assert_ne!(
14638            configured, spawned,
14639            "fixture must exercise different path spellings"
14640        );
14641        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14642        let lower_case = configured.to_string_lossy().to_lowercase();
14643        for spelling in [
14644            &configured,
14645            Path::new(&forward_slashes),
14646            Path::new(&lower_case),
14647        ] {
14648            let verdict = reload_verdict(
14649                spelling,
14650                Some(&spawned),
14651                subc_control::RunningImageAgreement::Unavailable {
14652                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14653                },
14654            );
14655            assert_eq!(
14656                verdict.path,
14657                ReloadPathAgreement::Match,
14658                "configured {spelling:?}, spawned {spawned:?}"
14659            );
14660        }
14661    }
14662
14663    #[cfg(windows)]
14664    #[test]
14665    fn windows_reload_path_keeps_different_existing_program_pending() {
14666        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14667        let old = dir.join("old.exe");
14668        let new = dir.join("new.exe");
14669        std::fs::write(&old, b"same executable content").unwrap();
14670        std::fs::copy(&old, &new).unwrap();
14671        let spawned = std::fs::canonicalize(&old).unwrap();
14672        let image = subc_control::RunningImageAgreement::Unavailable {
14673            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14674        };
14675        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14676        assert!(
14677            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14678            if configured == new && spawned_from == spawned)
14679        );
14680        let missing = dir.join("missing.exe");
14681        assert!(matches!(
14682            reload_verdict(&missing, Some(&spawned), image).path,
14683            ReloadPathAgreement::Mismatch { .. }
14684        ));
14685    }
14686
14687    #[test]
14688    fn reload_verdict_detects_replaced_image_at_same_path() {
14689        let image = subc_control::RunningImageAgreement::Mismatch {
14690            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14691                digest: "old".into(),
14692            },
14693            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14694                digest: "new".into(),
14695            },
14696        };
14697        let verdict = reload_verdict(
14698            std::path::Path::new("/bin/same"),
14699            Some(std::path::Path::new("/bin/same")),
14700            image.clone(),
14701        );
14702        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14703        assert_eq!(verdict.image, image);
14704    }
14705
14706    #[test]
14707    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14708        let image = subc_control::RunningImageAgreement::Unavailable {
14709            reason: subc_control::RunningImageUnavailableReason::NotRunning,
14710        };
14711        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14712        assert_eq!(
14713            verdict.path,
14714            subc_control::ReloadPathAgreement::Unavailable {
14715                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14716            }
14717        );
14718        assert_eq!(verdict.image, image);
14719
14720        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14721            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14722        };
14723        let verdict = reload_verdict(
14724            std::path::Path::new("/bin/same"),
14725            Some(std::path::Path::new("/bin/same")),
14726            unconfirmed.clone(),
14727        );
14728        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14729        assert_eq!(verdict.image, unconfirmed);
14730    }
14731
14732    #[test]
14733    fn reload_verdict_preserves_each_image_unavailability_reason() {
14734        use subc_control::RunningImageUnavailableReason as Reason;
14735
14736        for reason in [
14737            Reason::NotRunning,
14738            Reason::UnsupportedPlatform,
14739            Reason::RunningExecutableUnreadable,
14740            Reason::SpawnedPathUnreadable,
14741            Reason::HashFailed,
14742            Reason::ProcessIdentityUnconfirmed,
14743            Reason::Unknown("future_probe_reason".to_string()),
14744        ] {
14745            let image = subc_control::RunningImageAgreement::Unavailable {
14746                reason: reason.clone(),
14747            };
14748            let verdict = reload_verdict(
14749                std::path::Path::new("/bin/same"),
14750                Some(std::path::Path::new("/bin/same")),
14751                image.clone(),
14752            );
14753            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14754            assert_eq!(verdict.image, image, "{reason:?}");
14755        }
14756    }
14757
14758    #[tokio::test]
14759    async fn malformed_control_bodies_return_invalid_control_body() {
14760        let handler = ControlHandler::default();
14761        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14762
14763        for (corr, body) in [
14764            (56, br#"{"route_channel":1}"#.as_slice()),
14765            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14766            (
14767                58,
14768                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14769            ),
14770        ] {
14771            let request = Frame::build(
14772                FrameType::Request,
14773                control_flags(),
14774                0,
14775                0,
14776                corr,
14777                body.to_vec(),
14778            )
14779            .unwrap();
14780            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14781
14782            assert_eq!(response.len(), 1);
14783            assert_eq!(response[0].header.ty, FrameType::Error);
14784            assert_eq!(response[0].header.corr, corr);
14785            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14786        }
14787    }
14788
14789    #[tokio::test]
14790    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14791        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14792        let registry = Arc::new(Registry::default());
14793        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14794        let router = Router::with_control_handler(Arc::clone(&control));
14795        let connection = router.begin_connection();
14796        let (ctx, mut rx) = route_ctx(connection.id());
14797
14798        router
14799            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14800            .await
14801            .unwrap();
14802        let response = rx.recv().await.unwrap();
14803        let ack = parse_ack(&response);
14804        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14805        let channel = 1;
14806
14807        let goodbye =
14808            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14809        router.route_for_connection(&ctx, goodbye).await.unwrap();
14810        assert!(rx.try_recv().is_err());
14811        assert!(registry.get_module("aft").unwrap().is_none());
14812
14813        router
14814            .route_for_connection(&ctx, channel_request(channel, 13))
14815            .await
14816            .unwrap();
14817        let error_frame = rx.recv().await.unwrap();
14818        assert_eq!(error_frame.header.ty, FrameType::Error);
14819        assert_eq!(error_frame.header.channel, channel);
14820        let captured = crate::router::test_log::captured_logs(&logs);
14821        assert_eq!(
14822            captured
14823                .lines()
14824                .filter(|line| {
14825                    line.contains("module_id=aft")
14826                        && line.contains("reason=explicit_goodbye")
14827                        && line.contains("module registration ended")
14828                })
14829                .count(),
14830            1,
14831            "unexpected GOODBYE registry log: {captured}"
14832        );
14833    }
14834
14835    #[tokio::test]
14836    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14837        let registry = Arc::new(Registry::default());
14838        let handler = ControlHandler::new(registry).with_capability_config(
14839            [("prov".to_string(), true), ("cons".to_string(), true)],
14840            BTreeMap::new(),
14841        );
14842        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14843        register_capability_manifest(
14844            &handler,
14845            &provider_ctx,
14846            &mut provider_rx,
14847            capability_manifest("prov", &["thing/v1"], &[]),
14848            1,
14849        )
14850        .await;
14851        let mut consumer = capability_manifest("cons", &[], &[]);
14852        consumer.capabilities.as_mut().unwrap().requires.push(
14853            subc_protocol::manifest::CapabilityRequirement {
14854                capability: "thing/v1".to_string(),
14855                need: subc_protocol::manifest::CapabilityNeed::Required,
14856            },
14857        );
14858        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14859        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14860        assert_eq!(
14861            handler.capability_evaluator.verdict("cons", "thing/v1"),
14862            Some(CapabilityVerdict::Provided)
14863        );
14864        let (mut client_rx, _) = open_route_for_capability_test(
14865            &handler,
14866            &provider_ctx,
14867            &mut provider_rx,
14868            703,
14869            3,
14870            "prov",
14871            None,
14872        )
14873        .await;
14874        let goodbye =
14875            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14876        handler
14877            .handle_control_frame(&provider_ctx, goodbye)
14878            .await
14879            .unwrap();
14880        assert_eq!(
14881            handler.capability_evaluator.verdict("cons", "thing/v1"),
14882            Some(CapabilityVerdict::NeverProvided)
14883        );
14884        let closed = client_rx
14885            .try_recv()
14886            .expect("GOODBYE pushes route.closed before route GOODBYE");
14887        assert!(
14888            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14889            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14890        );
14891        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14892        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14893    }
14894
14895    struct RouteClosedFixture {
14896        handler: ControlHandler,
14897        #[cfg(unix)]
14898        supervisor: Supervisor,
14899        module_ctx: RouteCtx,
14900        #[cfg(unix)]
14901        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14902        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14903        route: Arc<crate::forwarding::RouteBinding>,
14904    }
14905
14906    async fn route_closed_fixture() -> RouteClosedFixture {
14907        let registry = Arc::new(Registry::default());
14908        let handler = ControlHandler::new(Arc::clone(&registry));
14909        #[cfg(unix)]
14910        let supervisor = Supervisor::new(registry, RestartPolicy::default())
14911            .with_forwarding(Arc::clone(&handler.forwarding));
14912        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14913        register_capability_manifest(
14914            &handler,
14915            &module_ctx,
14916            &mut module_rx,
14917            capability_manifest("prov", &[], &[]),
14918            1,
14919        )
14920        .await;
14921        let (client_rx, bind) = open_route_for_capability_test(
14922            &handler,
14923            &module_ctx,
14924            &mut module_rx,
14925            702,
14926            2,
14927            "prov",
14928            None,
14929        )
14930        .await;
14931        let ModuleControlRequest::RouteBind {
14932            route_channel: channel,
14933            epoch: route_epoch,
14934            ..
14935        } = bind
14936        else {
14937            panic!("expected route.bind");
14938        };
14939        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14940            handler
14941                .forwarding
14942                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14943                .unwrap()
14944        else {
14945            panic!("the acknowledged route must be bound");
14946        };
14947        RouteClosedFixture {
14948            handler,
14949            #[cfg(unix)]
14950            supervisor,
14951            module_ctx,
14952            #[cfg(unix)]
14953            module_rx,
14954            client_rx,
14955            route,
14956        }
14957    }
14958
14959    impl RouteClosedFixture {
14960        #[cfg(unix)]
14961        async fn cut(&mut self) {
14962            let notices = async {
14963                let mut draining = self.module_rx.recv().await.unwrap();
14964                assert!(matches!(
14965                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14966                        &draining.body
14967                    )
14968                    .unwrap(),
14969                    subc_protocol::session::ModuleControlCommand::Draining {
14970                        reason: RouteCloseReason::Restart,
14971                        ..
14972                    }
14973                ));
14974                draining.flushed.take().unwrap().send(()).unwrap();
14975                let mut closing = self.client_rx.recv().await.unwrap();
14976                assert!(matches!(
14977                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14978                    ClientControlPush::RouteClosing {
14979                        reason: RouteCloseReason::Restart,
14980                        ..
14981                    }
14982                ));
14983                closing.flushed.take().unwrap().send(()).unwrap();
14984            };
14985            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14986            result.unwrap();
14987        }
14988
14989        fn disconnect(&mut self) -> ClientControlPush {
14990            self.handler
14991                .cleanup_connection(self.module_ctx.connection_id)
14992                .unwrap();
14993            let closed = self.client_rx.try_recv().unwrap();
14994            assert_eq!(closed.header.ty, FrameType::Push);
14995            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14996            assert!(matches!(
14997                &push,
14998                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14999                    if channels == &[self.route.client_channel]
15000            ));
15001            assert_eq!(
15002                self.client_rx.try_recv().unwrap().header.ty,
15003                FrameType::Goodbye
15004            );
15005            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15006            push
15007        }
15008    }
15009
15010    #[cfg(unix)]
15011    #[tokio::test(start_paused = true)]
15012    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15013        let mut fixture = route_closed_fixture().await;
15014        fixture.cut().await;
15015        assert!(matches!(
15016            fixture.disconnect(),
15017            ClientControlPush::RouteClosed {
15018                reason: RouteCloseReason::Restart,
15019                drained: true,
15020                ..
15021            }
15022        ));
15023    }
15024
15025    #[cfg(unix)]
15026    #[tokio::test(start_paused = true)]
15027    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
15028        let mut fixture = route_closed_fixture().await;
15029        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
15030        let started = tokio::time::Instant::now();
15031        fixture.cut().await;
15032        assert!(started.elapsed() >= Duration::from_secs(2));
15033        assert_eq!(fixture.route.flow.in_flight(), 1);
15034        assert!(matches!(
15035            fixture.disconnect(),
15036            ClientControlPush::RouteClosed {
15037                reason: RouteCloseReason::Restart,
15038                drained: false,
15039                ..
15040            }
15041        ));
15042    }
15043
15044    #[tokio::test]
15045    async fn module_connection_crash_reports_crash_and_not_drained() {
15046        let mut fixture = route_closed_fixture().await;
15047        // The server calls cleanup_connection on module EOF. Without a planned
15048        // daemon shutdown, that disconnect must remain a crash with no prior
15049        // route.closing notice, even when the route has no in-flight calls.
15050        assert!(matches!(
15051            fixture.disconnect(),
15052            ClientControlPush::RouteClosed {
15053                reason: RouteCloseReason::Crash,
15054                drained: false,
15055                ..
15056            }
15057        ));
15058    }
15059
15060    #[tokio::test]
15061    async fn dropping_router_connection_releases_registration() {
15062        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15063        let registry = Arc::new(Registry::default());
15064        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
15065        let router = Router::with_control_handler(Arc::clone(&control));
15066        let connection = router.begin_connection();
15067        let connection_id = connection.id();
15068        let (ctx, mut rx) = route_ctx(connection_id);
15069
15070        router
15071            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15072            .await
15073            .unwrap();
15074        let response = rx.recv().await.unwrap();
15075        let ack = parse_ack(&response);
15076        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15077        assert!(registry.get_module("aft").unwrap().is_some());
15078
15079        drop(connection);
15080
15081        assert!(registry.get_module("aft").unwrap().is_none());
15082        assert_eq!(registry.active_registration_count().unwrap(), 0);
15083
15084        control
15085            .cleanup_connection(ConnectionId::new(u64::MAX))
15086            .unwrap();
15087        let captured = crate::router::test_log::captured_logs(&logs);
15088        println!("captured registry lifecycle logs:\n{captured}");
15089        let events: Vec<_> = captured
15090            .lines()
15091            .filter(|line| {
15092                line.contains("module registered module_id=aft ")
15093                    || line.contains("module registration ended")
15094            })
15095            .collect();
15096        assert_eq!(
15097            events.len(),
15098            2,
15099            "unexpected registry lifecycle logs: {captured}"
15100        );
15101        assert!(events[0].contains("module registered module_id=aft "));
15102        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15103        assert!(events[1].contains(&format!(
15104            "module_id=aft connection_id={}",
15105            connection_id.get()
15106        )));
15107        assert!(events[1].contains("reason=connection_closed"));
15108        assert!(events[1].contains("module registration ended"));
15109        assert_eq!(
15110            captured
15111                .lines()
15112                .filter(|line| line.contains("module registered module_id=aft "))
15113                .count(),
15114            1,
15115            "legacy registration admission line must appear once: {captured}"
15116        );
15117    }
15118
15119    #[tokio::test]
15120    async fn hello_registration_keeps_legacy_module_registered_line_once() {
15121        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15122        let registry = Arc::new(Registry::default());
15123        let control = ControlHandler::new(Arc::clone(&registry));
15124        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15125        hello_via_sink(
15126            &control,
15127            &ctx,
15128            &mut rx,
15129            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15130        )
15131        .await;
15132
15133        let captured = crate::router::test_log::captured_logs(&logs);
15134        let admissions: Vec<_> = captured
15135            .lines()
15136            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15137            .collect();
15138        assert_eq!(
15139            admissions.len(),
15140            1,
15141            "legacy admission line must remain exactly once: {captured}"
15142        );
15143        assert!(admissions[0].contains("routable_provider=true"));
15144        assert!(admissions[0].contains("connection_id=777"));
15145    }
15146
15147    fn capability_manifest(
15148        module_id: &str,
15149        provides: &[&str],
15150        must_never_reach: &[&str],
15151    ) -> ModuleManifest {
15152        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15153        manifest.capabilities = Some(CapabilityDeclarations {
15154            provides: provides
15155                .iter()
15156                .map(|capability| (*capability).to_string())
15157                .collect(),
15158            requires: Vec::new(),
15159            must_never_reach: must_never_reach
15160                .iter()
15161                .map(|capability| (*capability).to_string())
15162                .collect(),
15163        });
15164        manifest
15165    }
15166
15167    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15168        Frame::build(
15169            FrameType::Hello,
15170            control_flags(),
15171            0,
15172            0,
15173            corr,
15174            serde_json::to_vec(&ModuleHelloBody {
15175                protocol_ver: manifest.protocol_ver,
15176                manifest,
15177                control_ops: None,
15178                launch_nonce: None,
15179            })
15180            .expect("capability test HELLO serializes"),
15181        )
15182        .expect("capability test HELLO frame builds")
15183    }
15184
15185    fn catalog_update_with_capabilities_frame(
15186        corr: u64,
15187        capabilities: CapabilityDeclarations,
15188    ) -> Frame {
15189        Frame::build(
15190            FrameType::Request,
15191            control_flags(),
15192            0,
15193            0,
15194            corr,
15195            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15196                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15197                capabilities: Some(capabilities),
15198                ready: None,
15199            })
15200            .expect("capability catalog.update serializes"),
15201        )
15202        .expect("capability catalog.update frame builds")
15203    }
15204
15205    async fn register_capability_manifest(
15206        handler: &ControlHandler,
15207        ctx: &RouteCtx,
15208        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15209        manifest: ModuleManifest,
15210        corr: u64,
15211    ) {
15212        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15213    }
15214
15215    async fn open_route_for_capability_test(
15216        handler: &ControlHandler,
15217        target_ctx: &RouteCtx,
15218        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15219        client_connection_id: u64,
15220        corr: u64,
15221        target_module_id: &str,
15222        consumer_identity: Option<ConsumerIdentity>,
15223    ) -> (
15224        mpsc::Receiver<crate::router::OutboundFrame>,
15225        ModuleControlRequest,
15226    ) {
15227        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15228        let route_handler = handler.clone();
15229        let target_module_id = target_module_id.to_string();
15230        let route_task = tokio::spawn(async move {
15231            route_handler
15232                .handle_control_frame(
15233                    &client_ctx,
15234                    route_open_frame_with_admission_facts(
15235                        corr,
15236                        &target_module_id,
15237                        unique_project_root("admission-facts"),
15238                        consumer_identity,
15239                        None,
15240                    ),
15241                )
15242                .await
15243                .expect("capability test route.open succeeds")
15244        });
15245        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15246            .await
15247            .expect("capability test route.open must reach route.bind")
15248            .expect("target control receiver stays open");
15249        let bind_request: ModuleControlRequest =
15250            serde_json::from_slice(&bind.body).expect("route.bind decodes");
15251        handler
15252            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15253            .await
15254            .expect("capability test route.bind ACK succeeds");
15255        assert!(route_task.await.expect("route.open task joins").is_empty());
15256        let opened = client_rx
15257            .recv()
15258            .await
15259            .expect("successful route.open publishes a response");
15260        assert!(matches!(
15261            serde_json::from_slice::<ClientControlResponse>(&opened.body),
15262            Ok(ClientControlResponse::RouteOpen { .. })
15263        ));
15264        (client_rx, bind_request)
15265    }
15266
15267    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15268        assert_eq!(frame.header.ty, FrameType::Push);
15269        assert_eq!(frame.header.channel, 0);
15270        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15271            .expect("route.closed control push decodes");
15272        let ClientControlPush::RouteClosed { channels, .. } = &push else {
15273            panic!("expected route.closed");
15274        };
15275        assert_eq!(channels.len(), 1, "exactly one violating route closed");
15276        let closed_channel = channels[0];
15277        let channels = channels.clone();
15278        assert_eq!(
15279            push,
15280            ClientControlPush::RouteClosed {
15281                module_id: target_module_id.to_string(),
15282                channels,
15283                reason: RouteCloseReason::CapabilityDenied,
15284                drained: false,
15285                abandoned: 0,
15286                excluded_subscriptions: 0,
15287                terminal: Some(false),
15288            }
15289        );
15290        closed_channel
15291    }
15292
15293    #[tokio::test]
15294    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15295        let registry = Arc::new(Registry::default());
15296        let forwarding = Arc::new(ForwardingTable::default());
15297        let supervisor = SupervisorHandle::new();
15298        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15299        let handler =
15300            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15301                .with_supervisor(supervisor);
15302        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15303        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15304        register_capability_manifest(
15305            &handler,
15306            &target_ctx,
15307            &mut target_rx,
15308            capability_manifest("target", &["credentials-provider/v1"], &[]),
15309            1,
15310        )
15311        .await;
15312        register_capability_manifest(
15313            &handler,
15314            &opener_ctx,
15315            &mut opener_rx,
15316            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15317            2,
15318        )
15319        .await;
15320
15321        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15322        let replies = handler
15323            .handle_control_frame(
15324                &client_ctx,
15325                route_open_frame_with_admission_facts(
15326                    3,
15327                    "target",
15328                    unique_project_root("admission-facts"),
15329                    Some(ConsumerIdentity {
15330                        module_id: "opener".to_string(),
15331                        launch_nonce: "opener-nonce".to_string(),
15332                    }),
15333                    None,
15334                ),
15335            )
15336            .await
15337            .expect("denied route.open returns a typed frame");
15338        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15339        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15340        assert!(
15341            target_rx.try_recv().is_err(),
15342            "forbidden route.open must not relay route.bind"
15343        );
15344    }
15345
15346    #[tokio::test]
15347    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15348        let registry = Arc::new(Registry::default());
15349        let forwarding = Arc::new(ForwardingTable::default());
15350        let supervisor = SupervisorHandle::new();
15351        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15352        let handler =
15353            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15354                .with_supervisor(supervisor);
15355        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15356        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15357        register_capability_manifest(
15358            &handler,
15359            &target_ctx,
15360            &mut target_rx,
15361            capability_manifest("target", &["credentials-provider/v1"], &[]),
15362            1,
15363        )
15364        .await;
15365        register_capability_manifest(
15366            &handler,
15367            &old_opener_ctx,
15368            &mut old_opener_rx,
15369            capability_manifest("opener", &[], &[]),
15370            2,
15371        )
15372        .await;
15373        let (mut client_rx, _) = open_route_for_capability_test(
15374            &handler,
15375            &target_ctx,
15376            &mut target_rx,
15377            712,
15378            3,
15379            "target",
15380            Some(ConsumerIdentity {
15381                module_id: "opener".to_string(),
15382                launch_nonce: "opener-nonce".to_string(),
15383            }),
15384        )
15385        .await;
15386        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15387
15388        handler
15389            .cleanup_connection(old_opener_ctx.connection_id)
15390            .expect("old opener registration cleans up");
15391        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15392        register_capability_manifest(
15393            &handler,
15394            &new_opener_ctx,
15395            &mut new_opener_rx,
15396            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15397            4,
15398        )
15399        .await;
15400
15401        let closed_channel = assert_capability_denied_push(
15402            client_rx
15403                .try_recv()
15404                .expect("HELLO deny addition must emit route.closed")
15405                .frame,
15406            "target",
15407        );
15408        let goodbye = client_rx
15409            .try_recv()
15410            .expect("HELLO deny addition must send the client a route GOODBYE")
15411            .frame;
15412        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15413        assert_eq!(goodbye.header.channel, closed_channel);
15414        assert_ne!(goodbye.header.channel, 0);
15415        assert_ne!(goodbye.header.epoch, 0);
15416        assert!(client_rx.try_recv().is_err());
15417        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15418        assert!(matches!(
15419            target_rx.try_recv(),
15420            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15421        ));
15422    }
15423
15424    #[tokio::test]
15425    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15426        let registry = Arc::new(Registry::default());
15427        let forwarding = Arc::new(ForwardingTable::default());
15428        let supervisor = SupervisorHandle::new();
15429        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15430        let handler =
15431            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15432                .with_supervisor(supervisor);
15433        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15434        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15435        register_capability_manifest(
15436            &handler,
15437            &target_ctx,
15438            &mut target_rx,
15439            capability_manifest("target", &[], &[]),
15440            1,
15441        )
15442        .await;
15443        register_capability_manifest(
15444            &handler,
15445            &opener_ctx,
15446            &mut opener_rx,
15447            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15448            2,
15449        )
15450        .await;
15451        let (mut client_rx, _) = open_route_for_capability_test(
15452            &handler,
15453            &target_ctx,
15454            &mut target_rx,
15455            722,
15456            3,
15457            "target",
15458            Some(ConsumerIdentity {
15459                module_id: "opener".to_string(),
15460                launch_nonce: "opener-nonce".to_string(),
15461            }),
15462        )
15463        .await;
15464        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15465
15466        let replies = handler
15467            .handle_control_frame(
15468                &target_ctx,
15469                catalog_update_with_capabilities_frame(
15470                    4,
15471                    CapabilityDeclarations {
15472                        provides: vec!["credentials-provider/v1".to_string()],
15473                        requires: Vec::new(),
15474                        must_never_reach: Vec::new(),
15475                    },
15476                ),
15477            )
15478            .await
15479            .expect("claim catalog.update succeeds");
15480        assert!(matches!(
15481            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15482            Ok(ModuleControlResponseToModule::CatalogUpdate {})
15483        ));
15484        let closed_channel = assert_capability_denied_push(
15485            client_rx
15486                .try_recv()
15487                .expect("claim addition must emit route.closed")
15488                .frame,
15489            "target",
15490        );
15491        let goodbye = client_rx
15492            .try_recv()
15493            .expect("claim addition must send the client a route GOODBYE")
15494            .frame;
15495        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15496        assert_eq!(goodbye.header.channel, closed_channel);
15497        assert_ne!(goodbye.header.channel, 0);
15498        assert_ne!(goodbye.header.epoch, 0);
15499        assert!(client_rx.try_recv().is_err());
15500        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15501        assert!(matches!(
15502            target_rx.try_recv(),
15503            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15504        ));
15505    }
15506
15507    #[tokio::test]
15508    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15509        let registry = Arc::new(Registry::default());
15510        let forwarding = Arc::new(ForwardingTable::default());
15511        let supervisor = SupervisorHandle::new();
15512        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15513        let handler =
15514            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15515                .with_supervisor(supervisor);
15516        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15517        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15518        register_capability_manifest(
15519            &handler,
15520            &target_ctx,
15521            &mut target_rx,
15522            capability_manifest("target", &["credentials-provider/v1"], &[]),
15523            1,
15524        )
15525        .await;
15526        register_capability_manifest(
15527            &handler,
15528            &opener_ctx,
15529            &mut opener_rx,
15530            capability_manifest("opener", &[], &[]),
15531            2,
15532        )
15533        .await;
15534        let (mut client_rx, _) = open_route_for_capability_test(
15535            &handler,
15536            &target_ctx,
15537            &mut target_rx,
15538            732,
15539            3,
15540            "target",
15541            Some(ConsumerIdentity {
15542                module_id: "opener".to_string(),
15543                launch_nonce: "opener-nonce".to_string(),
15544            }),
15545        )
15546        .await;
15547        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15548
15549        handler
15550            .handle_control_frame(
15551                &target_ctx,
15552                catalog_update_with_capabilities_frame(
15553                    4,
15554                    CapabilityDeclarations {
15555                        provides: Vec::new(),
15556                        requires: Vec::new(),
15557                        must_never_reach: Vec::new(),
15558                    },
15559                ),
15560            )
15561            .await
15562            .expect("claim removal catalog.update succeeds");
15563        assert_eq!(
15564            forwarding.active_binding_count().unwrap(),
15565            1,
15566            "removing an attested target claim must leave the route census unchanged"
15567        );
15568        assert!(
15569            client_rx.try_recv().is_err(),
15570            "claim removal must not emit route.closed or a client route GOODBYE"
15571        );
15572        assert!(
15573            target_rx.try_recv().is_err(),
15574            "claim removal must not send the target a route GOODBYE"
15575        );
15576    }
15577
15578    /// A direct client may open a route to a denied capability provider; this
15579    /// policy applies only to attested supervised module origins, not to direct clients.
15580    #[tokio::test]
15581    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15582        let registry = Arc::new(Registry::default());
15583        let forwarding = Arc::new(ForwardingTable::default());
15584        let supervisor = SupervisorHandle::new();
15585        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15586        let handler =
15587            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15588                .with_supervisor(supervisor);
15589        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15590        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15591        register_capability_manifest(
15592            &handler,
15593            &target_ctx,
15594            &mut target_rx,
15595            capability_manifest("target", &["credentials-provider/v1"], &[]),
15596            1,
15597        )
15598        .await;
15599        register_capability_manifest(
15600            &handler,
15601            &opener_ctx,
15602            &mut opener_rx,
15603            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15604            2,
15605        )
15606        .await;
15607
15608        let (_client_rx, bind) = open_route_for_capability_test(
15609            &handler,
15610            &target_ctx,
15611            &mut target_rx,
15612            742,
15613            3,
15614            "target",
15615            None,
15616        )
15617        .await;
15618        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15619            panic!("direct scope-honesty route must bind");
15620        };
15621        assert_eq!(principal, Some(Principal::Direct));
15622        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15623    }
15624
15625    /// A module that denies a capability receives no self-route exemption when it
15626    /// also attestedly provides that capability.
15627    #[tokio::test]
15628    async fn must_never_reach_self_route_is_capability_forbidden() {
15629        let registry = Arc::new(Registry::default());
15630        let forwarding = Arc::new(ForwardingTable::default());
15631        let supervisor = SupervisorHandle::new();
15632        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15633        let handler =
15634            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15635                .with_supervisor(supervisor);
15636        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15637        register_capability_manifest(
15638            &handler,
15639            &self_ctx,
15640            &mut self_rx,
15641            capability_manifest(
15642                "self-provider",
15643                &["credentials-provider/v1"],
15644                &["credentials-provider/v1"],
15645            ),
15646            1,
15647        )
15648        .await;
15649
15650        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15651        let replies = handler
15652            .handle_control_frame(
15653                &client_ctx,
15654                route_open_frame_with_admission_facts(
15655                    2,
15656                    "self-provider",
15657                    unique_project_root("admission-facts"),
15658                    Some(ConsumerIdentity {
15659                        module_id: "self-provider".to_string(),
15660                        launch_nonce: "self-nonce".to_string(),
15661                    }),
15662                    None,
15663                ),
15664            )
15665            .await
15666            .expect("self-route refusal returns a typed frame");
15667        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15668        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15669        assert!(
15670            self_rx.try_recv().is_err(),
15671            "self denial must not relay route.bind"
15672        );
15673    }
15674
15675    #[test]
15676    fn unsupported_channel_zero_frame_returns_error() {
15677        let handler = ControlHandler::default();
15678        let request = Frame::build(
15679            FrameType::Request,
15680            control_flags(),
15681            0,
15682            0,
15683            21,
15684            b"opaque".to_vec(),
15685        )
15686        .unwrap();
15687
15688        let response = handler
15689            .handle_control(ConnectionId::new(1), request)
15690            .unwrap();
15691
15692        assert_eq!(response[0].header.ty, FrameType::Error);
15693        assert_eq!(
15694            parse_error(&response[0])["code"],
15695            "unsupported_control_frame"
15696        );
15697    }
15698
15699    /// Blue/green swap at the control-plane boundary. The supervisor that opens
15700    /// a swap is not wired yet, so the candidate is registered here directly
15701    /// into the registry and forwarding candidate slots, the way the swap's
15702    /// HELLO admission will.
15703    mod swap {
15704        use super::*;
15705
15706        const INCUMBENT: ConnectionId = ConnectionId::new(30);
15707        const CANDIDATE: ConnectionId = ConnectionId::new(40);
15708
15709        struct Swap {
15710            registry: Arc<Registry>,
15711            forwarding: Arc<ForwardingTable>,
15712            handler: ControlHandler,
15713            incumbent_ctx: RouteCtx,
15714            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15715            candidate_ctx: RouteCtx,
15716            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15717        }
15718
15719        async fn swap_with_incumbent() -> Swap {
15720            let registry = Arc::new(Registry::default());
15721            let forwarding = Arc::new(ForwardingTable::default());
15722            let handler =
15723                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15724            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15725            hello_via_sink(
15726                &handler,
15727                &incumbent_ctx,
15728                &mut incumbent_rx,
15729                hello_frame("aft", PROTOCOL_VERSION, 7),
15730            )
15731            .await;
15732            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15733            Swap {
15734                registry,
15735                forwarding,
15736                handler,
15737                incumbent_ctx,
15738                incumbent_rx,
15739                candidate_ctx,
15740                candidate_rx,
15741            }
15742        }
15743
15744        fn register_candidate(swap: &Swap, ready: Option<bool>) {
15745            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15746            candidate_manifest.ready = ready;
15747            let registration = swap
15748                .registry
15749                .register_candidate_with_control_ops(
15750                    candidate_manifest,
15751                    PROTOCOL_VERSION,
15752                    CANDIDATE,
15753                    module_baseline_control_ops(),
15754                )
15755                .unwrap();
15756            swap.forwarding
15757                .register_candidate_module_connection(
15758                    CANDIDATE,
15759                    "aft".to_string(),
15760                    PROTOCOL_VERSION,
15761                    manifest_concurrency(&registration.manifest),
15762                    swap.candidate_ctx.egress.clone(),
15763                )
15764                .unwrap();
15765        }
15766
15767        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15768            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15769            swap.registry.promote_candidate("aft").unwrap().unwrap();
15770            cutover.incumbent.unwrap()
15771        }
15772
15773        /// A swap candidate's HELLO reserves its candidate registry entry
15774        /// before installing its candidate forwarding endpoint. The swap
15775        /// supervisor promotes a ready candidate as soon as it reads it as
15776        /// registered, and promotion cuts forwarding over to the candidate
15777        /// endpoint, so the lookup the supervisor reads
15778        /// (`Registry::published_registration`) must not return the candidate
15779        /// until that endpoint exists. Otherwise the cutover finds no candidate
15780        /// endpoint and the swap fails as if the candidate had disconnected.
15781        #[tokio::test]
15782        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15783            let mut swap = swap_with_incumbent().await;
15784            let hold = Arc::new(std::sync::Barrier::new(2));
15785            let handler = swap
15786                .handler
15787                .clone()
15788                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15789            handler
15790                .supervisor
15791                .open_swap("aft", "candidate-nonce".to_string());
15792            let hello = {
15793                let handler = handler.clone();
15794                let candidate_ctx = swap.candidate_ctx.clone();
15795                std::thread::spawn(move || {
15796                    handler
15797                        .handle_hello(
15798                            candidate_ctx.connection_id,
15799                            Some(candidate_ctx.egress.clone()),
15800                            hello_frame_with_nonce(
15801                                "aft",
15802                                PROTOCOL_VERSION,
15803                                8,
15804                                Some("candidate-nonce"),
15805                            ),
15806                        )
15807                        .unwrap()
15808                })
15809            };
15810
15811            hold.wait();
15812            // Parked: the candidate slot is claimed but not published, and
15813            // forwarding has no candidate endpoint.
15814            assert!(swap
15815                .registry
15816                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15817                .unwrap()
15818                .is_some());
15819            assert!(swap
15820                .registry
15821                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15822                .unwrap()
15823                .is_none());
15824            assert!(swap
15825                .forwarding
15826                .module_endpoint_for_connection(CANDIDATE)
15827                .unwrap()
15828                .is_none());
15829
15830            hold.wait();
15831            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15832            assert_eq!(
15833                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15834                FrameType::HelloAck
15835            );
15836            let published = swap
15837                .registry
15838                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15839                .unwrap()
15840                .expect("candidate is published once its HELLO completes");
15841            assert_eq!(published.connection_id, CANDIDATE);
15842            assert!(swap
15843                .forwarding
15844                .module_endpoint_for_connection(CANDIDATE)
15845                .unwrap()
15846                .is_some());
15847        }
15848
15849        fn keyed_total(counters: &Value, key: &str) -> u64 {
15850            counters[key]
15851                .as_object()
15852                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15853                .unwrap_or(0)
15854        }
15855
15856        #[tokio::test]
15857        async fn replacement_logs_old_end_and_new_admission_once() {
15858            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15859            let mut swap = swap_with_incumbent().await;
15860            swap.handler
15861                .supervisor
15862                .open_swap("aft", "candidate-nonce".to_string());
15863            hello_via_sink(
15864                &swap.handler,
15865                &swap.candidate_ctx,
15866                &mut swap.candidate_rx,
15867                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15868            )
15869            .await;
15870            cutover(&swap);
15871            swap.handler.cleanup_connection(INCUMBENT).unwrap();
15872
15873            let captured = crate::router::test_log::captured_logs(&logs);
15874            println!("captured replacement registry logs:\n{captured}");
15875            let new_admissions: Vec<_> = captured
15876                .lines()
15877                .filter(|line| {
15878                    line.contains("connection_id=40")
15879                        && line.contains("swap candidate registered; not routable until cutover")
15880                })
15881                .collect();
15882            assert_eq!(
15883                new_admissions.len(),
15884                1,
15885                "unexpected admission logs: {captured}"
15886            );
15887            assert!(new_admissions[0].contains("module_id=aft"));
15888            assert!(new_admissions[0].contains("connection_id=40"));
15889            assert!(
15890                new_admissions[0].contains("swap candidate registered; not routable until cutover")
15891            );
15892            assert!(new_admissions[0].contains("ready=true"));
15893
15894            let old_admissions: Vec<_> = captured
15895                .lines()
15896                .filter(|line| {
15897                    line.contains("module registered module_id=aft ")
15898                        && line.contains("connection_id=30")
15899                })
15900                .collect();
15901            assert_eq!(
15902                old_admissions.len(),
15903                1,
15904                "unexpected incumbent admission: {captured}"
15905            );
15906
15907            let old_ends: Vec<_> = captured
15908                .lines()
15909                .filter(|line| {
15910                    line.contains("connection_id=30") && line.contains("module registration ended")
15911                })
15912                .collect();
15913            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15914            assert!(old_ends[0].contains("module_id=aft"));
15915            assert!(old_ends[0].contains("connection_id=30"));
15916            assert!(old_ends[0].contains("reason=replaced"));
15917            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15918            assert_eq!(
15919                captured
15920                    .lines()
15921                    .filter(|line| line.contains("module registration promoted"))
15922                    .count(),
15923                1,
15924                "unexpected promotion logs: {captured}"
15925            );
15926        }
15927
15928        /// An ack from the incumbent for a bind it was sent before cutover,
15929        /// arriving before the incumbent is drained. The incumbent is the live
15930        /// connection carrying every other client's routes, so the ack must
15931        /// not end it: the waiting client is told to retry, the reservation is
15932        /// given back, and the incumbent is told to drop just that binding.
15933        #[tokio::test]
15934        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15935            let mut swap = swap_with_incumbent().await;
15936            let handler = swap.handler.clone();
15937
15938            // A co-tenant route, bound on the incumbent before the swap.
15939            let cotenant = ConnectionId::new(31);
15940            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15941            let (cotenant_task, cotenant_bind) = relay_route_open(
15942                &handler,
15943                cotenant,
15944                &cotenant_ctx.egress,
15945                &mut swap.incumbent_rx,
15946                100,
15947                "aft",
15948                "swap-cotenant",
15949            )
15950            .await;
15951            handler
15952                .handle_control_frame(
15953                    &swap.incumbent_ctx,
15954                    route_bind_ack(cotenant_bind.header.corr),
15955                )
15956                .await
15957                .unwrap();
15958            assert!(cotenant_task.await.unwrap().is_empty());
15959            let (cotenant_channel, cotenant_epoch) =
15960                published_route(&cotenant_rx.recv().await.unwrap());
15961
15962            // A second route.open, relayed to the incumbent and not yet acked.
15963            let caller = ConnectionId::new(32);
15964            let (caller_ctx, mut caller_rx) = route_ctx(caller);
15965            let (caller_task, caller_bind) = relay_route_open(
15966                &handler,
15967                caller,
15968                &caller_ctx.egress,
15969                &mut swap.incumbent_rx,
15970                101,
15971                "aft",
15972                "swap-caller",
15973            )
15974            .await;
15975            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15976
15977            register_candidate(&swap, None);
15978            cutover(&swap);
15979
15980            // The incumbent acks after promotion and before any drain.
15981            let ack = handler
15982                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15983                .await;
15984            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15985            if module_loop_error.is_some() {
15986                // What the connection loop does with an untranslated router
15987                // error: end the connection, releasing every route on it.
15988                handler.cleanup_connection(INCUMBENT).unwrap();
15989            }
15990
15991            // 1. The incumbent's other routes survive.
15992            assert!(
15993                cotenant_rx.try_recv().is_err(),
15994                "the co-tenant route on the incumbent was torn down by one late ack: \
15995                 {module_loop_error:?}"
15996            );
15997            assert!(matches!(
15998                swap.forwarding
15999                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16000                    .unwrap(),
16001                DataRoute::Client(DataRouteState::Bound(_))
16002            ));
16003            assert_eq!(module_loop_error, None);
16004            assert!(swap
16005                .registry
16006                .get_module_by_connection(INCUMBENT)
16007                .unwrap()
16008                .is_some());
16009
16010            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
16011            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16012                .await
16013                .expect("the incumbent is told to drop the abandoned binding")
16014                .unwrap()
16015                .frame;
16016            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16017            assert_eq!(goodbye.header.channel, abandoned_channel);
16018            assert_eq!(goodbye.header.epoch, abandoned_epoch);
16019            assert!(swap.incumbent_rx.try_recv().is_err());
16020
16021            // 3. The waiting client gets a retryable refusal and no route.
16022            let response = caller_task.await.unwrap();
16023            assert_eq!(response.len(), 1);
16024            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16025            assert!(caller_rx.try_recv().is_err());
16026
16027            // 4. The reservation pair is given back, and the pending bind
16028            //    settled exactly once: one accepted open (the co-tenant) and one
16029            //    refused open (the caller), nothing counted twice.
16030            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
16031            let counters = handler.counters().snapshot();
16032            assert_eq!(
16033                keyed_total(&counters, "route_open_accepted_by_principal"),
16034                1
16035            );
16036            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
16037            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
16038        }
16039
16040        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
16041        /// id would resolve to the promoted candidate and every new route.open
16042        /// would be refused as reloading, leaving neither process routable.
16043        #[tokio::test]
16044        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
16045            let mut swap = swap_with_incumbent().await;
16046            register_candidate(&swap, None);
16047            let incumbent = cutover(&swap);
16048            swap.forwarding
16049                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
16050                .unwrap()
16051                .expect("the incumbent is still registered");
16052
16053            let client = ConnectionId::new(33);
16054            let (client_ctx, mut client_rx) = route_ctx(client);
16055            let route_handler = swap.handler.clone();
16056            let open_ctx = RouteCtx {
16057                connection_id: client,
16058                egress: client_ctx.egress.clone(),
16059            };
16060            let mut route_task = tokio::spawn(async move {
16061                route_handler
16062                    .handle_control_frame(
16063                        &open_ctx,
16064                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16065                    )
16066                    .await
16067                    .unwrap()
16068            });
16069            let bind = tokio::select! {
16070                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16071                response = &mut route_task => {
16072                    let response = response.unwrap();
16073                    panic!(
16074                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
16075                        parse_error(&response[0])["code"]
16076                    );
16077                }
16078            };
16079            swap.handler
16080                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16081                .await
16082                .unwrap();
16083            assert!(route_task.await.unwrap().is_empty());
16084            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16085            match swap
16086                .forwarding
16087                .lookup_data_route(client, channel, epoch)
16088                .unwrap()
16089            {
16090                DataRoute::Client(DataRouteState::Bound(route)) => {
16091                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16092                }
16093                other => panic!("expected a bound route on the candidate, got {other:?}"),
16094            }
16095            assert!(swap.incumbent_rx.try_recv().is_err());
16096        }
16097
16098        /// A candidate declares itself ready with `catalog.update` on its own
16099        /// connection. If the connection-keyed registry lookups searched only the
16100        /// active slot, this would answer `not_registered` and the candidate
16101        /// would never become ready.
16102        #[tokio::test]
16103        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16104            let swap = swap_with_incumbent().await;
16105            register_candidate(&swap, Some(false));
16106            let update = Frame::build(
16107                FrameType::Request,
16108                control_flags(),
16109                0,
16110                0,
16111                55,
16112                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16113                    provides: manifest("aft", PROTOCOL_VERSION).provides,
16114                    capabilities: None,
16115                    ready: Some(true),
16116                })
16117                .unwrap(),
16118            )
16119            .unwrap();
16120
16121            let replies = swap
16122                .handler
16123                .handle_control_frame(&swap.candidate_ctx, update)
16124                .await
16125                .unwrap();
16126
16127            assert_eq!(replies.len(), 1);
16128            assert_eq!(
16129                replies[0].header.ty,
16130                FrameType::Response,
16131                "candidate catalog.update was refused: {:?}",
16132                serde_json::from_slice::<Value>(&replies[0].body).ok()
16133            );
16134            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16135            assert_eq!(
16136                swap.registry
16137                    .get_module("aft")
16138                    .unwrap()
16139                    .unwrap()
16140                    .connection_id,
16141                INCUMBENT
16142            );
16143        }
16144    }
16145
16146    /// The HELLO gate while the supervisor has a swap open: only the nonce it
16147    /// minted for the candidate admits a second process, into the candidate
16148    /// slot, and that check runs ahead of the reserved-module gate.
16149    mod swap_admission {
16150        use super::*;
16151
16152        const INCUMBENT_NONCE: &str = "incumbent-nonce";
16153        const CANDIDATE_NONCE: &str = "candidate-nonce";
16154
16155        fn handler_with_incumbent(
16156            module_id: &str,
16157            reserved: bool,
16158        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16159            let registry = Arc::new(Registry::default());
16160            let supervisor = SupervisorHandle::new();
16161            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16162            if reserved {
16163                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16164            }
16165            let handler =
16166                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
16167            let incumbent = handler
16168                .handle_control(
16169                    ConnectionId::new(1),
16170                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16171                )
16172                .unwrap();
16173            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16174            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16175            (registry, supervisor, handler)
16176        }
16177
16178        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
16179        /// admits every nonce, so while a swap is open the swap gate is the only
16180        /// thing between a key-holder and the candidate slot. A nonce the
16181        /// supervisor did not mint, or none at all, is refused, and neither the
16182        /// incumbent's registration nor the candidate slot moves.
16183        #[test]
16184        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16185            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16186
16187            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16188                let replies = handler
16189                    .handle_control(
16190                        ConnectionId::new(connection),
16191                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16192                    )
16193                    .unwrap();
16194                assert_eq!(replies[0].header.ty, FrameType::Error);
16195                assert_eq!(
16196                    parse_error(&replies[0])["code"],
16197                    "swap_token_invalid",
16198                    "nonce {nonce:?}"
16199                );
16200            }
16201            assert!(registry.get_candidate("aft").unwrap().is_none());
16202            assert_eq!(
16203                registry.get_module("aft").unwrap().unwrap().connection_id,
16204                ConnectionId::new(1)
16205            );
16206
16207            // Control: the minted token is admitted, into the candidate slot,
16208            // and only once.
16209            let admitted = handler
16210                .handle_control(
16211                    ConnectionId::new(4),
16212                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16213                )
16214                .unwrap();
16215            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16216            assert_eq!(
16217                registry
16218                    .get_candidate("aft")
16219                    .unwrap()
16220                    .unwrap()
16221                    .connection_id,
16222                ConnectionId::new(4)
16223            );
16224            assert_eq!(
16225                registry.get_module("aft").unwrap().unwrap().connection_id,
16226                ConnectionId::new(1),
16227                "the candidate must not take the active slot"
16228            );
16229            let replayed = handler
16230                .handle_control(
16231                    ConnectionId::new(5),
16232                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16233                )
16234                .unwrap();
16235            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16236
16237            // The case only this gate covers: the incumbent has died mid-swap,
16238            // so its duplicate refusal is gone too, and without the gate a
16239            // key-holder would take the id's ACTIVE slot.
16240            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16241            let squatter = handler
16242                .handle_control(
16243                    ConnectionId::new(6),
16244                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16245                )
16246                .unwrap();
16247            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16248            assert!(
16249                registry.get_module("aft").unwrap().is_none(),
16250                "a squatter took the active slot of an id being swapped"
16251            );
16252        }
16253
16254        /// Design mutation arm (iii). A reserved module's candidate presents a
16255        /// nonce the reserved gate has never seen (that gate holds the
16256        /// incumbent's), so the swap gate must run first or the candidate is
16257        /// refused `reserved_module` and a reserved module can never be swapped.
16258        #[test]
16259        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16260            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16261
16262            let replies = handler
16263                .handle_control(
16264                    ConnectionId::new(2),
16265                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16266                )
16267                .unwrap();
16268
16269            assert_eq!(
16270                replies[0].header.ty,
16271                FrameType::HelloAck,
16272                "reserved candidate refused: {:?}",
16273                serde_json::from_slice::<Value>(&replies[0].body).ok()
16274            );
16275            assert_eq!(
16276                registry
16277                    .get_candidate("vault")
16278                    .unwrap()
16279                    .unwrap()
16280                    .connection_id,
16281                ConnectionId::new(2)
16282            );
16283        }
16284
16285        /// With no swap open the gate is inert: the incumbent's reserved gate
16286        /// and duplicate refusal behave exactly as before.
16287        #[test]
16288        fn without_an_open_swap_the_ordinary_gates_decide() {
16289            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16290            supervisor.close_swap("vault");
16291
16292            let candidate = handler
16293                .handle_control(
16294                    ConnectionId::new(2),
16295                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16296                )
16297                .unwrap();
16298            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16299            let duplicate = handler
16300                .handle_control(
16301                    ConnectionId::new(3),
16302                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16303                )
16304                .unwrap();
16305            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16306            assert!(registry.get_candidate("vault").unwrap().is_none());
16307        }
16308    }
16309
16310    /// `scope.sync` and `scope.describe` through the real control handler: who
16311    /// may sync is decided by the registration and launch nonce of the module
16312    /// connection, never by the request body.
16313    mod scopes {
16314        mod idle_tests {
16315            use super::*;
16316            include!("scope_idle_tests.rs");
16317        }
16318        mod route_open_timings {
16319            use super::*;
16320            include!("route_open_timing_tests.rs");
16321        }
16322        mod route_open_refusal_log {
16323            use super::*;
16324            include!("route_open_refusal_log_tests.rs");
16325        }
16326        mod activity {
16327            include!("activity_scope_tests.rs");
16328        }
16329        mod delta {
16330            include!("scope_handler_delta_tests.rs");
16331        }
16332        mod call_modes {
16333            include!("scope_call_mode_tests.rs");
16334        }
16335        #[tokio::test]
16336        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16337            let rig = rig().await;
16338            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16339                "s".into(),
16340                1,
16341                "tool".into(),
16342            ))
16343            .unwrap();
16344            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16345            let (direct, _rx) = wide_ctx(999);
16346            for (ctx, expected) in [
16347                (&rig.owner, FrameType::Response),
16348                (&direct, FrameType::Error),
16349            ] {
16350                let frame =
16351                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16352                        .unwrap();
16353                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16354                assert_eq!(replies[0].header.ty, expected);
16355                if expected == FrameType::Response {
16356                    let reply: subc_control::ScopeServedByReply =
16357                        serde_json::from_slice(&replies[0].body).unwrap();
16358                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16359                } else {
16360                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16361                }
16362            }
16363            let reply = call(
16364                &rig.handler,
16365                &rig.owner,
16366                &ModuleControlRequestFromModule::ScopeDescribe {
16367                    owner: Principal::Reserved {
16368                        module_id: OWNER.into(),
16369                    },
16370                    scope_ref: "s".into(),
16371                },
16372            )
16373            .await;
16374            assert!(matches!(
16375                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16376                ModuleControlResponseToModule::ScopeDescribe { .. }
16377            ));
16378        }
16379
16380        use subc_protocol::scope::{
16381            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16382            ScopeStatus,
16383        };
16384
16385        use super::*;
16386
16387        const OWNER: &str = "prefrontal-core";
16388
16389        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16390            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16391        }
16392
16393        async fn call(
16394            handler: &ControlHandler,
16395            ctx: &RouteCtx,
16396            request: &ModuleControlRequestFromModule,
16397        ) -> Frame {
16398            let body = serde_json::to_vec(request).unwrap();
16399            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16400            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16401            assert_eq!(replies.len(), 1, "{replies:?}");
16402            replies.pop().unwrap()
16403        }
16404
16405        async fn sync(
16406            handler: &ControlHandler,
16407            ctx: &RouteCtx,
16408            generation: u64,
16409            scopes: Vec<ScopeRecord>,
16410        ) -> Result<ModuleControlResponseToModule, String> {
16411            let reply = call(
16412                handler,
16413                ctx,
16414                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16415            )
16416            .await;
16417            match reply.header.ty {
16418                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16419                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16420            }
16421        }
16422
16423        async fn describe(
16424            handler: &ControlHandler,
16425            ctx: &RouteCtx,
16426            owner: &str,
16427            scope_ref: &str,
16428        ) -> ModuleControlResponseToModule {
16429            let reply = call(
16430                handler,
16431                ctx,
16432                &ModuleControlRequestFromModule::ScopeDescribe {
16433                    owner: Principal::Reserved {
16434                        module_id: owner.to_string(),
16435                    },
16436                    scope_ref: scope_ref.to_string(),
16437                },
16438            )
16439            .await;
16440            assert_eq!(
16441                reply.header.ty,
16442                FrameType::Response,
16443                "{:?}",
16444                parse_error(&reply)
16445            );
16446            serde_json::from_slice(&reply.body).unwrap()
16447        }
16448
16449        #[tokio::test]
16450        async fn scope_describe_never_echoes_bind_proof_or_hash() {
16451            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16452            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16453            let mut rig = rig().await;
16454            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16455                .await;
16456            let _bound = rig
16457                .bound(
16458                    Some(AFT),
16459                    PLEXUS,
16460                    Some(
16461                        rig.selector("s", Some(1))
16462                            .with_bind_proof(Some(PROOF.into())),
16463                    ),
16464                )
16465                .await;
16466            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16467            let output = serde_json::to_string(&reply).unwrap();
16468            assert!(output.contains("\"status\":\"live\""));
16469            assert!(!output.contains(PROOF));
16470            assert!(!output.contains(HASH));
16471        }
16472
16473        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
16474        async fn module(
16475            handler: &ControlHandler,
16476            connection: u64,
16477            module_id: &str,
16478            nonce: Option<&str>,
16479        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16480            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16481            hello_via_sink(
16482                handler,
16483                &ctx,
16484                &mut rx,
16485                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16486            )
16487            .await;
16488            (ctx, rx)
16489        }
16490
16491        /// `direct` and every other client connection has no registration, so
16492        /// it can neither sync nor own a scope.
16493        #[tokio::test]
16494        async fn a_client_connection_cannot_sync_or_describe() {
16495            let handler = ControlHandler::new(Arc::new(Registry::default()));
16496            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16497            for request in [
16498                ModuleControlRequestFromModule::ScopeSync {
16499                    generation: 1,
16500                    scopes: vec![head("s", 1)],
16501                },
16502                ModuleControlRequestFromModule::ScopeDescribe {
16503                    owner: Principal::Direct,
16504                    scope_ref: "s".to_string(),
16505                },
16506            ] {
16507                let reply = call(&handler, &ctx, &request).await;
16508                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16509            }
16510            assert!(
16511                !handler
16512                    .scopes
16513                    .read()
16514                    .unwrap()
16515                    .describe(
16516                        &Principal::Reserved {
16517                            module_id: OWNER.to_string()
16518                        },
16519                        "s"
16520                    )
16521                    .owner_synced
16522            );
16523        }
16524
16525        /// A module the supervisor did not spawn registers without a launch
16526        /// nonce, so it is never an owner's current launch.
16527        #[tokio::test]
16528        async fn a_module_without_a_supervised_launch_cannot_sync() {
16529            let handler = ControlHandler::new(Arc::new(Registry::default()));
16530            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16531            assert_eq!(
16532                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16533                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16534            );
16535        }
16536
16537        #[tokio::test]
16538        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16539            let supervisor = SupervisorHandle::new();
16540            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16541            let handler = ControlHandler::new(Arc::new(Registry::default()))
16542                .with_supervisor(supervisor.clone());
16543            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16544            sync(&handler, &incumbent, 1, vec![head("s", 1)])
16545                .await
16546                .expect("the current launch syncs");
16547
16548            // A swap candidate registers with the swap token and is refused
16549            // while the incumbent keeps syncing.
16550            supervisor.open_swap(OWNER, "n2".to_string());
16551            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16552            assert_eq!(
16553                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16554                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16555            );
16556            sync(&handler, &incumbent, 2, vec![head("s", 1)])
16557                .await
16558                .expect("the serving owner syncs during the swap");
16559
16560            // The swap fails and is rolled back. The candidate never held sync
16561            // authority, and still cannot sync.
16562            supervisor.close_swap(OWNER);
16563            assert_eq!(
16564                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16565                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16566            );
16567            sync(&handler, &incumbent, 3, vec![head("s", 1)])
16568                .await
16569                .expect("the serving owner syncs after the rollback");
16570            handler.cleanup_connection(candidate.connection_id).unwrap();
16571
16572            // A swap that cuts over. Promotion records the candidate's nonce as
16573            // the module's spawn nonce, which is what `set_spawn_nonce` does
16574            // here; the promoted connection then takes authority at any
16575            // generation and the superseded incumbent is refused.
16576            supervisor.open_swap(OWNER, "n3".to_string());
16577            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16578            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16579            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16580                .await
16581                .expect("the promoted launch takes authority");
16582            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16583                panic!("unexpected reply {reply:?}");
16584            };
16585            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16586            assert_eq!(
16587                sync(&handler, &incumbent, 4, Vec::new()).await,
16588                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16589            );
16590        }
16591
16592        /// Authority dies with its connection: the cleanup path releases it,
16593        /// so the owner's next connection takes it at any generation.
16594        #[tokio::test]
16595        async fn closing_the_authority_connection_frees_sync_authority() {
16596            let supervisor = SupervisorHandle::new();
16597            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16598            let handler = ControlHandler::new(Arc::new(Registry::default()))
16599                .with_supervisor(supervisor.clone());
16600            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16601            sync(&handler, &first, 10, vec![head("s", 1)])
16602                .await
16603                .unwrap();
16604            handler.cleanup_connection(first.connection_id).unwrap();
16605
16606            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16607            sync(&handler, &second, 1, vec![head("s", 1)])
16608                .await
16609                .expect("the next connection takes the released authority");
16610        }
16611
16612        #[tokio::test]
16613        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16614            let supervisor = SupervisorHandle::new();
16615            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16616            let handler =
16617                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16618            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16619            sync(&handler, &first, 10, vec![head("s", 1)])
16620                .await
16621                .unwrap();
16622            handler
16623                .handle_control_frame(
16624                    &first,
16625                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16626                )
16627                .await
16628                .unwrap();
16629            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16630            sync(&handler, &second, 1, vec![head("s", 1)])
16631                .await
16632                .expect("GOODBYE releases authority even if the old socket remains open");
16633        }
16634
16635        #[tokio::test]
16636        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16637            let registry = Arc::new(Registry::default());
16638            let supervisor_handle = SupervisorHandle::new();
16639            let supervisor =
16640                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16641                    .with_handle(supervisor_handle.clone())
16642                    .with_daemon_incarnation("incarnation-7".to_string());
16643            // Configured with enabled: false, so the supervisor lists the
16644            // module without spawning a process for it.
16645            supervisor
16646                .supervise_configured(
16647                    ModuleSpec {
16648                        module_id: OWNER.to_string(),
16649                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16650                        args: Vec::new(),
16651                        env: Vec::new(),
16652                        reserved: false,
16653                        reserved_prefixes: Vec::new(),
16654                        protocol: ModuleProtocol::Subc,
16655                        overlap: Default::default(),
16656                    },
16657                    false,
16658                )
16659                .unwrap();
16660            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16661            let handler =
16662                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
16663            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16664
16665            // Configured but not yet synced: a reader waits for the owner.
16666            let ModuleControlResponseToModule::ScopeDescribe {
16667                status,
16668                daemon_incarnation,
16669                owner_synced,
16670                owner_configured,
16671                scope,
16672                ..
16673            } = describe(&handler, &reader, OWNER, "s").await
16674            else {
16675                panic!("not a describe reply");
16676            };
16677            assert_eq!(status, ScopeStatus::NotLive);
16678            assert_eq!(daemon_incarnation, "incarnation-7");
16679            assert!(!owner_synced);
16680            assert!(owner_configured);
16681            assert!(scope.is_none());
16682
16683            // Not a supervised module: the owner will never sync, and a reader
16684            // refuses rather than waits.
16685            let ModuleControlResponseToModule::ScopeDescribe {
16686                status,
16687                owner_configured,
16688                ..
16689            } = describe(&handler, &reader, "ghost", "s").await
16690            else {
16691                panic!("not a describe reply");
16692            };
16693            assert_eq!(status, ScopeStatus::NotLive);
16694            assert!(!owner_configured);
16695
16696            // Live, with the stamp fields and the computed owner_authorized.
16697            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16698            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16699            let ModuleControlResponseToModule::ScopeDescribe {
16700                status,
16701                scope_epoch,
16702                owner_synced,
16703                scope,
16704                ..
16705            } = describe(&handler, &reader, OWNER, "s").await
16706            else {
16707                panic!("not a describe reply");
16708            };
16709            assert_eq!(status, ScopeStatus::Live);
16710            assert_eq!(scope_epoch, Some(4));
16711            assert!(owner_synced);
16712            let stamp = scope.expect("a live scope carries its stamp");
16713            assert!(
16714                stamp.owner_authorized,
16715                "prefrontal-core is the default authority"
16716            );
16717            assert_eq!(stamp.kind, ScopeKind::Head);
16718        }
16719
16720        #[tokio::test]
16721        async fn scope_authority_owners_decides_owner_authorized() {
16722            let supervisor = SupervisorHandle::new();
16723            supervisor.set_spawn_nonce("broca", "b1".to_string());
16724            let handler = ControlHandler::new(Arc::new(Registry::default()))
16725                .with_supervisor(supervisor)
16726                .with_scope_authority_owners(vec!["broca".to_string()]);
16727            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16728            let mut gated = head("s", 1);
16729            gated.attributes.agent_id = Some("agent".to_string());
16730            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16731            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16732                describe(&handler, &broca, "broca", "s").await
16733            else {
16734                panic!("not a describe reply");
16735            };
16736            assert!(scope.unwrap().owner_authorized);
16737        }
16738
16739        /// With route admission, the stamp, the commit re-check and drains in
16740        /// place, the feature is advertised: the module ops in HELLO_ACK, and
16741        /// `scopes/v1` in HELLO_ACK and `server.describe`.
16742        #[tokio::test]
16743        async fn scope_ops_and_the_scopes_capability_are_advertised() {
16744            let handler = ControlHandler::new(Arc::new(Registry::default()));
16745            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16746            let ack = hello_via_sink(
16747                &handler,
16748                &ctx,
16749                &mut rx,
16750                hello_frame("m", PROTOCOL_VERSION, 1),
16751            )
16752            .await;
16753            let ack = parse_ack(&ack);
16754            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16755                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16756            }
16757            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16758
16759            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16760            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16761            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16762            let reply = handler
16763                .handle_control_frame(&client, frame)
16764                .await
16765                .unwrap()
16766                .pop()
16767                .unwrap();
16768            let ClientControlResponse::ServerDescribe { capabilities, .. } =
16769                serde_json::from_slice(&reply.body).unwrap()
16770            else {
16771                panic!("not a server.describe reply");
16772            };
16773            assert!(
16774                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16775                "{capabilities:?}"
16776            );
16777        }
16778
16779        // ---- route admission, stamps, commit re-check and drains ----------
16780
16781        const PLEXUS: &str = "plexus";
16782        const OTHER: &str = "other";
16783        const AFT: &str = "aft";
16784        const BROCA: &str = "broca";
16785        const MAGIC: &str = "magic-context";
16786
16787        fn nonce(module_id: &str) -> String {
16788            format!("nonce-{module_id}")
16789        }
16790
16791        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16792            let (tx, rx) = mpsc::channel(64);
16793            (
16794                RouteCtx {
16795                    connection_id: ConnectionId::new(connection),
16796                    egress: FrameSink::new(tx),
16797                },
16798                rx,
16799            )
16800        }
16801
16802        /// A daemon with a configured owner (prefrontal-core) registered on its
16803        /// own module connection, two routable targets (plexus, other), and
16804        /// launch nonces minted for the modules that open routes as carriers.
16805        struct Rig {
16806            handler: ControlHandler,
16807            forwarding: Arc<ForwardingTable>,
16808            owner: RouteCtx,
16809            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16810            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16811            generation: u64,
16812            next_connection: u64,
16813            _supervisor: Supervisor,
16814        }
16815
16816        async fn rig() -> Rig {
16817            rig_with_flow_support(true).await
16818        }
16819
16820        async fn rig_with_flow_support(flow_support: bool) -> Rig {
16821            rig_with_scope_capabilities(flow_support, false).await
16822        }
16823
16824        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16825            let mut provides = Vec::new();
16826            if flow_support {
16827                provides.push("flow-scopes/v1");
16828            }
16829            if run_support {
16830                provides.push("agent-run-scopes/v1");
16831            }
16832            rig_with_target_capabilities(&provides).await
16833        }
16834
16835        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16836            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16837            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16838            // Decoder versions alone cannot admit scoped calls: only a capability
16839            // declaration promises the required run or flow behaviour.
16840            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16841            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16842            Frame::build(
16843                FrameType::Hello,
16844                control_flags(),
16845                0,
16846                0,
16847                connection,
16848                serde_json::to_vec(&body).unwrap(),
16849            )
16850            .unwrap()
16851        }
16852
16853        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16854            let registry = Arc::new(Registry::default());
16855            let forwarding = Arc::new(ForwardingTable::default());
16856            let supervisor_handle = SupervisorHandle::new();
16857            let supervisor =
16858                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16859                    .with_handle(supervisor_handle.clone());
16860            supervisor
16861                .supervise_configured(
16862                    ModuleSpec {
16863                        module_id: OWNER.to_string(),
16864                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16865                        args: Vec::new(),
16866                        env: Vec::new(),
16867                        reserved: false,
16868                        reserved_prefixes: Vec::new(),
16869                        protocol: ModuleProtocol::Subc,
16870                        overlap: Default::default(),
16871                    },
16872                    false,
16873                )
16874                .unwrap();
16875            for module_id in [OWNER, AFT, BROCA, MAGIC] {
16876                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16877            }
16878            let handler =
16879                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16880                    .with_supervisor(supervisor_handle);
16881            let (owner, mut owner_rx) = wide_ctx(1);
16882            hello_via_sink(
16883                &handler,
16884                &owner,
16885                &mut owner_rx,
16886                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16887            )
16888            .await;
16889            let mut modules = BTreeMap::new();
16890            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16891                let (ctx, mut rx) = wide_ctx(connection);
16892                let hello = hello_with_capabilities(module_id, connection, provides);
16893                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16894                modules.insert(module_id.to_string(), (ctx, rx));
16895            }
16896            Rig {
16897                handler,
16898                forwarding,
16899                owner,
16900                _owner_rx: owner_rx,
16901                modules,
16902                generation: 0,
16903                next_connection: 100,
16904                _supervisor: supervisor,
16905            }
16906        }
16907
16908        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16909            ScopeCarrier::new(Principal::Reserved {
16910                module_id: module_id.to_string(),
16911            })
16912            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16913        }
16914
16915        /// The scope most tests open under: aft carries to any module, broca
16916        /// only to plexus and other, and the owner delegates as agent-1.
16917        fn session(scope_epoch: u64) -> ScopeRecord {
16918            let mut record = head("s", scope_epoch);
16919            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16920            record.attributes.agent_id = Some("agent-1".to_string());
16921            record.attributes.delegates = true;
16922            record
16923        }
16924
16925        impl Rig {
16926            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16927                self.generation += 1;
16928                sync(&self.handler, &self.owner, self.generation, scopes)
16929                    .await
16930                    .expect("the owner's sync is accepted");
16931            }
16932
16933            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16934                ScopeSelector::new(
16935                    Principal::Reserved {
16936                        module_id: OWNER.to_string(),
16937                    },
16938                    scope_ref.to_string(),
16939                    scope_epoch,
16940                )
16941            }
16942
16943            fn open_frame(
16944                &mut self,
16945                opener: Option<&str>,
16946                target: &str,
16947                scope: Option<ScopeSelector>,
16948            ) -> (
16949                RouteCtx,
16950                mpsc::Receiver<crate::router::OutboundFrame>,
16951                Frame,
16952            ) {
16953                self.next_connection += 1;
16954                let (ctx, rx) = wide_ctx(self.next_connection);
16955                let root = unique_project_root("scoped-open");
16956                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16957                    target: RouteTarget::ToolProvider {
16958                        module_id: target.to_string(),
16959                    },
16960                    identity: BindIdentity::new(
16961                        root.path().to_path_buf(),
16962                        "unit".to_string(),
16963                        "session".to_string(),
16964                    ),
16965                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
16966                        module_id: module_id.to_string(),
16967                        launch_nonce: nonce(module_id),
16968                    }),
16969                    consumer_capabilities: None,
16970                    role_versions: None,
16971                    admission_facts: None,
16972                    scope,
16973                })
16974                .unwrap();
16975                let frame = Frame::build(
16976                    FrameType::Request,
16977                    control_flags(),
16978                    0,
16979                    0,
16980                    self.next_connection,
16981                    body,
16982                )
16983                .unwrap();
16984                (ctx, rx, frame)
16985            }
16986
16987            /// Open and expect a refusal before anything is relayed.
16988            async fn refused(
16989                &mut self,
16990                opener: Option<&str>,
16991                target: &str,
16992                scope: Option<ScopeSelector>,
16993            ) -> String {
16994                self.refusal_body(opener, target, scope).await["code"]
16995                    .as_str()
16996                    .unwrap()
16997                    .to_string()
16998            }
16999
17000            async fn refusal_body(
17001                &mut self,
17002                opener: Option<&str>,
17003                target: &str,
17004                scope: Option<ScopeSelector>,
17005            ) -> Value {
17006                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17007                let replies = tokio::time::timeout(
17008                    Duration::from_secs(2),
17009                    self.handler.handle_control_frame(&ctx, frame),
17010                )
17011                .await
17012                .expect("the open must be refused before waiting for a bind ack")
17013                .unwrap();
17014                assert_eq!(replies.len(), 1, "{replies:?}");
17015                assert_eq!(replies[0].header.ty, FrameType::Error);
17016                let (_, module_rx) = self.modules.get_mut(target).unwrap();
17017                assert!(
17018                    module_rx.try_recv().is_err(),
17019                    "a refused open relays nothing"
17020                );
17021                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17022                parse_error(&replies[0])
17023            }
17024
17025            /// Start an open and return its task and the bind the target got.
17026            async fn relayed(
17027                &mut self,
17028                opener: Option<&str>,
17029                target: &str,
17030                scope: Option<ScopeSelector>,
17031            ) -> Relayed {
17032                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
17033                let handler = self.handler.clone();
17034                let task_ctx = ctx.clone();
17035                let task = tokio::spawn(async move {
17036                    handler
17037                        .handle_control_frame(&task_ctx, frame)
17038                        .await
17039                        .unwrap()
17040                });
17041                let (_, module_rx) = self.modules.get_mut(target).unwrap();
17042                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
17043                    .await
17044                    .expect("the target receives the relayed route.bind")
17045                    .unwrap()
17046                    .frame;
17047                Relayed {
17048                    target: target.to_string(),
17049                    client: ctx,
17050                    client_rx: rx,
17051                    task,
17052                    bind,
17053                }
17054            }
17055
17056            async fn ack(&self, relayed: &Relayed) {
17057                let (module, _) = &self.modules[&relayed.target];
17058                self.handler
17059                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
17060                    .await
17061                    .unwrap();
17062            }
17063
17064            /// Open, ack and return the bound route.
17065            async fn bound(
17066                &mut self,
17067                opener: Option<&str>,
17068                target: &str,
17069                scope: Option<ScopeSelector>,
17070            ) -> Bound {
17071                let relayed = self.relayed(opener, target, scope).await;
17072                self.ack(&relayed).await;
17073                let Relayed {
17074                    target,
17075                    client,
17076                    mut client_rx,
17077                    task,
17078                    bind,
17079                } = relayed;
17080                assert!(
17081                    task.await.unwrap().is_empty(),
17082                    "the open is answered by commit"
17083                );
17084                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17085                Bound {
17086                    target,
17087                    client,
17088                    client_rx,
17089                    channel,
17090                    epoch,
17091                    bind,
17092                }
17093            }
17094
17095            fn live(&self, route: &Bound) -> bool {
17096                matches!(
17097                    self.forwarding
17098                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17099                        .unwrap(),
17100                    DataRoute::Client(DataRouteState::Bound(_))
17101                )
17102            }
17103        }
17104
17105        struct Relayed {
17106            target: String,
17107            client: RouteCtx,
17108            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17109            task: tokio::task::JoinHandle<Vec<Frame>>,
17110            bind: Frame,
17111        }
17112
17113        struct Bound {
17114            target: String,
17115            client: RouteCtx,
17116            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17117            channel: u16,
17118            epoch: u32,
17119            bind: Frame,
17120        }
17121
17122        impl Bound {
17123            /// The reason of the `route.closed` this client was sent, after
17124            /// checking it also got a GOODBYE on exactly this route.
17125            fn closed_reason(&mut self) -> RouteCloseReason {
17126                let mut reason = None;
17127                let mut goodbye = false;
17128                while let Ok(outbound) = self.client_rx.try_recv() {
17129                    let frame = outbound.frame;
17130                    match frame.header.ty {
17131                        FrameType::Goodbye => {
17132                            assert_eq!(
17133                                (frame.header.channel, frame.header.epoch),
17134                                (self.channel, self.epoch)
17135                            );
17136                            goodbye = true;
17137                        }
17138                        FrameType::Push => {
17139                            let ClientControlPush::RouteClosed {
17140                                reason: r,
17141                                module_id,
17142                                ..
17143                            } = serde_json::from_slice(&frame.body).unwrap()
17144                            else {
17145                                panic!("unexpected push");
17146                            };
17147                            assert_eq!(module_id, self.target);
17148                            reason = Some(r);
17149                        }
17150                        other => panic!("unexpected frame {other:?}"),
17151                    }
17152                }
17153                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17154                reason.expect("the client is told why the route closed")
17155            }
17156
17157            fn untouched(&mut self) -> bool {
17158                self.client_rx.try_recv().is_err()
17159            }
17160
17161            fn stamp(&self) -> Option<ScopeStamp> {
17162                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17163                    ModuleControlRequest::RouteBind { scope, .. } => scope,
17164                    other => panic!("expected a route.bind, got {other:?}"),
17165                }
17166            }
17167        }
17168
17169        #[tokio::test]
17170        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17171        ) {
17172            let mut rig = rig().await;
17173            let mut record = session(1);
17174            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17175            record.child_owners = vec![Principal::Reserved {
17176                module_id: MAGIC.to_string(),
17177            }];
17178            rig.sync(vec![record]).await;
17179            let scope = || Some(rig_selector("s", Some(1)));
17180
17181            // Admitted: the owner, a bare carrier to any module, a targeted
17182            // carrier to its listed module.
17183            rig.bound(Some(OWNER), PLEXUS, scope()).await;
17184            rig.bound(Some(AFT), OTHER, scope()).await;
17185            rig.bound(Some(BROCA), PLEXUS, scope()).await;
17186
17187            // Refused scope_not_carrier: a targeted carrier to an unlisted
17188            // module, a module that is not listed at all (a child owner is not
17189            // a carrier), and a direct key-holder.
17190            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17191                assert_eq!(
17192                    rig.refused(opener, target, scope()).await,
17193                    error_codes::SCOPE_NOT_CARRIER,
17194                    "{opener:?} -> {target}"
17195                );
17196            }
17197        }
17198
17199        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17200            ScopeSelector::new(
17201                Principal::Reserved {
17202                    module_id: OWNER.to_string(),
17203                },
17204                scope_ref.to_string(),
17205                scope_epoch,
17206            )
17207        }
17208
17209        #[tokio::test]
17210        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17211            let mut rig = rig().await;
17212            rig.sync(vec![session(1)]).await;
17213            for opener in [OWNER, AFT] {
17214                assert_eq!(
17215                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17216                        .await,
17217                    error_codes::SCOPE_EPOCH_REQUIRED,
17218                    "{opener}"
17219                );
17220            }
17221        }
17222
17223        #[tokio::test]
17224        async fn admission_separates_not_synced_not_live_and_ended() {
17225            let mut rig = rig().await;
17226            // Before the configured owner's first sync: retryable.
17227            let code = rig
17228                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17229                .await;
17230            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17231            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17232
17233            // An owner that is not configured will never sync: terminal.
17234            let ghost = ScopeSelector::new(
17235                Principal::Reserved {
17236                    module_id: "ghost".to_string(),
17237                },
17238                "s".to_string(),
17239                Some(1),
17240            );
17241            assert_eq!(
17242                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17243                error_codes::SCOPE_NOT_LIVE
17244            );
17245
17246            rig.sync(vec![session(2)]).await;
17247            assert_eq!(
17248                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17249                    .await,
17250                error_codes::SCOPE_NOT_LIVE
17251            );
17252            for epoch in [1, 3] {
17253                assert_eq!(
17254                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17255                        .await,
17256                    error_codes::SCOPE_ENDED,
17257                    "epoch {epoch}"
17258                );
17259            }
17260            // Control: the live epoch is admitted.
17261            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17262                .await;
17263        }
17264
17265        #[tokio::test]
17266        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17267            let mut rig = rig().await;
17268            rig.sync(vec![session(1)]).await;
17269            let route = rig
17270                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17271                .await;
17272            let stamp = route.stamp().expect("a scoped bind carries the stamp");
17273            assert_eq!(stamp.scope_ref, "s");
17274            assert_eq!(stamp.scope_epoch, 1);
17275            assert_eq!(stamp.kind, ScopeKind::Head);
17276            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17277            assert!(stamp.attributes.delegates);
17278            assert!(stamp.owner_authorized);
17279            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17280            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17281
17282            // broca owns a scope of its own on its own module connection; it is
17283            // not in scope_authority_owners, so its stamp is not authorized.
17284            let (broca, mut broca_rx) = wide_ctx(50);
17285            hello_via_sink(
17286                &rig.handler,
17287                &broca,
17288                &mut broca_rx,
17289                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17290            )
17291            .await;
17292            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17293                .await
17294                .unwrap();
17295            let own = ScopeSelector::new(
17296                Principal::Reserved {
17297                    module_id: BROCA.to_string(),
17298                },
17299                "b".to_string(),
17300                Some(1),
17301            );
17302            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17303            assert!(!route.stamp().unwrap().owner_authorized);
17304        }
17305
17306        #[tokio::test]
17307        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17308            let mut rig = rig().await;
17309            let mut record = head("s", 1);
17310            record.carriers = vec![carrier(AFT, None)];
17311            let flow_id = "Flow:run-7/step_2!~";
17312            record.attributes.flow_id = Some(flow_id.to_string());
17313            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17314                .await
17315                .unwrap();
17316            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17317                panic!("not a sync reply");
17318            };
17319            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17320            let route = rig
17321                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17322                .await;
17323            assert!(rig.live(&route), "the stamped bind committed");
17324            let stamp = route.stamp().expect("a flow scope carries a stamp");
17325            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17326            assert_eq!(stamp.attributes.agent_id, None);
17327            assert!(!stamp.attributes.delegates);
17328            assert!(stamp.owner_authorized);
17329            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17330            assert_eq!(unscoped.stamp(), None);
17331        }
17332
17333        #[tokio::test]
17334        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17335            let mut rig = rig_with_flow_support(false).await;
17336            let mut record = session(1);
17337            record.attributes.flow_id = Some("flow:7".to_string());
17338            rig.sync(vec![record]).await;
17339            for opener in [OWNER, AFT] {
17340                let body = rig
17341                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17342                    .await;
17343                assert_eq!(body["code"], "target_flow_unsupported");
17344                let message = body["message"].as_str().unwrap();
17345                for required in [PLEXUS, "flow-scopes/v1"] {
17346                    assert!(message.contains(required), "{message}");
17347                }
17348            }
17349        }
17350
17351        #[tokio::test]
17352        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17353            let mut rig = rig_with_flow_support(true).await;
17354            let mut record = session(1);
17355            record.attributes.flow_id = Some("flow:7".to_string());
17356            rig.sync(vec![record]).await;
17357            for opener in [OWNER, AFT] {
17358                let route = rig
17359                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17360                    .await;
17361                assert!(rig.live(&route));
17362                assert_eq!(
17363                    route.stamp().unwrap().attributes.flow_id.as_deref(),
17364                    Some("flow:7")
17365                );
17366            }
17367        }
17368
17369        #[tokio::test]
17370        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17371            use std::future::Future;
17372
17373            let mut rig = rig_with_flow_support(true).await;
17374            let mut record = session(1);
17375            record.attributes.flow_id = Some("flow:7".to_string());
17376            rig.sync(vec![record]).await;
17377            let (client, mut client_rx, frame) =
17378                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17379            // Hold the route-open response permit so admission sees the first
17380            // target but relay reservation cannot capture an endpoint yet.
17381            for _ in 0..64 {
17382                client.egress.try_send(route_bind_ack(1)).unwrap();
17383            }
17384            let handler = rig.handler.clone();
17385            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17386            std::future::poll_fn(|cx| {
17387                assert!(open.as_mut().poll(cx).is_pending());
17388                std::task::Poll::Ready(())
17389            })
17390            .await;
17391            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17392
17393            let old_connection = rig.modules[PLEXUS].0.connection_id;
17394            rig.handler.cleanup_connection(old_connection).unwrap();
17395            let (replacement, mut replacement_rx) = wide_ctx(200);
17396            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17397            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17398            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17399            let hello = Frame::build(
17400                FrameType::Hello,
17401                control_flags(),
17402                0,
17403                0,
17404                200,
17405                serde_json::to_vec(&body).unwrap(),
17406            )
17407            .unwrap();
17408            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17409
17410            client_rx.try_recv().unwrap();
17411            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17412                .await
17413                .expect("the replacement is refused without waiting for a bind ack")
17414                .unwrap();
17415            assert_eq!(replies.len(), 1);
17416            let body = parse_error(&replies[0]);
17417            assert_eq!(body["code"], "target_flow_unsupported");
17418            for required in [PLEXUS, "flow-scopes/v1"] {
17419                assert!(body["message"].as_str().unwrap().contains(required));
17420            }
17421            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17422            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17423            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17424            assert!(rig
17425                .handler
17426                .registry
17427                .get_module_by_connection(replacement.connection_id)
17428                .unwrap()
17429                .is_some());
17430        }
17431
17432        #[tokio::test]
17433        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17434            use std::future::Future;
17435            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17436            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17437            const NEXT_HASH: &str =
17438                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17439            let mut rig = rig().await;
17440            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17441            record.carriers.push(ScopeCarrier::new(Principal::Direct));
17442            rig.sync(vec![record.clone()]).await;
17443            let (client, mut rx, frame) = rig.open_frame(
17444                None,
17445                PLEXUS,
17446                Some(
17447                    rig.selector("s", Some(1))
17448                        .with_bind_proof(Some(PROOF.into())),
17449                ),
17450            );
17451            // Hold egress reservation while the scope owner changes its bind-proof
17452            // hash, so a proof valid at admission must be checked again at relay.
17453            for _ in 0..64 {
17454                client.egress.try_send(route_bind_ack(1)).unwrap();
17455            }
17456            let handler = rig.handler.clone();
17457            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17458            std::future::poll_fn(|cx| {
17459                assert!(open.as_mut().poll(cx).is_pending());
17460                std::task::Poll::Ready(())
17461            })
17462            .await;
17463            record.bind_proof_sha256 = Some(NEXT_HASH.into());
17464            rig.sync(vec![record]).await;
17465            rx.try_recv().unwrap();
17466            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17467                .await
17468                .unwrap()
17469                .unwrap();
17470            assert_eq!(
17471                parse_error(&replies[0])["code"],
17472                error_codes::SCOPE_BIND_PROOF_MISMATCH
17473            );
17474            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17475            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17476            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17477        }
17478
17479        #[tokio::test]
17480        async fn scope_bind_proof_reserved_openers_ignore_proof() {
17481            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17482            let mut rig = rig().await;
17483            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17484                .await;
17485            for opener in [OWNER, AFT] {
17486                for proof in [None, Some("malformed+proof=")] {
17487                    let route = rig
17488                        .bound(
17489                            Some(opener),
17490                            PLEXUS,
17491                            Some(
17492                                rig.selector("s", Some(1))
17493                                    .with_bind_proof(proof.map(str::to_string)),
17494                            ),
17495                        )
17496                        .await;
17497                    assert!(rig.live(&route));
17498                }
17499            }
17500        }
17501
17502        #[tokio::test]
17503        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17504            use std::future::Future;
17505            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17506            const NEXT_HASH: &str =
17507                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17508            let mut rig = rig().await;
17509            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17510                .await;
17511            let (client, mut rx, frame) =
17512                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17513            for _ in 0..64 {
17514                client.egress.try_send(route_bind_ack(1)).unwrap();
17515            }
17516            let handler = rig.handler.clone();
17517            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17518            std::future::poll_fn(|cx| {
17519                assert!(open.as_mut().poll(cx).is_pending());
17520                std::task::Poll::Ready(())
17521            })
17522            .await;
17523            rig.sync(vec![
17524                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17525            ])
17526            .await;
17527            rx.try_recv().unwrap();
17528            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17529            let bind = std::future::poll_fn(|cx| {
17530                assert!(open.as_mut().poll(cx).is_pending());
17531                module_rx.poll_recv(cx)
17532            })
17533            .await
17534            .unwrap()
17535            .frame;
17536            handler
17537                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17538                .await
17539                .unwrap();
17540            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17541                .await
17542                .unwrap()
17543                .unwrap();
17544            assert_eq!(replies.len(), 1);
17545            // Proof rechecking allowed the relay. Independent tag settlement
17546            // still asks the opener to retry after any same-epoch content change.
17547            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17548            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17549            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17550            let route = rig
17551                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17552                .await;
17553            assert!(rig.live(&route));
17554        }
17555
17556        #[tokio::test]
17557        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17558        {
17559            let mut rig = rig_with_flow_support(false).await;
17560            rig.sync(vec![session(1)]).await;
17561            let route = rig
17562                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17563                .await;
17564            assert!(rig.live(&route));
17565            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17566            let mut record = session(1);
17567            record.attributes.flow_id = Some("flow:7".to_string());
17568            rig.sync(vec![record]).await;
17569            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17570            assert!(rig.live(&unscoped));
17571            assert_eq!(unscoped.stamp(), None);
17572        }
17573
17574        #[tokio::test]
17575        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17576            let mut rig = rig().await;
17577            let mut record = session(1);
17578            record.attributes.flow_id = Some("flow:7".to_string());
17579            rig.sync(vec![record.clone()]).await;
17580            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17581            let mut owner_route = rig
17582                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17583                .await;
17584            let mut carrier_route = rig
17585                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17586                .await;
17587            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17588            rig.sync(vec![record.clone()]).await;
17589            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17590            assert!(rig.live(&owner_route) && owner_route.untouched());
17591            assert!(rig.live(&carrier_route) && carrier_route.untouched());
17592
17593            record.attributes.flow_id = Some("flow:8".to_string());
17594            rig.sync(vec![record]).await;
17595            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17596            let before = before.unwrap();
17597            assert_eq!(after.scope_epoch, before.scope_epoch);
17598            assert!(after.version > before.version);
17599            for route in [&mut owner_route, &mut carrier_route] {
17600                assert!(!rig.live(route));
17601                assert_eq!(
17602                    route.closed_reason(),
17603                    RouteCloseReason::ScopeDelegationChanged
17604                );
17605            }
17606            assert!(rig.live(&unscoped) && unscoped.untouched());
17607            // Each provider also receives a GOODBYE for its drained route;
17608            // consume it before expecting the next route.bind on that sink.
17609            for target in [PLEXUS, OTHER] {
17610                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17611                let goodbye = module_rx
17612                    .try_recv()
17613                    .expect("the provider sees the drain")
17614                    .frame;
17615                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17616                assert!(module_rx.try_recv().is_err());
17617            }
17618            let rebound = rig
17619                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17620                .await;
17621            assert_eq!(
17622                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17623                Some("flow:8")
17624            );
17625        }
17626
17627        /// The owner's sync lands between admission and the module's ack. The
17628        /// open is refused by name, the module's other routes stay up, and the
17629        /// reserved pair is released. Changed content is retryable; an ended
17630        /// scope is not.
17631        #[tokio::test]
17632        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17633            let mut rig = rig().await;
17634            rig.sync(vec![session(1)]).await;
17635            let mut cotenant = rig
17636                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17637                .await;
17638
17639            let mut changed = session(1);
17640            changed.child_owners.push(Principal::Reserved {
17641                module_id: MAGIC.to_string(),
17642            });
17643            let mut ended = None;
17644            for (code, next) in [
17645                (error_codes::SCOPE_CHANGED, vec![changed]),
17646                (error_codes::SCOPE_ENDED, Vec::new()),
17647            ] {
17648                let relayed = rig
17649                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17650                    .await;
17651                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17652                ended = Some(next.is_empty());
17653                rig.sync(next).await;
17654                rig.ack(&relayed).await;
17655                let replies = relayed.task.await.unwrap();
17656                assert_eq!(replies.len(), 1, "{replies:?}");
17657                assert_eq!(parse_error(&replies[0])["code"], code);
17658                assert_eq!(
17659                    subc_protocol::error_codes::is_retryable_route_open(code),
17660                    code == error_codes::SCOPE_CHANGED
17661                );
17662                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17663                // The module is told to drop just the binding it created.
17664                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17665                // Collected, because ending the scope also closes the co-tenant
17666                // route, whose GOODBYE comes first.
17667                let mut goodbyes = Vec::new();
17668                while let Ok(outbound) = plexus_rx.try_recv() {
17669                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17670                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17671                }
17672                assert!(
17673                    goodbyes.contains(&(bind_channel, bind_epoch)),
17674                    "{goodbyes:?}"
17675                );
17676                assert!(rig
17677                    .handler
17678                    .registry
17679                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17680                    .unwrap()
17681                    .is_some());
17682            }
17683            assert_eq!(ended, Some(true));
17684            // The co-tenant stayed up through the change, and closed only when
17685            // the scope ended, by the drain rule rather than by the commit.
17686            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17687        }
17688
17689        /// Each row of the drain table on one set of routes: the owner's, a
17690        /// bare carrier's, and a targeted carrier's to each of its targets.
17691        #[tokio::test]
17692        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17693            struct Case {
17694                name: &'static str,
17695                change: fn(&mut ScopeRecord),
17696                /// Closed routes by index: owner->plexus, aft->plexus,
17697                /// broca->plexus, broca->other.
17698                closed: [Option<RouteCloseReason>; 4],
17699            }
17700            use RouteCloseReason::*;
17701            let cases = [
17702                Case {
17703                    name: "a carrier entry removed",
17704                    change: |r| {
17705                        r.carriers.retain(|c| {
17706                            c.principal
17707                                != Principal::Reserved {
17708                                    module_id: AFT.to_string(),
17709                                }
17710                        })
17711                    },
17712                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17713                },
17714                Case {
17715                    name: "a target removed from a carrier",
17716                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17717                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
17718                },
17719                Case {
17720                    name: "a bare carrier narrowed to targets",
17721                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17722                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17723                },
17724                Case {
17725                    name: "delegates turned off",
17726                    change: |r| r.attributes.delegates = false,
17727                    closed: [Some(ScopeDelegationChanged); 4],
17728                },
17729                Case {
17730                    name: "agent_id changed",
17731                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17732                    closed: [Some(ScopeDelegationChanged); 4],
17733                },
17734                Case {
17735                    name: "a carrier added, child owners changed, the record re-sent",
17736                    change: |r| {
17737                        r.carriers.push(carrier(MAGIC, None));
17738                        r.child_owners.push(Principal::Reserved {
17739                            module_id: MAGIC.to_string(),
17740                        });
17741                    },
17742                    closed: [None; 4],
17743                },
17744                Case {
17745                    name: "a target added",
17746                    change: |r| {
17747                        r.carriers[1]
17748                            .targets
17749                            .as_mut()
17750                            .unwrap()
17751                            .push("third".to_string())
17752                    },
17753                    closed: [None; 4],
17754                },
17755                Case {
17756                    name: "delegates turned on",
17757                    change: |r| r.attributes.delegates = true,
17758                    closed: [None; 4],
17759                },
17760            ];
17761            for case in cases {
17762                let mut rig = rig().await;
17763                rig.sync(vec![session(1)]).await;
17764                let scope = || Some(rig_selector("s", Some(1)));
17765                let mut routes = [
17766                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
17767                    rig.bound(Some(AFT), PLEXUS, scope()).await,
17768                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
17769                    rig.bound(Some(BROCA), OTHER, scope()).await,
17770                ];
17771                let mut record = session(1);
17772                (case.change)(&mut record);
17773                rig.sync(vec![record]).await;
17774                for (index, expected) in case.closed.iter().enumerate() {
17775                    let route = &mut routes[index];
17776                    match expected {
17777                        Some(reason) => {
17778                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
17779                            assert_eq!(
17780                                route.closed_reason(),
17781                                *reason,
17782                                "{}: route {index}",
17783                                case.name
17784                            );
17785                        }
17786                        None => {
17787                            assert!(rig.live(route), "{}: route {index} closed", case.name);
17788                            assert!(
17789                                route.untouched(),
17790                                "{}: route {index} was told something",
17791                                case.name
17792                            );
17793                        }
17794                    }
17795                }
17796            }
17797        }
17798
17799        #[tokio::test]
17800        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17801            // Removed, and replaced by a higher epoch.
17802            for next in [Vec::new(), vec![session(2)]] {
17803                let mut rig = rig().await;
17804                rig.sync(vec![session(1)]).await;
17805                let mut route = rig
17806                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17807                    .await;
17808                rig.sync(next).await;
17809                assert!(!rig.live(&route));
17810                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17811            }
17812
17813            // A child whose parent ends: its routes close as parent-ended, the
17814            // child stays live, and routes under the parent close as ended.
17815            let mut rig = rig().await;
17816            let mut child = session(1);
17817            child.scope_ref = "child".to_string();
17818            child.kind = ScopeKind::Worker;
17819            child.parent = Some(ScopeParent::new(
17820                Principal::Reserved {
17821                    module_id: OWNER.to_string(),
17822                },
17823                "s".to_string(),
17824                1,
17825            ));
17826            rig.sync(vec![session(1), child.clone()]).await;
17827            let mut child_route = rig
17828                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17829                .await;
17830            assert_eq!(
17831                child_route.stamp().unwrap().parent_state,
17832                Some(ParentState::Linked)
17833            );
17834            rig.sync(vec![child]).await;
17835            assert!(!rig.live(&child_route));
17836            assert_eq!(
17837                child_route.closed_reason(),
17838                RouteCloseReason::ScopeParentEnded
17839            );
17840        }
17841
17842        #[tokio::test]
17843        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17844        ) {
17845            let mut rig = rig().await;
17846            rig.sync(vec![session(1)]).await;
17847            let mut route = rig
17848                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17849                .await;
17850            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17851            rig.sync(vec![session(1)]).await;
17852            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17853            assert!(rig.live(&route) && route.untouched());
17854
17855            // A call in flight on the route when another carrier is added. A
17856            // forwarded REQUEST holds one credit on the route's flow until the
17857            // module answers; the router takes it exactly like this.
17858            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17859                .forwarding
17860                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17861                .unwrap()
17862            else {
17863                panic!("the route is bound");
17864            };
17865            binding.flow.acquire_tagged(9, false).await.unwrap();
17866            let mut widened = session(1);
17867            widened.carriers.push(carrier(MAGIC, None));
17868            rig.sync(vec![widened]).await;
17869            assert!(rig.live(&route) && route.untouched());
17870            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17871            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17872            // The call's credit is still held on an open flow, so its answer
17873            // will be delivered: closing the route would have closed the flow.
17874            assert_eq!(binding.flow.in_flight(), 1);
17875            binding
17876                .flow
17877                .acquire_tagged(10, false)
17878                .await
17879                .expect("the flow is still open");
17880        }
17881
17882        /// A swap's superseded endpoint keeps its routes until drained; ending
17883        /// the scope closes them there too.
17884        #[tokio::test]
17885        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17886            let mut rig = rig().await;
17887            rig.sync(vec![session(1)]).await;
17888            let mut on_incumbent = rig
17889                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17890                .await;
17891
17892            // Swap plexus: register a candidate and cut over, leaving the
17893            // incumbent superseded with the route still on it.
17894            let (candidate, _candidate_rx) = wide_ctx(9);
17895            let registration = rig
17896                .handler
17897                .registry
17898                .register_candidate_with_control_ops(
17899                    manifest(PLEXUS, PROTOCOL_VERSION),
17900                    PROTOCOL_VERSION,
17901                    candidate.connection_id,
17902                    module_baseline_control_ops(),
17903                )
17904                .unwrap();
17905            rig.forwarding
17906                .register_candidate_module_connection(
17907                    candidate.connection_id,
17908                    PLEXUS.to_string(),
17909                    PROTOCOL_VERSION,
17910                    manifest_concurrency(&registration.manifest),
17911                    candidate.egress.clone(),
17912                )
17913                .unwrap();
17914            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17915            rig.handler
17916                .registry
17917                .promote_candidate(PLEXUS)
17918                .unwrap()
17919                .unwrap();
17920            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17921
17922            rig.sync(Vec::new()).await;
17923            assert!(!rig.live(&on_incumbent));
17924            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17925            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17926            let goodbye = incumbent_rx
17927                .try_recv()
17928                .expect("the superseded endpoint is told")
17929                .frame;
17930            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17931        }
17932    }
17933}
17934
17935#[cfg(test)]
17936mod concurrency_default_exposure_tests {
17937    use super::*;
17938
17939    fn hello_body(role_json: &str) -> Vec<u8> {
17940        format!(
17941            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17942        )
17943        .into_bytes()
17944    }
17945
17946    fn manifest_from(body: &[u8]) -> ModuleManifest {
17947        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17948        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17949            .expect("manifest parses")
17950    }
17951
17952    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17953
17954    #[test]
17955    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17956        let body = hello_body(&format!(
17957            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17958        ));
17959        let manifest = manifest_from(&body);
17960        // Precondition: serde really resolved it to the default, so the typed
17961        // manifest alone cannot answer the question this probe exists for.
17962        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17963        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17964    }
17965
17966    #[test]
17967    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17968        let body = hello_body(&format!(
17969            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17970        ));
17971        let manifest = manifest_from(&body);
17972        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17973        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17974    }
17975
17976    #[test]
17977    fn non_management_roles_are_never_reported() {
17978        let body = hello_body(
17979            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17980        );
17981        let manifest = manifest_from(&body);
17982        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17983    }
17984}