Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27    },
28    scope::{
29        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31        SCOPE_SYNC_OP,
32    },
33    session::{
34        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37        ROLE_VERSIONS_FIELD,
38    },
39    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40    Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46    caller_identity::CallerIdentity,
47    capability_requirements::{
48        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50        RuntimeModule,
51    },
52    child_roster::DaemonShutdownFlag,
53    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
54    forwarding::{
55        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
56        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
57        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
58    },
59    observability::{
60        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
61    },
62    privileged_prompt::{prompt_text, PrivilegedOperation},
63    provenance::{
64        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
65    },
66    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
67    router::{RouteCtx, RouterError},
68    scopes::{HelloLaunchNonces, ScopeTable},
69    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
70    stderr_tail::{CaptureState, TailEntry},
71    supervise::{
72        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
73        SupervisorHandle, SwapHelloAdmission,
74    },
75    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
76};
77
78/// Lowest envelope version this subc build will negotiate.
79///
80/// Module HELLO negotiation is exact: peers must use the daemon's locked
81/// protocol version. Older and newer peers receive `version_unsupported` and
82/// are not registered.
83pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
84
85const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
86const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
87const CAP_PING_PONG: &str = "ping_pong_v1";
88const CAP_SESSION_ATTACH: &str = "session_attach_v1";
89const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
90
91const SUBC_CONTROL_OPS: &[&str] = &[
92    ops::ACTIVITY_RECORD,
93    ops::ACTIVITY_QUERY,
94    ops::ACTIVITY_SUBSCRIBE,
95    ops::SERVER_DESCRIBE,
96    ops::CATALOG_LIST,
97    ops::ROUTE_OPEN,
98    ops::ROUTE_POLL,
99    ops::ROUTE_CLOSING,
100    ops::ROUTE_CLOSED,
101    ops::SUPERVISOR_LIST,
102    ops::SUPERVISOR_RESTART,
103    ops::SUPERVISOR_SWAP,
104    ops::SUPERVISOR_RELOAD,
105    ops::SUPERVISOR_RESCAN,
106    ops::SUPERVISOR_RELEASE_RESERVED,
107    ops::SUPERVISOR_SET_ENABLED,
108    ops::SUPERVISOR_HEALTH_PROBE,
109    ops::SUPERVISOR_HEALTH,
110    ops::SUPERVISOR_STDERR_TAIL,
111    ops::SUPERVISOR_TERMINALS,
112    ops::SUPERVISOR_ROUTES,
113    ops::SUPERVISOR_PROVENANCE,
114    ops::SUPERVISOR_SPAWN_SNAPSHOT,
115    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
116];
117
118/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
119/// announced by a capability in `server.describe` instead, and clients check
120/// that capability before sending it. `ck daemon --json` prints `subc_ops`
121/// verbatim and scripts read that output, so a new name there would change it.
122/// A malformed body for one of these is still refused `invalid_control_body`
123/// rather than as an unknown op.
124const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
125
126mod rescan_checked;
127use rescan_checked::{RescanMode, RescanPlan};
128
129const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
130    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
131    "supervisor.live_roots",
132    SCOPE_SYNC_OP,
133    SCOPE_APPLY_OP,
134    SCOPE_DESCRIBE_OP,
135    "operator.confirm",
136];
137
138/// Module-originated ops the daemon answers but does not advertise in
139/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
140/// belongs to is incomplete, so no module is told it works before it does.
141const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
142
143const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
144
145/// How long subc waits for a module to ack a relayed route.bind before returning
146/// `module_timeout`. The ack waits on the module's own configure, which for AFT
147/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
148/// and DB-open work — on a cold page cache or a large repo that legitimately
149/// exceeds a couple of seconds. The default is generous because rejecting a VALID
150/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
151/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
152pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
153
154/// How many CONSECUTIVE full-budget relay timeouts against one target module
155/// open that module's bind-relay breaker.
156///
157/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
158/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
159/// so three consecutive full-budget timeouts take ~36s to observe: every client
160/// whose open contributed to opening the breaker had already given up on its
161/// own. That is what makes opening the breaker unable to turn a call that would
162/// have succeeded into a refusal — it can only make an already-failing module
163/// fail faster.
164///
165/// Two would be reachable inside one default deadline. One would convict a
166/// module on a single cold-cache bind, which is exactly the valid-but-slow case
167/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
168pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
169
170/// How long a module's bind-relay breaker stays open before exactly one
171/// `route.open` is let through as a probe.
172///
173/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
174/// re-pays a full-budget stall almost continuously, and the breaker stops being
175/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
176/// deadline: a client that starts retrying after the module recovers has to get
177/// a probe opportunity inside its own deadline, or the breaker converts a
178/// recovered module into a failed call — the failure it exists to prevent,
179/// pointed the other way.
180///
181/// 20s sits between those with room on both sides, and it caps what a wedged
182/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
183/// rather than one per `route.open` per connection. The stall that motivated
184/// this, with its measurements, is written up in
185/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
186/// waited the whole budget out.
187pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
188
189const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
190const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
191
192fn reload_verdict(
193    configured: &Path,
194    spawned_from: Option<&Path>,
195    image: subc_control::RunningImageAgreement,
196) -> PendingReloadVerdict {
197    let path = match spawned_from {
198        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
199            ReloadPathAgreement::Match
200        }
201        Some(spawned_from) => ReloadPathAgreement::Mismatch {
202            configured: configured.to_path_buf(),
203            spawned_from: spawned_from.to_path_buf(),
204        },
205        None => ReloadPathAgreement::Unavailable {
206            reason: if matches!(
207                image,
208                subc_control::RunningImageAgreement::Unavailable {
209                    reason: subc_control::RunningImageUnavailableReason::NotRunning
210                }
211            ) {
212                ReloadPathUnavailableReason::NotRunning
213            } else {
214                ReloadPathUnavailableReason::SpawnedPathUnavailable
215            },
216        },
217    };
218    PendingReloadVerdict {
219        path,
220        image,
221        spec: None,
222    }
223}
224
225fn launch_spec_agreement(
226    configured: &crate::supervise::ModuleSpec,
227    running: bool,
228    spawned: Option<&crate::supervise::LaunchSpecDigest>,
229) -> subc_control::LaunchSpecAgreement {
230    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
231    if !running {
232        return Agreement::Unavailable {
233            reason: Reason::NotRunning,
234        };
235    }
236    match spawned {
237        None => Agreement::Unavailable {
238            reason: Reason::SpawnedSpecUnavailable,
239        },
240        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
241            Agreement::Match
242        }
243        Some(_) => Agreement::Mismatch,
244    }
245}
246
247fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
248    if configured == spawned_from {
249        return true;
250    }
251    #[cfg(windows)]
252    {
253        // Windows spawn records the resolved verbatim path (\\?\...), while
254        // configuration keeps its original spelling. Resolve both with the OS
255        // instead of stripping prefixes or folding case, which could conflate
256        // distinct names in case-sensitive directories. Different resolved
257        // paths still differ even if their files have identical content.
258        match (
259            std::fs::canonicalize(configured),
260            std::fs::canonicalize(spawned_from),
261        ) {
262            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
263            _ => false,
264        }
265    }
266    #[cfg(not(windows))]
267    {
268        false
269    }
270}
271
272#[derive(Clone)]
273struct DaemonProvenanceFacts {
274    build: DaemonBuildProvenance,
275    pid: Option<u32>,
276    started_at_ms: Option<u64>,
277    start_clock: Option<crate::clock::StartClock>,
278    executable_path: Option<PathBuf>,
279    executable_identity: Option<SpawnedFileIdentity>,
280    process_start_time: Option<u64>,
281    probe: ExecutableIdentityProbe,
282}
283
284impl Default for DaemonProvenanceFacts {
285    fn default() -> Self {
286        Self {
287            build: DaemonBuildProvenance {
288                build_git_sha: None,
289                build_lock_digest: None,
290            },
291            pid: None,
292            started_at_ms: None,
293            start_clock: None,
294            executable_path: None,
295            executable_identity: None,
296            process_start_time: None,
297            probe: ExecutableIdentityProbe::default(),
298        }
299    }
300}
301
302#[derive(Debug, Clone)]
303struct SupervisorRescanContext {
304    supervisor: Supervisor,
305    config_path: PathBuf,
306    configured_port: Option<u16>,
307    storage_config: Option<crate::daemon_config::StorageConfig>,
308    admission_facts_carrier_module_id: Option<String>,
309    admission_facts_targets: Option<Vec<String>>,
310    scope_authority_owners: Vec<String>,
311    privileged_operator_prompt: PrivilegedOperatorPrompt,
312    /// Counts executed rescans that set out to change the supervised modules'
313    /// staged or applied configuration, counted before the first change so a
314    /// rescan that fails partway still counts. With the daemon incarnation it
315    /// names the state a `supervisor.rescan_checked` preview was computed
316    /// against.
317    generation: Arc<std::sync::atomic::AtomicU64>,
318}
319
320/// Refusal labels passed to `observe_route_open_refusal` that mean the target
321/// module is not serving right now, and so open or extend an outage in the
322/// route outage tracker. Every one of them is only reachable after the target
323/// was found in the registry, which is what keeps an arbitrary client-chosen
324/// id from ever creating tracker state.
325///
326/// Deliberately absent: `not_registered` and `removed` (the id may be
327/// anything a client sent, and a removed module is gone on purpose),
328/// `protocol_none` (such a module never serves routes, so nothing is out),
329/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
330/// capability and admission-facts refusals (they refuse the caller, not a
331/// module outage), and `relay_reservation_failed` (its code ranges over
332/// capacity limits as well as a vanished connection). Capacity, breaker,
333/// relay-timeout and module-rejection refusals do not pass through that
334/// function at all; the breaker logs its own transitions.
335///
336/// The two not-serving refusals that bypass that function record themselves
337/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
338/// `required_capability_unprovided` is not tracked: the module itself is up,
339/// and the outage belongs to the missing provider.
340const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
341    "reloading",
342    "supervisor_not_live",
343    "registration_not_active",
344    "no_forwarding_connection",
345    "relay_send_failed",
346];
347
348/// Real channel-0 control handler for subc itself.
349#[derive(Clone)]
350pub struct ControlHandler {
351    registry: Arc<Registry>,
352    activity: Option<crate::activity::ActivityLedger>,
353    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
354    forwarding: Arc<ForwardingTable>,
355    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
356    supervisor: SupervisorHandle,
357    subc_capabilities: Arc<[String]>,
358    /// Daemon-wide route.bind relay budget. Used as the fallback when the
359    /// target module has no per-module override in
360    /// `route_bind_relay_timeouts`.
361    route_bind_relay_timeout: Duration,
362    /// Per-module route.bind relay budget overrides, keyed by module id. When
363    /// `handle_route_open` resolves the deadline for a target module, a
364    /// per-module entry wins over the daemon-wide value above.
365    route_bind_relay_timeouts: BTreeMap<String, Duration>,
366    /// Per-target-module bind-relay breaker state. Shared with the forwarding
367    /// table, which is where a new module connection resets it.
368    route_bind_breakers: RouteBindBreakers,
369    /// Live relay admissions keyed by target module. Shared through the
370    /// forwarding table so cloned or separately built handlers enforce one cap.
371    route_bind_concurrency: RouteBindConcurrency,
372    /// Start and end of each module's not-serving period as seen by
373    /// `route.open`, so an outage gets one line at each edge instead of only
374    /// the per-refusal INFO lines. Taken from the forwarding table, so every
375    /// handler built over one table shares it.
376    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
377    /// Consecutive relay timeouts that open a module's breaker.
378    route_bind_breaker_threshold: u32,
379    /// How long a breaker stays open before one probe is admitted.
380    route_bind_breaker_cooldown: Duration,
381    health_probe_timeout: Duration,
382    /// Central storage policy. When set, each registering module receives its
383    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
384    storage_config: Option<crate::daemon_config::StorageConfig>,
385    /// The machine id established at boot, served on every HELLO_ACK and on
386    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
387    /// changes the file, never this value. `None` serves no id.
388    machine_id: Option<crate::machine_id::MachineId>,
389    admission_facts_carrier_module_id: Option<String>,
390    admission_facts_targets: Option<Vec<String>>,
391    /// Scope records with their sync authorities and tombstones; see
392    /// `crate::scopes`. Shared by clones of this handler, so every connection
393    /// reads and writes one table.
394    scopes: Arc<RwLock<ScopeTable>>,
395    /// Wakes the expiry loop when an accepted scope change may have added a
396    /// deadline, so the loop can wait without ticking while no live scope has
397    /// one. A permit is stored when nothing waits, so no wake is lost.
398    scope_deadline_added: Arc<tokio::sync::Notify>,
399    /// The configured `scope_authority_owners`, kept so a rescan can report a
400    /// changed value as needing a daemon restart; rescan never applies it.
401    scope_authority_owners: Vec<String>,
402    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
403    privileged_operator_prompt: PrivilegedOperatorPrompt,
404    /// The local and peer address of each accepted TCP client connection, read
405    /// from the daemon's own socket, so a privileged request can name its caller.
406    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
407    #[cfg(test)]
408    caller_lookup: Option<TestCallerLookup>,
409    /// The launch nonce each module connection presented at HELLO, which is how
410    /// a `scope.sync` is matched to the owner's current launch.
411    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
412    rescan: Option<SupervisorRescanContext>,
413    connected_clients: ConnectedClients,
414    counters: DaemonCounters,
415    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
416    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
417    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
418    /// their usual severity (see [`log_requirement_events`]). A handler built
419    /// without one has a flag that is never set.
420    daemon_shutdown: DaemonShutdownFlag,
421    daemon_provenance: DaemonProvenanceFacts,
422    #[cfg(test)]
423    control_dispatch_delay: Option<Duration>,
424    #[cfg(test)]
425    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
426    /// Holds `route.open` after the route.bind is queued to the module and
427    /// before it is marked enqueued, so a test can make the module's reply (or
428    /// a client teardown) land exactly inside that window.
429    #[cfg(test)]
430    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
431    /// Holds a HELLO after its registry entry is reserved and before its
432    /// forwarding endpoint is installed, so a test can read the catalog and
433    /// open routes while the module is registered but not yet routable. A
434    /// blocking barrier because the HELLO handler is synchronous; the test
435    /// runs the HELLO on its own thread.
436    #[cfg(test)]
437    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
438}
439
440impl fmt::Debug for ControlHandler {
441    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
442        f.debug_struct("ControlHandler")
443            .field("registry", &self.registry)
444            .field("forwarding", &self.forwarding)
445            .field("process_liveness", &self.process_liveness.is_some())
446            .field("supervisor", &self.supervisor)
447            .field("subc_capabilities", &self.subc_capabilities)
448            .finish()
449    }
450}
451
452struct RouteOpenRequest {
453    target: RouteTarget,
454    identity: BindIdentity,
455    consumer_identity: Option<ConsumerIdentity>,
456    consumer_capabilities: Option<Vec<String>>,
457    role_versions: Option<BTreeMap<String, String>>,
458    admission_facts: Option<serde_json::Value>,
459    scope: Option<ScopeSelector>,
460}
461
462#[derive(Clone, Copy)]
463enum RouteOpenPhase {
464    Admission,
465    ProjectRoot,
466    EgressReservation,
467    BindSend,
468    BindAckWait,
469}
470
471/// Store elapsed time for each route-open phase, including attempts that end
472/// in an early refusal or backend error. Admission also includes rechecks and
473/// commit bookkeeping performed between the other timed phases.
474struct RouteOpenTimings {
475    module_id: String,
476    principal: String,
477    admission: Duration,
478    project_root: Duration,
479    egress_reservation: Duration,
480    bind_send: Duration,
481    bind_ack_wait: Duration,
482    phase: Option<RouteOpenPhase>,
483    phase_started_at: Instant,
484}
485
486impl RouteOpenTimings {
487    fn new() -> Self {
488        Self {
489            module_id: String::new(),
490            // Refusals before attestation must not claim a verified principal.
491            principal: "unresolved".to_string(),
492            admission: Duration::ZERO,
493            project_root: Duration::ZERO,
494            egress_reservation: Duration::ZERO,
495            bind_send: Duration::ZERO,
496            bind_ack_wait: Duration::ZERO,
497            phase: Some(RouteOpenPhase::Admission),
498            phase_started_at: Instant::now(),
499        }
500    }
501
502    fn finish(&mut self) {
503        let elapsed = self.phase_started_at.elapsed();
504        match self.phase.take() {
505            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
506            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
507            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
508            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
509            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
510            None => {}
511        }
512    }
513
514    fn enter(&mut self, phase: RouteOpenPhase) {
515        self.finish();
516        self.phase = Some(phase);
517        self.phase_started_at = Instant::now();
518    }
519}
520
521struct RouteBindReservationGuard {
522    forwarding: Arc<ForwardingTable>,
523    endpoint: ModuleEndpointId,
524    relay_corr: u64,
525    armed: bool,
526}
527
528struct ModuleControlRpcGuard {
529    forwarding: Arc<ForwardingTable>,
530    endpoint: ModuleEndpointId,
531    corr: u64,
532    armed: bool,
533}
534
535impl ModuleControlRpcGuard {
536    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
537        Self {
538            forwarding,
539            endpoint,
540            corr,
541            armed: true,
542        }
543    }
544
545    fn disarm(&mut self) {
546        self.armed = false;
547    }
548}
549
550impl Drop for ModuleControlRpcGuard {
551    fn drop(&mut self) {
552        if self.armed {
553            let _ = self
554                .forwarding
555                .cancel_module_control_rpc(self.endpoint, self.corr);
556        }
557    }
558}
559
560impl RouteBindReservationGuard {
561    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
562        Self {
563            forwarding,
564            endpoint,
565            relay_corr,
566            armed: true,
567        }
568    }
569
570    fn release_and_disarm(&mut self) {
571        if !self.armed {
572            return;
573        }
574        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
575            self.endpoint,
576            self.relay_corr,
577            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
578        ) {
579            send_goodbye_target_best_effort(
580                &self.forwarding.counters(),
581                &target,
582                "canceled route.bind",
583            );
584        }
585        self.armed = false;
586    }
587
588    fn disarm(&mut self) {
589        self.armed = false;
590    }
591}
592
593impl Drop for RouteBindReservationGuard {
594    fn drop(&mut self) {
595        self.release_and_disarm();
596    }
597}
598
599/// Per-target-module circuit breaker around the `route.bind` relay.
600///
601/// The connection reader is serial per connection, so a module whose `on_bind`
602/// sits on the ack blocks every LATER frame on the connections that call it,
603/// including calls to unrelated modules. This does not make any module's bind
604/// fast; it stops the daemon paying the full budget again and again for a
605/// condition it has already observed.
606///
607/// State is keyed by TARGET MODULE and shared by every connection: a wedged
608/// module wedges everyone, so what one connection learned should protect the
609/// rest.
610///
611/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
612/// relay to that module has actually timed out, and is removed again when a
613/// relay is accepted or the module reconnects, so it cannot grow with traffic
614/// or with modules that behave.
615///
616/// # Why a `std` mutex here is not the head-of-line defect again
617///
618/// Acquisition never awaits. The critical section is a hash lookup plus a few
619/// integer updates, with no I/O and no `.await` inside it, so a reader task
620/// cannot be descheduled behind it the way it can behind
621/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
622/// primitive, held for the same kind of work, as the refusal counter this very
623/// path already increments.
624///
625/// It is also NOT on the data-plane splice path: only `route.open` and module
626/// registration touch it, so bound-route frames gain no state check and no
627/// contention.
628#[derive(Debug, Clone, Default)]
629pub(crate) struct RouteBindBreakers {
630    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
631}
632
633#[derive(Debug, Clone, Default)]
634pub(crate) struct RouteBindConcurrency {
635    modules: Arc<Mutex<HashMap<String, usize>>>,
636}
637
638struct RouteBindConcurrencyGuard {
639    concurrency: RouteBindConcurrency,
640    module_id: String,
641}
642
643impl RouteBindConcurrency {
644    /// Admit without waiting. Waiting here would move the bind stall from the
645    /// module reply to a semaphore and restore reader head-of-line blocking.
646    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
647        let mut modules = self
648            .modules
649            .lock()
650            .expect("route.bind concurrency mutex poisoned");
651        let in_flight = modules.entry(module_id.to_string()).or_default();
652        if *in_flight >= limit {
653            return Err(*in_flight);
654        }
655        *in_flight += 1;
656        Ok(RouteBindConcurrencyGuard {
657            concurrency: self.clone(),
658            module_id: module_id.to_string(),
659        })
660    }
661}
662
663impl Drop for RouteBindConcurrencyGuard {
664    fn drop(&mut self) {
665        let mut modules = self
666            .concurrency
667            .modules
668            .lock()
669            .expect("route.bind concurrency mutex poisoned");
670        let remove = {
671            let in_flight = modules
672                .get_mut(&self.module_id)
673                .expect("admitted route.bind has a concurrency entry");
674            *in_flight -= 1;
675            *in_flight == 0
676        };
677        if remove {
678            modules.remove(&self.module_id);
679        }
680    }
681}
682
683#[derive(Debug, Default)]
684struct ModuleBreakerState {
685    /// Relay timeouts observed with no accepted relay in between.
686    consecutive_timeouts: u32,
687    /// `Some` while the breaker is open: the instant the cooldown expires and
688    /// the next arrival may probe. `None` means closed.
689    cooldown_until: Option<Instant>,
690    /// A half-open probe has been admitted and has not settled yet. This is
691    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
692    /// that read the cooldown, so concurrent opens arriving at the moment the
693    /// cooldown expires cannot all decide that they are the probe.
694    probe_in_flight: Option<Arc<()>>,
695}
696
697/// What the breaker decided for one `route.open`, before any relay work.
698enum RouteBindAdmission<'a> {
699    Admitted {
700        guard: RouteBindBreakerGuard<'a>,
701        /// This open is the single half-open probe, so the transition is worth
702        /// one log line.
703        probe: bool,
704    },
705    Refused {
706        consecutive_timeouts: u32,
707        /// What is left of the cooldown. Zero when the refusal is because the
708        /// one probe is already in flight rather than because the cooldown has
709        /// not elapsed.
710        retry_in: Duration,
711        probe_in_flight: bool,
712    },
713}
714
715/// An outstanding admission, which must be told how its relay settled.
716///
717/// `Drop` settles it as inconclusive, so an early return between admission and
718/// the relay -- or the whole handler being cancelled when the client
719/// disconnects -- releases a half-open probe slot instead of leaving the
720/// breaker wedged half-open with no further probes.
721struct RouteBindBreakerGuard<'a> {
722    breakers: RouteBindBreakers,
723    module_id: &'a str,
724    probe_token: Option<Arc<()>>,
725    settled: bool,
726}
727
728impl RouteBindBreakerGuard<'_> {
729    /// The module answered within the budget and took the bind. THE ONLY
730    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
731    /// breaker, which is a transition worth logging.
732    fn record_accepted(&mut self) -> bool {
733        self.settled = true;
734        self.breakers.record_accepted(self.module_id)
735    }
736
737    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
738    /// COUNTS TOWARD OPENING.
739    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
740        self.settled = true;
741        self.breakers.record_timeout(
742            self.module_id,
743            self.probe_token.as_ref(),
744            threshold,
745            cooldown,
746        )
747    }
748
749    /// Everything else: the module REJECTED the bind, its connection went away
750    /// mid-relay, or the waiter was cancelled.
751    ///
752    /// None of these is evidence that a module is slow, and each already has
753    /// its own refusal with its own code. A module that rejects a bind in
754    /// microseconds is healthy and must never be convicted for it; a module
755    /// that died has said nothing about the module that replaces it. So these
756    /// neither increment nor reset the count -- they only release a probe slot.
757    fn record_inconclusive(&mut self) {
758        self.settled = true;
759        self.breakers
760            .record_inconclusive(self.module_id, self.probe_token.as_ref());
761    }
762}
763
764impl Drop for RouteBindBreakerGuard<'_> {
765    fn drop(&mut self) {
766        if !self.settled {
767            self.breakers
768                .record_inconclusive(self.module_id, self.probe_token.as_ref());
769        }
770    }
771}
772
773/// The breaker moved to open, reported so the caller can log it outside the
774/// lock. Opening is rare and load-bearing; the refusals that follow are
775/// frequent and are counted rather than logged.
776struct BreakerOpened {
777    consecutive_timeouts: u32,
778    /// True when a failed probe re-opened an already-open breaker, which reads
779    /// very differently in a log from a first opening.
780    reopened_after_probe: bool,
781}
782
783impl RouteBindBreakers {
784    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
785        self.modules
786            .lock()
787            .expect("route.bind breaker mutex poisoned")
788    }
789
790    /// Decide whether this `route.open` may attempt its relay. Takes the map
791    /// lock and nothing else, and never awaits.
792    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
793        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
794            probe: probe_token.is_some(),
795            guard: RouteBindBreakerGuard {
796                breakers: self.clone(),
797                module_id,
798                probe_token,
799                settled: false,
800            },
801        };
802
803        let mut modules = self.lock();
804        let Some(state) = modules.get_mut(module_id) else {
805            return admitted(None);
806        };
807        let Some(cooldown_until) = state.cooldown_until else {
808            return admitted(None);
809        };
810        if state.probe_in_flight.is_some() {
811            return RouteBindAdmission::Refused {
812                consecutive_timeouts: state.consecutive_timeouts,
813                retry_in: Duration::ZERO,
814                probe_in_flight: true,
815            };
816        }
817        let now = Instant::now();
818        if now < cooldown_until {
819            return RouteBindAdmission::Refused {
820                consecutive_timeouts: state.consecutive_timeouts,
821                retry_in: cooldown_until - now,
822                probe_in_flight: false,
823            };
824        }
825        let token = Arc::new(());
826        state.probe_in_flight = Some(Arc::clone(&token));
827        admitted(Some(token))
828    }
829
830    fn record_accepted(&self, module_id: &str) -> bool {
831        self.lock()
832            .remove(module_id)
833            .is_some_and(|state| state.cooldown_until.is_some())
834    }
835
836    fn record_timeout(
837        &self,
838        module_id: &str,
839        probe_token: Option<&Arc<()>>,
840        threshold: u32,
841        cooldown: Duration,
842    ) -> Option<BreakerOpened> {
843        let mut modules = self.lock();
844        let state = modules.entry(module_id.to_string()).or_default();
845        let was_open = state.cooldown_until.is_some();
846        let was_probe = Self::owns_probe(state, probe_token);
847        if was_probe {
848            state.probe_in_flight = None;
849        }
850        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
851        if state.consecutive_timeouts < threshold {
852            return None;
853        }
854        state.cooldown_until = Some(Instant::now() + cooldown);
855        Some(BreakerOpened {
856            consecutive_timeouts: state.consecutive_timeouts,
857            reopened_after_probe: was_open && was_probe,
858        })
859    }
860
861    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
862        // A relay can finish after the breaker was reset or after it opened
863        // again and started a new probe. Only the guard whose token matches the
864        // active probe may release it, so a late relay never frees a newer probe.
865        state
866            .probe_in_flight
867            .as_ref()
868            .zip(token)
869            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
870    }
871
872    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
873        if let Some(state) = self.lock().get_mut(module_id) {
874            if Self::owns_probe(state, probe_token) {
875                state.probe_in_flight = None;
876            }
877        }
878    }
879
880    /// Discard what was learned about a module, because the process it was
881    /// learned about is gone. Returns the discarded count when it was non-zero.
882    ///
883    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
884    /// `module_id` is a configuration identity that outlives any particular
885    /// child; what the breaker observed was the process behind the module
886    /// connection of the moment. When a new connection registers under that id
887    /// the verdict's subject no longer exists, so the verdict is stale by
888    /// construction rather than merely likely to be wrong. Keeping it would
889    /// apply a dead process's record to a live one, which is the same defect
890    /// class this breaker exists to stop the daemon committing.
891    ///
892    /// A half-open probe in flight is discarded with the rest: it was a
893    /// question about the old process.
894    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
895        self.lock()
896            .remove(module_id)
897            .map(|state| state.consecutive_timeouts)
898            .filter(|discarded| *discarded > 0)
899    }
900
901    /// Open breakers, for the `server.describe` counters object. `None` when
902    /// none is open, so the key stays absent rather than present-and-empty.
903    ///
904    /// This is the operator's answer to "is this module refusing instantly or
905    /// is it fine?", which look identical from a client that retries and then
906    /// succeeds.
907    fn open_snapshot(&self) -> Option<serde_json::Value> {
908        let now = Instant::now();
909        let modules = self.lock();
910        let open = modules
911            .iter()
912            .filter_map(|(module_id, state)| {
913                let cooldown_until = state.cooldown_until?;
914                Some((
915                    module_id.clone(),
916                    serde_json::json!({
917                        "consecutive_timeouts": state.consecutive_timeouts,
918                        "cooldown_remaining_ms":
919                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
920                        "probe_in_flight": state.probe_in_flight.is_some(),
921                    }),
922                ))
923            })
924            .collect::<serde_json::Map<String, serde_json::Value>>();
925        (!open.is_empty()).then_some(serde_json::Value::Object(open))
926    }
927}
928
929impl ControlHandler {
930    pub fn new(registry: Arc<Registry>) -> Self {
931        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
932    }
933
934    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
935        let counters = forwarding.counters();
936        // Taken from the forwarding table rather than created here, so that the
937        // breaker a `route.open` consults is the same one a module's
938        // registration resets, however many handlers are built over one table.
939        let route_bind_breakers = forwarding.route_bind_breakers();
940        let route_bind_concurrency = forwarding.route_bind_concurrency();
941        let route_outages = forwarding.route_outages();
942        Self {
943            registry,
944            activity: None,
945            activity_streams: Arc::new(Mutex::new(HashMap::new())),
946            forwarding,
947            process_liveness: None,
948            supervisor: SupervisorHandle::new(),
949            subc_capabilities: Arc::from([
950                CAP_MANIFEST_REGISTRATION.to_string(),
951                CAP_CHANNEL_LIFECYCLE.to_string(),
952                CAP_PING_PONG.to_string(),
953                CAP_SESSION_ATTACH.to_string(),
954                CAP_ADMISSION_FACTS_RELAY.to_string(),
955                CAP_SCOPES_V1.to_string(),
956                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
957                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
958                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
959                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
961                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
962                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
963                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
964            ]),
965            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
966            route_bind_relay_timeouts: BTreeMap::new(),
967            route_bind_breakers,
968            route_bind_concurrency,
969            route_outages,
970            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
971            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
972            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
973            storage_config: None,
974            machine_id: None,
975            admission_facts_carrier_module_id: None,
976            admission_facts_targets: None,
977            scopes: Arc::new(RwLock::new(ScopeTable::new(
978                crate::daemon_config::default_scope_authority_owners(),
979            ))),
980            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
981            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
982            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
983            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
984            #[cfg(test)]
985            caller_lookup: None,
986            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
987            rescan: None,
988            connected_clients: ConnectedClients::new(),
989            counters,
990            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
991            daemon_shutdown: DaemonShutdownFlag::default(),
992            daemon_provenance: DaemonProvenanceFacts::default(),
993            #[cfg(test)]
994            control_dispatch_delay: None,
995            #[cfg(test)]
996            provenance_probe_override: None,
997            #[cfg(test)]
998            route_bind_relay_sent_hold: None,
999            #[cfg(test)]
1000            hello_registry_to_forwarding_hold: None,
1001        }
1002    }
1003
1004    /// Set the central storage policy: registering modules then receive their
1005    /// resolved storage descriptor in HELLO_ACK.
1006    pub fn with_storage_config(
1007        mut self,
1008        storage_config: Option<crate::daemon_config::StorageConfig>,
1009    ) -> Self {
1010        self.storage_config = storage_config;
1011        self
1012    }
1013
1014    /// Open the ledger on its own writer thread; an unavailable sink does not
1015    /// prevent the daemon from serving routing or supervisor requests.
1016    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1017        self.activity = Some(crate::activity::ActivityLedger::open(
1018            run_dir.join("activity.jsonl"),
1019            incarnation,
1020        ));
1021        self.supervisor.set_activity(self.activity.clone());
1022        self
1023    }
1024
1025    /// Set the machine id served to every registering module (HELLO_ACK) and on
1026    /// `server.describe`.
1027    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1028        self.machine_id = machine_id;
1029        self
1030    }
1031
1032    /// Configure the exact reserved module and target ids permitted to relay
1033    /// opaque admission facts. Config-file loading validates this authority;
1034    /// this builder keeps the same policy available to embedded test daemons.
1035    pub fn with_admission_facts_config(
1036        mut self,
1037        carrier_module_id: Option<String>,
1038        targets: Option<Vec<String>>,
1039    ) -> Self {
1040        self.admission_facts_carrier_module_id = carrier_module_id;
1041        self.admission_facts_targets = targets;
1042        self
1043    }
1044
1045    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1046    /// Replaces the scope table with an empty one under the new list, so call it
1047    /// while building the handler, before any module can sync.
1048    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1049        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1050        self.scope_authority_owners = owners;
1051        self
1052    }
1053
1054    /// Set the startup policy before installing the rescan context or serving requests.
1055    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1056        self.privileged_operator_prompt = mode;
1057        self
1058    }
1059
1060    /// Override the route.bind relay timeout. Used by tests that assert the
1061    /// timeout path so they don't block on the production-safe default.
1062    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1063        self.route_bind_relay_timeout = timeout;
1064        self
1065    }
1066
1067    /// Install per-module route.bind relay budget overrides. A module id
1068    /// listed here wins over the daemon-wide default set via
1069    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1070    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1071    /// the same `Duration` the bind path will use.
1072    pub fn with_route_bind_relay_timeouts(
1073        mut self,
1074        timeouts: impl IntoIterator<Item = (String, Duration)>,
1075    ) -> Self {
1076        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1077        self
1078    }
1079
1080    /// Resolve the route.bind relay budget for a specific target module id.
1081    /// Per-module overrides win; the daemon-wide value (set via
1082    /// `with_route_bind_relay_timeout` or the built-in default) is the
1083    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1084    /// the same resolution `handle_route_open` will use.
1085    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1086        self.route_bind_relay_timeouts
1087            .get(module_id)
1088            .copied()
1089            .unwrap_or(self.route_bind_relay_timeout)
1090    }
1091
1092    /// Override the per-module bind-relay breaker policy.
1093    ///
1094    /// Used by tests, which cannot spend three production budgets opening a
1095    /// breaker or twenty seconds waiting for its cooldown. The production
1096    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1097    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1098    /// reasoning for the numbers.
1099    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1100        self.route_bind_breaker_threshold = threshold.max(1);
1101        self.route_bind_breaker_cooldown = cooldown;
1102        self
1103    }
1104
1105    #[cfg(test)]
1106    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1107        self.health_probe_timeout = timeout;
1108        self
1109    }
1110
1111    #[cfg(test)]
1112    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1113        self.control_dispatch_delay = Some(delay);
1114        self
1115    }
1116
1117    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1118    /// announce it is parked, once to be released. Build the barrier for two
1119    /// parties, the test being the other one.
1120    #[cfg(test)]
1121    pub(crate) fn with_route_bind_relay_sent_hold(
1122        mut self,
1123        hold: Arc<tokio::sync::Barrier>,
1124    ) -> Self {
1125        self.route_bind_relay_sent_hold = Some(hold);
1126        self
1127    }
1128
1129    /// A HELLO waits on `hold` twice between reserving its registry entry and
1130    /// installing its forwarding endpoint: once to announce it is parked, once
1131    /// to be released. Build the barrier for two parties, the test being the
1132    /// other one, and run the HELLO on a thread of its own.
1133    #[cfg(test)]
1134    pub(crate) fn with_hello_registry_to_forwarding_hold(
1135        mut self,
1136        hold: Arc<std::sync::Barrier>,
1137    ) -> Self {
1138        self.hello_registry_to_forwarding_hold = Some(hold);
1139        self
1140    }
1141
1142    #[cfg(test)]
1143    fn hold_between_registry_and_forwarding(&self) {
1144        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1145            hold.wait();
1146            hold.wait();
1147        }
1148    }
1149
1150    pub fn with_process_liveness(
1151        mut self,
1152        process_liveness: Arc<dyn ModuleProcessLiveness>,
1153    ) -> Self {
1154        self.process_liveness = Some(process_liveness);
1155        self
1156    }
1157
1158    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1159        self.supervisor = supervisor;
1160        self.supervisor.set_activity(self.activity.clone());
1161        self
1162    }
1163
1164    pub fn with_daemon_provenance(
1165        mut self,
1166        pid: u32,
1167        started_at_ms: u64,
1168        executable_path: Option<PathBuf>,
1169        build_git_sha: Option<String>,
1170        build_lock_digest: Option<String>,
1171    ) -> Self {
1172        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1173        let process_start_time = process_start_time(pid);
1174        self.daemon_provenance = DaemonProvenanceFacts {
1175            build: DaemonBuildProvenance {
1176                build_git_sha,
1177                build_lock_digest,
1178            },
1179            pid: Some(pid),
1180            started_at_ms: Some(started_at_ms),
1181            start_clock: None,
1182            executable_path,
1183            executable_identity,
1184            process_start_time,
1185            probe: ExecutableIdentityProbe::default(),
1186        };
1187        self
1188    }
1189
1190    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1191        self.daemon_provenance.start_clock = Some(clock);
1192        self
1193    }
1194
1195    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1196    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1197        self.scopes
1198            .write()
1199            .unwrap_or_else(|poisoned| poisoned.into_inner())
1200            .set_wall_clock(Arc::new(clock));
1201        self
1202    }
1203
1204    #[cfg(test)]
1205    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1206        self.provenance_probe_override = Some(result);
1207        self
1208    }
1209
1210    /// Install the configured module set and its reserved capability bindings.
1211    /// Bindings are configuration-scoped and may point at a provider that has not
1212    /// been installed yet, so this does not require the bound module to exist.
1213    pub fn with_capability_config(
1214        self,
1215        modules: impl IntoIterator<Item = (String, bool)>,
1216        reserved_capabilities: BTreeMap<String, String>,
1217    ) -> Self {
1218        self.capability_evaluator
1219            .configure(modules, reserved_capabilities);
1220        self
1221    }
1222
1223    /// Share the supervisor's daemon-shutdown flag with this handler, so
1224    /// requirement changes after the daemon has begun its announced shutdown
1225    /// are not reported as faults.
1226    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1227        self.daemon_shutdown = flag;
1228        self
1229    }
1230
1231    pub fn with_supervisor_rescan(
1232        mut self,
1233        supervisor: Supervisor,
1234        config_path: impl Into<PathBuf>,
1235        configured_port: Option<u16>,
1236    ) -> Self {
1237        self.rescan = Some(SupervisorRescanContext {
1238            supervisor,
1239            config_path: config_path.into(),
1240            configured_port,
1241            storage_config: self.storage_config.clone(),
1242            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1243            admission_facts_targets: self.admission_facts_targets.clone(),
1244            scope_authority_owners: self.scope_authority_owners.clone(),
1245            privileged_operator_prompt: self.privileged_operator_prompt,
1246            generation: Arc::default(),
1247        });
1248        self
1249    }
1250
1251    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1252        self.connected_clients = connected_clients;
1253        self
1254    }
1255
1256    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1257        Arc::clone(&self.forwarding)
1258    }
1259
1260    pub(crate) fn counters(&self) -> DaemonCounters {
1261        self.counters.clone()
1262    }
1263
1264    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1265    /// requirement event without depending on an operator polling a status command.
1266    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1267        tokio::spawn(async move {
1268            loop {
1269                self.capability_evaluator
1270                    .wait_for_change_or_deadline()
1271                    .await;
1272                self.refresh_capability_requirements();
1273            }
1274        });
1275    }
1276
1277    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1278    /// deadline passed during sleep is enforced on the first tick after wake.
1279    /// While no live scope carries a deadline the loop waits for an accepted
1280    /// scope change instead of ticking, so a daemon without deadlines never
1281    /// wakes for expiry.
1282    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1283        tokio::spawn(async move {
1284            loop {
1285                let has_deadlines = self
1286                    .scopes
1287                    .read()
1288                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1289                    .has_deadlines();
1290                if !has_deadlines {
1291                    self.scope_deadline_added.notified().await;
1292                    continue;
1293                }
1294                tokio::time::sleep(Duration::from_secs(1)).await;
1295                if let Err(error) = self.sweep_expired_scopes() {
1296                    warn!(%error, "scope expiry sweep failed");
1297                }
1298            }
1299        });
1300    }
1301
1302    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1303        let mut table = self
1304            .scopes
1305            .write()
1306            .unwrap_or_else(|poisoned| poisoned.into_inner());
1307        let swept = table.sweep_expired();
1308        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1309        drop(table);
1310        self.close_scope_drained_routes(drained);
1311        Ok(())
1312    }
1313
1314    fn publish_scope_changes(
1315        &self,
1316        changes: &[crate::scopes::ScopeTagChange],
1317        expired: &[crate::scopes::ScopeExpired],
1318    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1319        if changes.is_empty() {
1320            return Ok(Vec::new());
1321        }
1322        let drained = self
1323            .forwarding
1324            .publish_scope_changes(changes)
1325            .map_err(RouterError::Forwarding)?;
1326        let mut counts = HashMap::new();
1327        for route in &drained {
1328            *counts
1329                .entry((
1330                    route.scope.owner.as_str(),
1331                    route.scope.scope_ref.as_str(),
1332                    route.scope.tag.scope_epoch,
1333                ))
1334                .or_insert(0usize) += 1;
1335        }
1336        for scope in expired {
1337            let routes_closed = counts
1338                .get(&(
1339                    scope.owner.as_str(),
1340                    scope.scope_ref.as_str(),
1341                    scope.scope_epoch,
1342                ))
1343                .copied()
1344                .unwrap_or(0);
1345            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1346                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1347                routes_closed, "scope expired");
1348        }
1349        Ok(drained)
1350    }
1351
1352    fn runtime_capability_snapshot(
1353        &self,
1354    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1355        let runtime = self
1356            .supervisor
1357            .list()
1358            .into_iter()
1359            .map(|module| {
1360                let status = module.status().map_err(|err| {
1361                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1362                })?;
1363                Ok(RuntimeModule {
1364                    module_id: status.module_id,
1365                    state: status.state,
1366                    enabled: status.enabled,
1367                })
1368            })
1369            .collect::<Result<Vec<_>, RouterError>>()?;
1370        // Include every registration, published or not. The capability
1371        // evaluator counts a module as providing its capabilities as soon as it
1372        // has registered, even before `catalog.list` shows it, so a HELLO in
1373        // progress does not change other modules' required-capability
1374        // verdicts. See `Registry::list_registered_modules`.
1375        let registrations = self.registry.list_registered_modules().map_err(|err| {
1376            RouterError::backend(
1377                0,
1378                0,
1379                format!("failed to list capability registrations: {err}"),
1380            )
1381        })?;
1382        let registrations = registrations
1383            .into_iter()
1384            .map(|registration| RegisteredModule {
1385                module_id: registration.manifest.module_id,
1386                module_version: registration.manifest.module_version,
1387                capabilities: registration.manifest.capabilities,
1388            })
1389            .collect();
1390        Ok((runtime, registrations))
1391    }
1392
1393    /// The capability side effects of a module becoming the active registration
1394    /// for its id: cache its manifest (warning if its claims drifted), run the
1395    /// deny census when its declarations call for one, and recompute the
1396    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1397    /// candidate's does not, and the supervisor does it at promotion instead,
1398    /// through [`crate::supervise::SwapPromotionObserver`].
1399    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1400        let cached_registration = RegisteredModule {
1401            module_id: registration.manifest.module_id.clone(),
1402            module_version: registration.manifest.module_version.clone(),
1403            capabilities: registration.manifest.capabilities.clone(),
1404        };
1405        if self.capability_evaluator.record_hello(&cached_registration) {
1406            warn!(
1407                module_id = %cached_registration.module_id,
1408                "capability claims drifted from the cached manifest"
1409            );
1410        }
1411        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1412            self.enforce_capability_denies();
1413        }
1414        self.refresh_capability_requirements();
1415    }
1416
1417    /// Point the shared supervisor handle at this handler for swap promotions.
1418    /// Called wherever a handler is put behind the `Arc` the router serves, so
1419    /// it can be held weakly.
1420    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1421        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1422            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1423        self.supervisor.set_swap_promotion_observer(observer);
1424    }
1425
1426    pub fn refresh_capability_requirements(&self) {
1427        match self.runtime_capability_snapshot() {
1428            Ok((runtime, registrations)) => {
1429                log_requirement_events(
1430                    self.capability_evaluator
1431                        .evaluate_now(&runtime, &registrations),
1432                    &self.daemon_shutdown,
1433                );
1434            }
1435            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1436        }
1437    }
1438
1439    /// Reconcile only live, attested route bindings after a capability deny edge
1440    /// or target claim was added. This is deliberately a control-plane census:
1441    /// the opaque forwarding hot path must not grow a per-frame capability check.
1442    fn enforce_capability_denies(&self) {
1443        let registrations = match self.registry.list_registered_modules() {
1444            Ok(snapshot) => snapshot,
1445            Err(err) => {
1446                warn!(error = %err, "failed to read registrations for capability deny census");
1447                return;
1448            }
1449        };
1450        let manifests = registrations
1451            .into_iter()
1452            .map(|registration| {
1453                (
1454                    registration.manifest.module_id.clone(),
1455                    registration.manifest,
1456                )
1457            })
1458            .collect::<BTreeMap<_, _>>();
1459        let census = match self.forwarding.route_census(None) {
1460            Ok(census) => census,
1461            Err(err) => {
1462                warn!(error = %err, "failed to read route census for capability deny enforcement");
1463                return;
1464            }
1465        };
1466
1467        for (target_module_id, routes) in census {
1468            let Some(target_manifest) = manifests.get(&target_module_id) else {
1469                continue;
1470            };
1471            let mut closed_routes = Vec::new();
1472            let mut module_goodbyes = Vec::new();
1473            for route in routes {
1474                let Principal::Reserved {
1475                    module_id: opening_module_id,
1476                } = &route.principal
1477                else {
1478                    continue;
1479                };
1480                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1481                    continue;
1482                };
1483                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1484                    continue;
1485                };
1486
1487                match self.forwarding.release_client_route(
1488                    route.goodbye_target.connection_id,
1489                    route.goodbye_target.channel,
1490                    route.goodbye_target.epoch,
1491                ) {
1492                    Ok(RouteRelease::Removed(module_goodbye)) => {
1493                        warn!(
1494                            opening_module_id,
1495                            target_module_id,
1496                            capability,
1497                            "force-closing route because an attested capability deny edge now matches"
1498                        );
1499                        closed_routes.push(route);
1500                        module_goodbyes.push(module_goodbye);
1501                    }
1502                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1503                    Err(err) => warn!(
1504                        opening_module_id,
1505                        target_module_id,
1506                        capability,
1507                        error = %err,
1508                        "failed to force-close capability-denied route"
1509                    ),
1510                }
1511            }
1512
1513            if closed_routes.is_empty() {
1514                continue;
1515            }
1516            send_route_control_pushes(
1517                &self.forwarding,
1518                closed_routes,
1519                ClientControlPush::RouteClosed {
1520                    module_id: target_module_id,
1521                    channels: Vec::new(),
1522                    reason: RouteCloseReason::CapabilityDenied,
1523                    drained: false,
1524                    abandoned: 0,
1525                    excluded_subscriptions: 0,
1526                    terminal: Some(false),
1527                },
1528            );
1529            self.emit_route_goodbyes(module_goodbyes);
1530        }
1531    }
1532
1533    /// Why a registered module is not accepting new route binds, or `None` when
1534    /// it is. This is the module's effective readiness: its declared readiness
1535    /// first, then every `need: required` capability it declares evaluating to
1536    /// `provided`. `route.open` and `catalog.list` both read it here so the
1537    /// catalog never reports a module routable that `route.open` would refuse.
1538    fn not_ready_reason(
1539        &self,
1540        registration: &crate::registry::ModuleRegistration,
1541    ) -> Option<NotReadyReason> {
1542        if !registration.ready {
1543            return Some(NotReadyReason {
1544                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1545                capability: None,
1546            });
1547        }
1548        self.first_unprovided_required_capability(registration)
1549            .map(|capability| NotReadyReason {
1550                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1551                capability: Some(capability),
1552            })
1553    }
1554
1555    /// The lexicographically first capability this registration declares
1556    /// `need: required` whose evaluator verdict is not `provided`.
1557    ///
1558    /// The verdicts are the capability evaluator's own; nothing here decides
1559    /// what "provided" means. The evaluator counts a capability provided as
1560    /// soon as a module claiming it has REGISTERED, not once that module is
1561    /// ready. That distinction is what keeps two modules that require each
1562    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1563    /// is ready", each would wait for the other to become ready first and
1564    /// neither ever would. Do not tighten it to readiness.
1565    ///
1566    /// A required capability with no verdict at all means this registration's
1567    /// HELLO or catalog.update landed after the last recompute; recompute once
1568    /// rather than let a missing verdict read as either answer. If it is still
1569    /// missing (the recompute itself failed) the capability counts as
1570    /// unprovided: the refusal is retryable, and routing a module whose
1571    /// required provider is unknown is the outcome this check exists to stop.
1572    fn first_unprovided_required_capability(
1573        &self,
1574        registration: &crate::registry::ModuleRegistration,
1575    ) -> Option<String> {
1576        let required = registration
1577            .manifest
1578            .capabilities
1579            .iter()
1580            .flat_map(|declarations| declarations.requires.iter())
1581            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1582            .map(|requirement| requirement.capability.as_str())
1583            .collect::<BTreeSet<_>>();
1584        if required.is_empty() {
1585            return None;
1586        }
1587        let module_id = registration.manifest.module_id.as_str();
1588        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1589        if required
1590            .iter()
1591            .any(|capability| verdict(capability).is_none())
1592        {
1593            self.refresh_capability_requirements();
1594        }
1595        required
1596            .into_iter()
1597            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1598            .map(str::to_string)
1599    }
1600
1601    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1602        self.capability_evaluator
1603            .statuses()
1604            .into_iter()
1605            .map(capability_requirement_status)
1606            .collect()
1607    }
1608
1609    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1610    /// registration-release watch. The signal is what the supervisor waits on
1611    /// before spawning a replacement, so it must only fire once forwarding
1612    /// teardown is also done (see [`Self::cleanup_connection`] /
1613    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1614    /// admitted but its forwarding endpoint could not be installed.
1615    fn deregister_connection(
1616        &self,
1617        connection_id: ConnectionId,
1618        reason: RegistrationEndReason,
1619    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1620        self.registry
1621            .deregister_connection_with_reason(connection_id, reason)
1622    }
1623
1624    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1625        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1626            return None;
1627        }
1628        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1629            parse_client_control_request(&frame.body)
1630        else {
1631            return None;
1632        };
1633        Some(target_module_id(&target).to_string())
1634    }
1635
1636    pub(crate) fn route_open_capacity_refusal(
1637        &self,
1638        ctx: &RouteCtx,
1639        frame: &Frame,
1640        target_module_id: &str,
1641        in_flight: usize,
1642        limit: usize,
1643    ) -> Result<Frame, RouterError> {
1644        self.route_open_admission_refusal_frame(
1645            ctx,
1646            frame,
1647            target_module_id,
1648            "open_admission_full",
1649            (in_flight, limit),
1650            format!(
1651                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1652            ),
1653        )
1654    }
1655
1656    fn route_open_target_capacity_refusal(
1657        &self,
1658        ctx: &RouteCtx,
1659        frame: &Frame,
1660        target_module_id: &str,
1661        in_flight: usize,
1662    ) -> Result<Frame, RouterError> {
1663        self.route_open_admission_refusal_frame(
1664            ctx,
1665            frame,
1666            target_module_id,
1667            "target_binds_full",
1668            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1669            format!(
1670                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1671            ),
1672        )
1673    }
1674
1675    /// Admission pressure clears as existing binds settle, so its refusal must
1676    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1677    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1678    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1679    /// cannot currently reach its target; `module_timeout` would falsely claim
1680    /// that a wait expired. A new, cleaner code would be terminal to deployed
1681    /// clients, so it requires a client-tolerance rollout before daemon emission.
1682    fn route_open_admission_refusal_frame(
1683        &self,
1684        ctx: &RouteCtx,
1685        frame: &Frame,
1686        target_module_id: &str,
1687        reason: &'static str,
1688        (in_flight, limit): (usize, usize),
1689        message: impl Into<String>,
1690    ) -> Result<Frame, RouterError> {
1691        let code = error_codes::TARGET_UNAVAILABLE;
1692        self.counters.increment_route_open_refused(code);
1693        info!(
1694            target: "control",
1695            code,
1696            reason,
1697            module_id = ?target_module_id,
1698            connection_id = ctx.connection_id.get(),
1699            in_flight,
1700            limit,
1701            "route.open refused"
1702        );
1703        control_error_frame(frame, code, message.into())
1704    }
1705
1706    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1707    ///
1708    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1709    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1710    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1711    #[cfg(test)]
1712    pub fn handle_control(
1713        &self,
1714        connection_id: ConnectionId,
1715        frame: Frame,
1716    ) -> Result<Vec<Frame>, RouterError> {
1717        match frame.header.ty {
1718            FrameType::Ping => Ok(vec![pong(&frame)?]),
1719            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1720            FrameType::Goodbye => self.handle_goodbye(connection_id),
1721            ty => Ok(vec![control_error_frame(
1722                &frame,
1723                "unsupported_control_frame",
1724                format!("unsupported channel-0 frame {ty:?}"),
1725            )?]),
1726        }
1727    }
1728
1729    pub async fn handle_control_frame(
1730        &self,
1731        ctx: &RouteCtx,
1732        frame: Frame,
1733    ) -> Result<Vec<Frame>, RouterError> {
1734        self.handle_control_frame_timed(ctx, frame, None).await
1735    }
1736
1737    pub(crate) fn record_caller_endpoints(
1738        &self,
1739        connection: ConnectionId,
1740        endpoints: Option<(SocketAddr, SocketAddr)>,
1741    ) {
1742        let mut callers = self
1743            .caller_endpoints
1744            .lock()
1745            .unwrap_or_else(|p| p.into_inner());
1746        callers.remove(&connection);
1747        if let Some(endpoints) = endpoints {
1748            callers.insert(connection, endpoints);
1749        }
1750    }
1751
1752    #[cfg(test)]
1753    pub(crate) fn with_caller_lookup_for_test(
1754        mut self,
1755        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1756    ) -> Self {
1757        self.caller_lookup = Some(Arc::new(lookup));
1758        self
1759    }
1760
1761    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1762        let endpoints = self
1763            .caller_endpoints
1764            .lock()
1765            .unwrap_or_else(|p| p.into_inner())
1766            .get(&connection)
1767            .copied();
1768        #[cfg(test)]
1769        let lookup = self.caller_lookup.clone();
1770        tokio::task::spawn_blocking(move || {
1771            #[cfg(test)]
1772            if let Some(lookup) = lookup {
1773                return lookup(endpoints);
1774            }
1775            endpoints
1776                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1777                .unwrap_or_default()
1778        })
1779        .await
1780        .unwrap_or_default()
1781    }
1782
1783    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1784        frame.header.channel == 0
1785            && frame.header.ty == FrameType::Request
1786            && self
1787                .forwarding
1788                .module_endpoint_for_connection(ctx.connection_id)
1789                .is_ok_and(|endpoint| endpoint.is_none())
1790            && parse_client_control_request(&frame.body)
1791                .ok()
1792                .is_some_and(|request| privileged_operation(&request).is_some())
1793    }
1794
1795    pub(crate) async fn handle_control_frame_timed(
1796        &self,
1797        ctx: &RouteCtx,
1798        frame: Frame,
1799        dispatch_started_at: Option<StdInstant>,
1800    ) -> Result<Vec<Frame>, RouterError> {
1801        match frame.header.ty {
1802            FrameType::Ping => Ok(vec![pong(&frame)?]),
1803            FrameType::Hello => {
1804                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1805            }
1806            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1807            FrameType::Cancel => {
1808                // A Cancel on channel 0 names a waiting operator.confirm or a
1809                // subscription. Successful cancellation has no ordinary reply.
1810                if self
1811                    .forwarding
1812                    .operator_confirms()
1813                    .cancel(ctx.connection_id, frame.header.corr)
1814                    || self
1815                        .supervisor
1816                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1817                    || self.cancel_activity_subscription(ctx, &frame)?
1818                {
1819                    Ok(Vec::new())
1820                } else {
1821                    Ok(vec![control_error_frame(
1822                        &frame,
1823                        "unknown_subscription",
1824                        "no subscription has this correlation id",
1825                    )?])
1826                }
1827            }
1828            FrameType::Request => {
1829                // This additive operation is not part of the existing exhaustive
1830                // module-control enum. Probe the op before decoding that enum.
1831                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1832                if op
1833                    .as_ref()
1834                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1835                {
1836                    return self.handle_activity_record(ctx, frame).await;
1837                }
1838                if op.as_ref().is_some_and(|probe| {
1839                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1840                }) {
1841                    return self.handle_activity_read(ctx, frame).await;
1842                }
1843                if op
1844                    .as_ref()
1845                    .is_some_and(|probe| probe.op == "operator.confirm")
1846                {
1847                    return self.handle_operator_confirm(ctx, frame);
1848                }
1849                if op
1850                    .as_ref()
1851                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1852                {
1853                    return self.handle_scope_served_by(ctx.connection_id, frame);
1854                }
1855                if self
1856                    .forwarding
1857                    .module_endpoint_for_connection(ctx.connection_id)
1858                    .map_err(RouterError::Forwarding)?
1859                    .is_some()
1860                {
1861                    if !is_known_module_request_op(&frame.body) {
1862                        return Ok(vec![control_error_frame(
1863                            &frame,
1864                            "unsupported_control_frame",
1865                            "module-originated channel-0 REQUEST is not supported",
1866                        )?]);
1867                    }
1868                    let request = match parse_module_control_request_from_module(&frame.body) {
1869                        Ok(request) => request,
1870                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1871                            return Ok(vec![control_error_frame(
1872                                &frame,
1873                                "unsupported_control_frame",
1874                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1875                            )?])
1876                        }
1877                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1878                            return Ok(vec![control_error_frame(
1879                                &frame,
1880                                "invalid_control_body",
1881                                format!("malformed module control body: {err}"),
1882                            )?])
1883                        }
1884                    };
1885                    let op = module_control_request_op(&request);
1886                    let corr = frame.header.corr;
1887                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1888                    let result =
1889                        self.handle_module_control_request(ctx.connection_id, frame, request);
1890                    log_slow_control_dispatch(
1891                        dispatch_started_at,
1892                        op,
1893                        ctx.connection_id,
1894                        corr,
1895                        None,
1896                    );
1897                    return result;
1898                }
1899
1900                if is_known_module_request_op(&frame.body) {
1901                    return Ok(vec![control_error_frame(
1902                        &frame,
1903                        "not_registered",
1904                        "catalog.update requires an active module registration owned by this connection",
1905                    )?]);
1906                }
1907
1908                let request = match parse_client_control_request(&frame.body) {
1909                    Ok(request) => request,
1910                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1911                        return Ok(vec![control_error_frame(
1912                            &frame,
1913                            "unknown_control_op",
1914                            format!("unknown client control op: {err}"),
1915                        )?])
1916                    }
1917                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1918                        return Ok(vec![control_error_frame(
1919                            &frame,
1920                            "invalid_control_body",
1921                            format!("malformed client control body: {err}"),
1922                        )?])
1923                    }
1924                };
1925                let op = client_control_request_op(&request);
1926                let corr = frame.header.corr;
1927                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1928                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1929                #[cfg(test)]
1930                if let Some(delay) = self.control_dispatch_delay {
1931                    tokio::time::sleep(delay).await;
1932                }
1933                let result = self
1934                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1935                    .await;
1936                if let Some(timings) = route_open_timings.as_mut() {
1937                    timings.finish();
1938                }
1939                log_slow_control_dispatch(
1940                    dispatch_started_at,
1941                    op,
1942                    ctx.connection_id,
1943                    corr,
1944                    route_open_timings.as_ref(),
1945                );
1946                result
1947            }
1948            FrameType::Push => {
1949                let Some(endpoint) = self
1950                    .forwarding
1951                    .module_endpoint_for_connection(ctx.connection_id)
1952                    .map_err(RouterError::Forwarding)?
1953                else {
1954                    return Ok(vec![control_error_frame(
1955                        &frame,
1956                        "unsupported_control_frame",
1957                        "client-originated channel-0 PUSH is not supported",
1958                    )?]);
1959                };
1960                self.handle_status_update(endpoint, frame)
1961            }
1962            FrameType::Response | FrameType::Error
1963                if self
1964                    .forwarding
1965                    .module_endpoint_for_connection(ctx.connection_id)
1966                    .map_err(RouterError::Forwarding)?
1967                    .is_some() =>
1968            {
1969                self.handle_module_relay_response(ctx.connection_id, frame)
1970            }
1971            ty => Ok(vec![control_error_frame(
1972                &frame,
1973                "unsupported_control_frame",
1974                format!("unsupported channel-0 frame {ty:?}"),
1975            )?]),
1976        }
1977    }
1978
1979    pub fn cleanup_connection(
1980        &self,
1981        connection_id: ConnectionId,
1982    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1983        self.caller_endpoints
1984            .lock()
1985            .unwrap_or_else(|p| p.into_inner())
1986            .remove(&connection_id);
1987        self.forwarding
1988            .operator_confirms()
1989            .connection_closed(connection_id);
1990        self.cleanup_connection_with_end_reason(
1991            connection_id,
1992            RegistrationEndReason::ConnectionClosed,
1993        )
1994    }
1995
1996    fn cleanup_connection_with_end_reason(
1997        &self,
1998        connection_id: ConnectionId,
1999        requested_reason: RegistrationEndReason,
2000    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2001        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2002            self.registry
2003                .get_module_by_connection(connection_id)?
2004                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2005                .and_then(|module| module.registration_end_reason().ok().flatten())
2006                .unwrap_or(requested_reason)
2007        } else {
2008            requested_reason
2009        };
2010        let crash_closed = self
2011            .registry
2012            .get_module_by_connection(connection_id)?
2013            .and_then(|registration| {
2014                self.forwarding
2015                    .module_endpoint_for_connection(connection_id)
2016                    .ok()
2017                    .flatten()
2018                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2019                    .map(|routes| (registration.manifest.module_id, routes))
2020            });
2021        let crash_closed = crash_closed.map(|(module_id, routes)| {
2022            let terminal = match self.supervisor.get(&module_id) {
2023                None => false,
2024                Some(module) => match module.will_recover_after_connection_loss() {
2025                    Ok(will_recover) => !will_recover,
2026                    Err(err) => {
2027                        warn!(
2028                            %module_id,
2029                            error = %err,
2030                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2031                        );
2032                        false
2033                    }
2034                },
2035            };
2036            // The forwarding table gates all providers at the start of daemon
2037            // shutdown, before their connections are closed. An ordinary
2038            // module disconnect still reports crash if that gate is not set.
2039            let (reason, drained) = match self
2040                .forwarding
2041                .daemon_drain_result_for_connection(connection_id)
2042            {
2043                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2044                Ok(None) => (RouteCloseReason::Crash, false),
2045                Err(err) => {
2046                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2047                    (RouteCloseReason::Crash, false)
2048                }
2049            };
2050            (module_id, routes, reason, drained, terminal)
2051        });
2052        let registrations = self.deregister_connection(connection_id, end_reason);
2053        let cleanup = if crash_closed.is_some() {
2054            self.forwarding.cleanup_connection_counted(connection_id)
2055        } else {
2056            // A module connection's teardown needs the count of abandoned
2057            // route.bind relays for its route.closed notice below. Any other
2058            // connection, such as a client's, sends no such notice and needs
2059            // only its routes released, so it uses the route-only wrapper and
2060            // reports zero.
2061            self.forwarding
2062                .cleanup_connection(connection_id)
2063                .map(|released| crate::forwarding::ConnectionCleanup {
2064                    released,
2065                    abandoned_relays: 0,
2066                })
2067        };
2068        // The route.closed push waits for forwarding teardown because only
2069        // teardown knows how many pending route.bind relays it aborted. It still
2070        // goes out before the GOODBYEs for the released routes, and its targets
2071        // were captured above, before teardown removed those routes.
2072        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2073            let abandoned = cleanup
2074                .as_ref()
2075                .map_or(0, |cleanup| cleanup.abandoned_relays);
2076            send_route_control_pushes(
2077                &self.forwarding,
2078                routes,
2079                ClientControlPush::RouteClosed {
2080                    module_id,
2081                    channels: Vec::new(),
2082                    reason,
2083                    drained,
2084                    abandoned,
2085                    excluded_subscriptions: 0,
2086                    terminal: Some(terminal),
2087                },
2088            );
2089        }
2090        if let Ok(cleanup) = cleanup {
2091            self.emit_route_goodbyes(cleanup.released);
2092        }
2093        // Signal the registration-release watch only now that BOTH registry and
2094        // forwarding teardown are done, so a supervisor waiting to spawn a
2095        // replacement never observes release while old routes still exist.
2096        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2097            crate::supervise::notify_registration_release();
2098            self.capability_evaluator.wake_deadline_loop();
2099            self.refresh_capability_requirements();
2100        }
2101        self.supervisor.remove_spawn_subscribers(connection_id);
2102        self.activity_streams
2103            .lock()
2104            .unwrap_or_else(|p| p.into_inner())
2105            .retain(|(id, _), task| {
2106                if *id == connection_id {
2107                    task.abort();
2108                    false
2109                } else {
2110                    true
2111                }
2112            });
2113        // Sync authority dies with its connection, so the owner's next
2114        // connection can take it; the owner's scopes stay as they are.
2115        self.hello_launch_nonces
2116            .lock()
2117            .unwrap_or_else(|poisoned| poisoned.into_inner())
2118            .forget(connection_id);
2119        self.scopes
2120            .write()
2121            .unwrap_or_else(|poisoned| poisoned.into_inner())
2122            .release_connection(connection_id);
2123        registrations
2124    }
2125
2126    pub(crate) fn handle_route_goodbye(
2127        &self,
2128        connection_id: ConnectionId,
2129        route_channel: u16,
2130        route_epoch: u32,
2131    ) -> Result<bool, RouterError> {
2132        debug!(
2133            connection_id = connection_id.get(),
2134            route_channel, route_epoch, "handling route GOODBYE"
2135        );
2136        let RouteRelease::Removed(released_route) = self
2137            .forwarding
2138            .release_client_route(connection_id, route_channel, route_epoch)
2139            .map_err(RouterError::Forwarding)?
2140        else {
2141            return Ok(false);
2142        };
2143        self.emit_route_goodbyes(vec![released_route]);
2144        Ok(true)
2145    }
2146
2147    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2148        for released in released_routes {
2149            let frame = match Frame::build_with_version(
2150                released.negotiated_ver,
2151                FrameType::Goodbye,
2152                control_flags(),
2153                released.channel,
2154                released.epoch,
2155                0,
2156                Vec::new(),
2157            ) {
2158                Ok(frame) => frame,
2159                Err(err) => {
2160                    warn!(
2161                        route_channel = released.channel,
2162                        error = %err,
2163                        "failed to build route GOODBYE frame"
2164                    );
2165                    continue;
2166                }
2167            };
2168            if !released.close_on_delivery_failure() {
2169                crate::forwarding::send_module_route_goodbye(
2170                    &self.counters,
2171                    &released.sink,
2172                    frame,
2173                    released.module_id.as_deref(),
2174                    "client route released",
2175                );
2176                continue;
2177            }
2178            if let Err(err) = released.sink.try_send(frame) {
2179                warn!(
2180                    target_connection_id = released.connection_id.get(),
2181                    route_channel = released.channel,
2182                    error = %err,
2183                    "route GOODBYE was not delivered to client; closing target connection"
2184                );
2185                if self
2186                    .forwarding
2187                    .escalate_client_delivery_failure(
2188                        released.connection_id,
2189                        released.channel,
2190                        released.epoch,
2191                        CloseReason::new(
2192                            "route_goodbye_delivery_failed",
2193                            format!(
2194                                "failed to enqueue route GOODBYE for channel {}: {err}",
2195                                released.channel
2196                            ),
2197                        ),
2198                        crate::forwarding::UndeliveredFrame {
2199                            module_id: released.module_id.as_deref(),
2200                            sink: &released.sink,
2201                        },
2202                    )
2203                    .unwrap_or(false)
2204                {
2205                    self.counters.increment_goodbye_relay_client_failed();
2206                }
2207            }
2208        }
2209    }
2210
2211    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2212    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2213    /// own commit failed after the module had already accepted). Without this, a
2214    /// module that accepts late keeps a binding subc has torn down, so a later
2215    /// frame on that module channel could misdeliver if the channel is reused.
2216    ///
2217    /// Never closes the shared module connection on failure: a dropped notification
2218    /// only wastes a bounded amount of warm module-side state, which the module's
2219    /// own idle reaper reclaims. Only call this once the route.bind relay was
2220    /// actually enqueued to the module — if the relay send itself failed, the
2221    /// module never created a binding and there is nothing to tear down.
2222    fn send_abandoned_route_bind_goodbye(
2223        &self,
2224        module_sink: &crate::FrameSink,
2225        negotiated_ver: u8,
2226        module_channel: u16,
2227        module_epoch: u32,
2228    ) {
2229        let frame = match Frame::build_with_version(
2230            negotiated_ver,
2231            FrameType::Goodbye,
2232            control_flags(),
2233            module_channel,
2234            module_epoch,
2235            0,
2236            Vec::new(),
2237        ) {
2238            Ok(frame) => frame,
2239            Err(err) => {
2240                warn!(
2241                    route_channel = module_channel,
2242                    error = %err,
2243                    "failed to build GOODBYE for abandoned route.bind"
2244                );
2245                return;
2246            }
2247        };
2248        crate::forwarding::send_module_route_goodbye(
2249            &self.counters,
2250            module_sink,
2251            frame,
2252            None,
2253            "abandoned route.bind",
2254        );
2255    }
2256
2257    fn handle_hello(
2258        &self,
2259        connection_id: ConnectionId,
2260        sink: Option<crate::FrameSink>,
2261        frame: Frame,
2262    ) -> Result<Vec<Frame>, RouterError> {
2263        debug!(
2264            connection_id = connection_id.get(),
2265            corr = frame.header.corr,
2266            "handling HELLO"
2267        );
2268        // A module connection has one identity for its entire lifetime. A second
2269        // registration would leave the old registry owner behind while replacing
2270        // its forwarding endpoint and launch nonce.
2271        if self
2272            .registry
2273            .get_module_by_connection(connection_id)
2274            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2275            .is_some()
2276        {
2277            return Ok(vec![control_error_frame(
2278                &frame,
2279                "invalid_hello",
2280                "connection is already registered as a module",
2281            )?]);
2282        }
2283        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2284            Ok(value) => value,
2285            Err(err) => {
2286                return Ok(vec![control_error_frame(
2287                    &frame,
2288                    "invalid_hello",
2289                    format!("malformed HELLO body: {err}"),
2290                )?])
2291            }
2292        };
2293        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2294            return Ok(vec![control_error_frame(
2295                &frame,
2296                "invalid_capability_grammar",
2297                err.to_string(),
2298            )?]);
2299        }
2300        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2301            return Ok(vec![control_error_frame(
2302                &frame,
2303                "invalid_manifest",
2304                err.to_string(),
2305            )?]);
2306        }
2307        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2308            return Ok(vec![control_error_frame(
2309                &frame,
2310                "invalid_event_declaration",
2311                err.to_string(),
2312            )?]);
2313        }
2314        if let Some(provenance) = hello_value
2315            .get("manifest")
2316            .and_then(|manifest| manifest.get("provenance"))
2317        {
2318            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2319                return Ok(vec![control_error_frame(
2320                    &frame,
2321                    "invalid_manifest",
2322                    format!("malformed manifest provenance: {err}"),
2323                )?]);
2324            }
2325        }
2326        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2327            Ok(hello) => hello,
2328            Err(err) => {
2329                return Ok(vec![control_error_frame(
2330                    &frame,
2331                    "invalid_hello",
2332                    format!("malformed HELLO body: {err}"),
2333                )?])
2334            }
2335        };
2336
2337        if hello.protocol_ver != hello.manifest.protocol_ver {
2338            return Ok(vec![control_error_frame(
2339                &frame,
2340                "invalid_manifest",
2341                format!(
2342                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2343                    hello.protocol_ver, hello.manifest.protocol_ver
2344                ),
2345            )?]);
2346        }
2347
2348        if hello.manifest.module_id.trim().is_empty() {
2349            return Ok(vec![control_error_frame(
2350                &frame,
2351                "invalid_manifest",
2352                "manifest module_id must not be empty",
2353            )?]);
2354        }
2355
2356        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2357            Ok(negotiated_ver) => negotiated_ver,
2358            Err(message) => {
2359                return Ok(vec![control_error_frame(
2360                    &frame,
2361                    "version_unsupported",
2362                    message,
2363                )?])
2364            }
2365        };
2366
2367        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2368        // swap is open for this id, the only HELLO admitted as a second process
2369        // is the one carrying the candidate's launch nonce (the swap token), and
2370        // it registers into the candidate slot rather than being refused as a
2371        // duplicate. Run after the reserved gate, a reserved module's candidate
2372        // would be refused `reserved_module` for presenting a nonce that gate
2373        // does not know. See `SupervisorHandle::swap_hello_admission`.
2374        let swap_admission = self
2375            .supervisor
2376            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2377        if swap_admission == SwapHelloAdmission::Refused {
2378            warn!(
2379                module_id = %hello.manifest.module_id,
2380                connection_id = connection_id.get(),
2381                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2382            );
2383            return Ok(vec![control_error_frame(
2384                &frame,
2385                "swap_token_invalid",
2386                format!(
2387                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2388                    hello.manifest.module_id
2389                ),
2390            )?]);
2391        }
2392        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2393
2394        // Reserved-module identity gate: a module_id configured `reserved` may be
2395        // registered ONLY by the process subc spawned for it, proven by echoing the
2396        // one-time launch nonce subc injected. A non-reserved id has no recorded
2397        // nonce and always passes. This blocks a key-holder from impersonating a
2398        // security-boundary module (e.g. the credential vault) while the real one is
2399        // down/restarting and its registration slot is momentarily free. A swap
2400        // candidate has already proven the same thing with its own nonce above.
2401        if let Some(rejection) = (!swap_candidate)
2402            .then(|| {
2403                self.supervisor.reserved_hello_rejection(
2404                    &hello.manifest.module_id,
2405                    hello.launch_nonce.as_deref(),
2406                )
2407            })
2408            .flatten()
2409        {
2410            let message = match rejection {
2411                ReservedHelloRejection::Exact { module_id } => format!(
2412                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2413                ),
2414                ReservedHelloRejection::Prefix {
2415                    prefix,
2416                    owner_module_id,
2417                } => format!(
2418                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2419                    hello.manifest.module_id
2420                ),
2421            };
2422            return Ok(vec![control_error_frame(
2423                &frame,
2424                "reserved_module",
2425                message,
2426            )?]);
2427        }
2428
2429        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2430            &hello.manifest.module_id,
2431            hello.manifest.capabilities.as_ref(),
2432        );
2433        if let Some(refusal) = reserved_capability_refusals.first() {
2434            let capability = refusal.capability.clone();
2435            let bound_module = refusal.claimants[0].clone();
2436            log_duplicate_claim_events(reserved_capability_refusals);
2437            return Ok(vec![control_error_frame(
2438                &frame,
2439                "reserved_capability",
2440                format!(
2441                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2442                    capability, bound_module, hello.manifest.module_id
2443                ),
2444            )?]);
2445        }
2446
2447        // A connection that already opened client routes must not also register as
2448        // a module: cleanup would then release only one side and leak the other.
2449        if self
2450            .forwarding
2451            .connection_has_client_routes(connection_id)
2452            .map_err(RouterError::Forwarding)?
2453        {
2454            return Ok(vec![control_error_frame(
2455                &frame,
2456                "invalid_hello",
2457                "connection has open client routes and cannot also register as a module",
2458            )?]);
2459        }
2460
2461        // Kept for scope sync authority, which goes only to the connection that
2462        // presented the module's current launch nonce. Recorded before the
2463        // registration is attempted: a connection whose registration then fails
2464        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2465        self.hello_launch_nonces
2466            .lock()
2467            .unwrap_or_else(|poisoned| poisoned.into_inner())
2468            .record(connection_id, hello.launch_nonce.as_deref());
2469        let control_ops = effective_module_control_ops(hello.control_ops);
2470        // Built before anything is registered so an encoding failure leaves no
2471        // registry or forwarding state behind.
2472        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2473        if swap_candidate {
2474            return self.register_swap_candidate(
2475                connection_id,
2476                sink,
2477                &frame,
2478                hello.manifest,
2479                negotiated_ver,
2480                control_ops,
2481                hello_ack,
2482            );
2483        }
2484        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2485        // is refused, but `catalog.list` does not show the module and the
2486        // catalog generation does not move until it is published below, once
2487        // its forwarding endpoint exists. Listing it now would advertise a
2488        // module that `route.open` still refuses for having no forwarding
2489        // connection.
2490        let registration = match self.registry.register_unpublished_with_control_ops(
2491            hello.manifest,
2492            negotiated_ver,
2493            connection_id,
2494            control_ops,
2495        ) {
2496            Ok(registration) => registration,
2497            Err(RegistryError::DuplicateModuleId { module_id }) => {
2498                return Ok(vec![control_error_frame(
2499                    &frame,
2500                    "duplicate_module_id",
2501                    format!(
2502                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2503                    ),
2504                )?])
2505            }
2506            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2507                return Ok(vec![control_error_frame(
2508                    &frame,
2509                    "invalid_module_id",
2510                    err.to_string(),
2511                )?])
2512            }
2513            Err(err) => {
2514                return Ok(vec![control_error_frame(
2515                    &frame,
2516                    "registry_error",
2517                    err.to_string(),
2518                )?])
2519            }
2520        };
2521        #[cfg(test)]
2522        self.hold_between_registry_and_forwarding();
2523
2524        let reply = if let Some(sink) = sink {
2525            // The forwarding table's module store is also the daemon-to-module
2526            // control-RPC lane, so every HELLO gets a live endpoint even when the
2527            // manifest has no routable provider role. Non-routable modules still
2528            // cannot receive route.bind in production: `handle_route_open` checks
2529            // the registry manifest with `target_has_required_role` before the
2530            // only production call to `begin_route_bind_relay_for` below that
2531            // route.open path. The remaining direct relay callers are unit tests
2532            // and benchmark harnesses that construct forwarding state explicitly.
2533            //
2534            // The HELLO_ACK is queued by the forwarding table itself, before the
2535            // endpoint becomes visible, and is NOT returned as a reply. A module
2536            // reads HELLO_ACK first and exits on anything else; a reply is only
2537            // written after this handler returns, by which time a route.open on
2538            // another connection could already have queued a route.bind request
2539            // for this module ahead of it.
2540            let concurrency = manifest_concurrency(&registration.manifest);
2541            if let Err(err) = self.forwarding.register_module_connection_acked(
2542                connection_id,
2543                registration.manifest.module_id.clone(),
2544                negotiated_ver,
2545                concurrency,
2546                sink,
2547                hello_ack,
2548            ) {
2549                // Forwarding registration failed, so there is no forwarding
2550                // state to tear down. Remove the registry entry, which was never
2551                // published and so never listed or counted in the catalog
2552                // generation, and signal the release watch directly.
2553                if matches!(
2554                    self.deregister_connection(
2555                        connection_id,
2556                        RegistrationEndReason::RegistrationFailed,
2557                    ),
2558                    Ok(r) if !r.is_empty()
2559                ) {
2560                    crate::supervise::notify_registration_release();
2561                }
2562                return Ok(vec![control_error_frame(
2563                    &frame,
2564                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2565                        "invalid_hello"
2566                    } else {
2567                        forwarding_error_code(&err)
2568                    },
2569                    err.to_string(),
2570                )?]);
2571            }
2572            Vec::new()
2573        } else {
2574            // No sink means no forwarding endpoint, so nothing can be routed
2575            // ahead of the ack; it goes out as the reply.
2576            vec![hello_ack]
2577        };
2578
2579        // The module is routable now (or, without a sink, as routable as it
2580        // will ever be), so the catalog may list it. Publishing bumps the
2581        // catalog generation in the same registry critical section that makes
2582        // the module listed.
2583        if !self
2584            .registry
2585            .publish_registration(connection_id)
2586            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2587        {
2588            debug!(
2589                module_id = %registration.manifest.module_id,
2590                connection_id = connection_id.get(),
2591                "registration was removed before it could be published"
2592            );
2593        }
2594
2595        // Exposure over assumption: Concurrency's serde default is pinned to the
2596        // pre-field behavior (ModuleManaged), so a management surface that is
2597        // genuinely Serial and just never declared it inherits concurrent
2598        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2599        // turns "no module has been bitten yet" into the checkable claim "no
2600        // module is exposed" -- one read of the boot log instead of a fleet
2601        // audit. Detected from the raw HELLO bytes because the serde default
2602        // deliberately erases the absent/declared distinction from the type.
2603        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2604            info!(
2605                module_id = %registration.manifest.module_id,
2606                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2607            );
2608        }
2609
2610        self.apply_registration_capabilities(&registration);
2611
2612        self.supervisor
2613            .record_activity_registration(&registration.manifest.module_id);
2614
2615        info!(
2616            module_id = %registration.manifest.module_id,
2617            module_version = %registration.manifest.module_version,
2618            negotiated_ver,
2619            routable_provider = manifest_provides_routable_role(&registration.manifest),
2620            connection_id = connection_id.get(),
2621            "module registered"
2622        );
2623
2624        Ok(reply)
2625    }
2626
2627    /// Register a HELLO the swap gate admitted into the candidate slot of the
2628    /// registry and of forwarding, where it is reachable over its own
2629    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2630    /// nothing routes to it until the supervisor cuts over.
2631    ///
2632    /// Registry first (unpublished), then forwarding, then publication, the
2633    /// same order as an ordinary HELLO; a forwarding failure removes the
2634    /// registry entry again. The swap supervisor treats a published, ready
2635    /// candidate as registered and promotes it, and promotion needs the
2636    /// candidate's forwarding endpoint, so the candidate is published only
2637    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2638    /// recompute capability verdicts or run the capability deny census
2639    /// (`enforce_capability_denies`): both are about the module in the active
2640    /// slot, which `route.open` can target. A candidate sits in the candidate
2641    /// slot, so it neither provides capabilities to other modules nor can be
2642    /// routed to until promotion moves it into the active slot.
2643    #[allow(clippy::too_many_arguments)]
2644    fn register_swap_candidate(
2645        &self,
2646        connection_id: ConnectionId,
2647        sink: Option<crate::FrameSink>,
2648        frame: &Frame,
2649        manifest: ModuleManifest,
2650        negotiated_ver: u8,
2651        control_ops: Vec<String>,
2652        hello_ack: Frame,
2653    ) -> Result<Vec<Frame>, RouterError> {
2654        let module_id = manifest.module_id.clone();
2655        let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2656            manifest,
2657            negotiated_ver,
2658            connection_id,
2659            control_ops,
2660        ) {
2661            Ok(registration) => registration,
2662            Err(RegistryError::DuplicateModuleId { module_id }) => {
2663                return Ok(vec![control_error_frame(
2664                    frame,
2665                    "duplicate_module_id",
2666                    format!(
2667                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2668                    ),
2669                )?])
2670            }
2671            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2672                return Ok(vec![control_error_frame(
2673                    frame,
2674                    "invalid_module_id",
2675                    err.to_string(),
2676                )?])
2677            }
2678            Err(err) => {
2679                return Ok(vec![control_error_frame(
2680                    frame,
2681                    "registry_error",
2682                    err.to_string(),
2683                )?])
2684            }
2685        };
2686        #[cfg(test)]
2687        self.hold_between_registry_and_forwarding();
2688        let reply = if let Some(sink) = sink {
2689            // Same ordering as an ordinary HELLO: the forwarding table queues
2690            // the HELLO_ACK before the candidate endpoint is inserted, because
2691            // a module exits if its first frame after HELLO is anything else.
2692            let concurrency = manifest_concurrency(&registration.manifest);
2693            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2694                connection_id,
2695                module_id.clone(),
2696                negotiated_ver,
2697                concurrency,
2698                sink,
2699                hello_ack,
2700            ) {
2701                if matches!(
2702                    self.deregister_connection(
2703                        connection_id,
2704                        RegistrationEndReason::RegistrationFailed,
2705                    ),
2706                    Ok(r) if !r.is_empty()
2707                ) {
2708                    crate::supervise::notify_registration_release();
2709                }
2710                return Ok(vec![control_error_frame(
2711                    frame,
2712                    forwarding_error_code(&err),
2713                    err.to_string(),
2714                )?]);
2715            }
2716            Vec::new()
2717        } else {
2718            vec![hello_ack]
2719        };
2720        if !self
2721            .registry
2722            .publish_registration(connection_id)
2723            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2724        {
2725            debug!(
2726                module_id = %module_id,
2727                connection_id = connection_id.get(),
2728                "swap candidate registration was removed before it could be published"
2729            );
2730        }
2731        self.supervisor.mark_swap_candidate_admitted(&module_id);
2732        info!(
2733            module_id = %module_id,
2734            module_version = %registration.manifest.module_version,
2735            negotiated_ver,
2736            ready = registration.ready,
2737            connection_id = connection_id.get(),
2738            "swap candidate registered; not routable until cutover"
2739        );
2740        Ok(reply)
2741    }
2742
2743    fn build_hello_ack(
2744        &self,
2745        frame: &Frame,
2746        negotiated_ver: u8,
2747        module_id: &str,
2748    ) -> Result<Frame, RouterError> {
2749        let ack = ModuleHelloAckBody {
2750            negotiated_ver,
2751            subc_ops: module_subc_ops(),
2752            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2753            storage: self
2754                .storage_config
2755                .as_ref()
2756                .map(|cfg| cfg.descriptor_for(module_id)),
2757            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2758        };
2759        let body = serde_json::to_vec(&ack).map_err(|err| {
2760            RouterError::backend(
2761                0,
2762                frame.header.corr,
2763                format!("failed to encode HELLO_ACK: {err}"),
2764            )
2765        })?;
2766
2767        Frame::build_with_version(
2768            negotiated_ver,
2769            FrameType::HelloAck,
2770            control_flags(),
2771            0,
2772            0,
2773            frame.header.corr,
2774            body,
2775        )
2776        .map_err(RouterError::FrameBuild)
2777    }
2778
2779    async fn handle_client_control_request(
2780        &self,
2781        ctx: &RouteCtx,
2782        frame: Frame,
2783        request: ClientControlRequest,
2784        route_open_timings: &mut Option<RouteOpenTimings>,
2785    ) -> Result<Vec<Frame>, RouterError> {
2786        if let Some(operation) = privileged_operation(&request) {
2787            let caller = self.resolve_caller(ctx.connection_id).await;
2788            let op = client_control_request_op(&request);
2789            let text = prompt_text(operation, &caller);
2790            match self.privileged_operator_prompt {
2791                PrivilegedOperatorPrompt::Log => {
2792                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2793                        target = ?operation, caller = %caller.description(), summary = %text,
2794                        "privileged request would prompt the operator");
2795                }
2796                PrivilegedOperatorPrompt::Enforce => {
2797                    let key =
2798                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2799                    let outcome = self
2800                        .forwarding
2801                        .operator_confirms()
2802                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2803                        .await;
2804                    if outcome.code != "confirmed" {
2805                        return Ok(vec![outcome.frame(&frame)]);
2806                    }
2807                }
2808            }
2809        }
2810        match request {
2811            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2812            ClientControlRequest::CatalogList { module_id } => {
2813                self.handle_catalog_list(frame, module_id)
2814            }
2815            ClientControlRequest::RouteOpen {
2816                target,
2817                identity,
2818                consumer_identity,
2819                consumer_capabilities,
2820                role_versions,
2821                admission_facts,
2822                scope,
2823            } => {
2824                self.handle_route_open(
2825                    ctx,
2826                    frame,
2827                    RouteOpenRequest {
2828                        target,
2829                        identity,
2830                        consumer_identity,
2831                        consumer_capabilities,
2832                        role_versions,
2833                        admission_facts,
2834                        scope,
2835                    },
2836                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2837                )
2838                .await
2839            }
2840            ClientControlRequest::RoutePoll {
2841                route_channel,
2842                route_epoch,
2843                kind,
2844            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2845            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2846            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2847                self.handle_supervisor_spawn_snapshot(frame)
2848            }
2849            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2850                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2851            }
2852            ClientControlRequest::SupervisorRestart {
2853                module_id,
2854                drain_timeout_ms,
2855            } => {
2856                self.log_supervisor_request_received(
2857                    ctx,
2858                    frame.header.corr,
2859                    ops::SUPERVISOR_RESTART,
2860                    Some(&module_id),
2861                    None,
2862                )?;
2863                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2864                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2865                    .await
2866            }
2867            ClientControlRequest::SupervisorSwap {
2868                module_id,
2869                ready_timeout_ms,
2870            } => {
2871                self.log_supervisor_request_received(
2872                    ctx,
2873                    frame.header.corr,
2874                    ops::SUPERVISOR_SWAP,
2875                    Some(&module_id),
2876                    None,
2877                )?;
2878                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2879                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2880                    .await
2881            }
2882            ClientControlRequest::SupervisorReload { module_id } => {
2883                self.log_supervisor_request_received(
2884                    ctx,
2885                    frame.header.corr,
2886                    ops::SUPERVISOR_RELOAD,
2887                    Some(&module_id),
2888                    None,
2889                )?;
2890                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2891                self.handle_supervisor_reload(frame, module_id, requester)
2892                    .await
2893            }
2894            ClientControlRequest::SupervisorRescan { preview } => {
2895                if !preview {
2896                    self.log_supervisor_request_received(
2897                        ctx,
2898                        frame.header.corr,
2899                        ops::SUPERVISOR_RESCAN,
2900                        None,
2901                        None,
2902                    )?;
2903                }
2904                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2905                self.handle_supervisor_rescan(
2906                    frame,
2907                    if preview {
2908                        RescanMode::Preview
2909                    } else {
2910                        RescanMode::Execute
2911                    },
2912                    requester,
2913                )
2914                .await
2915            }
2916            ClientControlRequest::SupervisorRescanChecked {
2917                preview,
2918                candidate,
2919                expect,
2920            } => {
2921                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2922                    .await
2923            }
2924            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2925                self.log_supervisor_request_received(
2926                    ctx,
2927                    frame.header.corr,
2928                    ops::SUPERVISOR_RELEASE_RESERVED,
2929                    Some(&module_id),
2930                    None,
2931                )?;
2932                self.handle_supervisor_release_reserved(frame, module_id)
2933                    .await
2934            }
2935            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2936                self.log_supervisor_request_received(
2937                    ctx,
2938                    frame.header.corr,
2939                    ops::SUPERVISOR_SET_ENABLED,
2940                    Some(&module_id),
2941                    Some(enabled),
2942                )?;
2943                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2944                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2945                    .await
2946            }
2947            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2948                self.handle_supervisor_health_probe(frame, module_id).await
2949            }
2950            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2951            ClientControlRequest::SupervisorRoutes { module_id } => {
2952                self.handle_supervisor_routes(frame, module_id)
2953            }
2954            ClientControlRequest::SupervisorProvenance { module_id } => {
2955                self.handle_supervisor_provenance(frame, module_id).await
2956            }
2957            ClientControlRequest::SupervisorStderrTail {
2958                module_id,
2959                max_lines,
2960                max_bytes,
2961            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2962            ClientControlRequest::SupervisorTerminals { module_id } => {
2963                self.handle_supervisor_terminals(frame, module_id).await
2964            }
2965        }
2966    }
2967
2968    fn log_supervisor_request_received(
2969        &self,
2970        ctx: &RouteCtx,
2971        corr: u64,
2972        op: &'static str,
2973        module_id: Option<&str>,
2974        enabled: Option<bool>,
2975    ) -> Result<(), RouterError> {
2976        let caller = self.supervisor_requester(ctx, corr)?;
2977        let caller = principal_label(&caller);
2978        // `Option` fields are recorded only when present, so a request with no
2979        // module id (rescan) or no enabled flag simply omits that field.
2980        info!(
2981            target: "control",
2982            op,
2983            module_id,
2984            enabled,
2985            connection_id = ctx.connection_id.get(),
2986            caller = %caller,
2987            "supervisor request received"
2988        );
2989        Ok(())
2990    }
2991
2992    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2993        Ok(self
2994            .registry
2995            .get_module_by_connection(ctx.connection_id)
2996            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2997            .map(|registration| Principal::Reserved {
2998                module_id: registration.manifest.module_id,
2999            })
3000            .unwrap_or(Principal::Direct))
3001    }
3002
3003    fn handle_module_control_request(
3004        &self,
3005        connection_id: ConnectionId,
3006        frame: Frame,
3007        request: ModuleControlRequestFromModule,
3008    ) -> Result<Vec<Frame>, RouterError> {
3009        match request {
3010            ModuleControlRequestFromModule::CatalogUpdate {
3011                provides,
3012                capabilities,
3013                ready,
3014            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3015            ModuleControlRequestFromModule::LiveRoots {} => {
3016                let registered = self
3017                    .registry
3018                    .get_module_by_connection(connection_id)
3019                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3020                let Some(registration) = registered else {
3021                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3022                };
3023                let response = self
3024                    .forwarding
3025                    .live_roots(&registration.manifest.module_id)
3026                    .map_err(RouterError::Forwarding)?;
3027                Ok(vec![control_response_body_frame(
3028                    &frame,
3029                    &response,
3030                    "ModuleControlResponseToModule::LiveRoots",
3031                )?])
3032            }
3033            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3034                self.handle_scope_sync(connection_id, frame, generation, scopes)
3035            }
3036            ModuleControlRequestFromModule::ScopeApply {
3037                generation,
3038                upsert,
3039                end,
3040            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3041            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3042                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3043            }
3044        }
3045    }
3046
3047    fn handle_scope_served_by(
3048        &self,
3049        connection_id: ConnectionId,
3050        frame: Frame,
3051    ) -> Result<Vec<Frame>, RouterError> {
3052        let registered = self
3053            .registry
3054            .get_module_by_connection(connection_id)
3055            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3056        let Some(registered) = registered else {
3057            return Ok(vec![control_error_frame(
3058                &frame,
3059                "not_registered",
3060                "scope.served_by requires an active module registration owned by this connection",
3061            )?]);
3062        };
3063        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3064        {
3065            Ok(request) => request,
3066            Err(err) => {
3067                return Ok(vec![control_error_frame(
3068                    &frame,
3069                    "invalid_control_body",
3070                    format!("malformed scope.served_by body: {err}"),
3071                )?])
3072            }
3073        };
3074        let answer = self
3075            .scopes
3076            .read()
3077            .unwrap_or_else(|poisoned| poisoned.into_inner())
3078            .served_by(
3079                &registered.manifest.module_id,
3080                &request.scope_ref,
3081                request.scope_epoch,
3082                &request.module_id,
3083            );
3084        let reply = subc_control::ScopeServedByReply::new(
3085            answer,
3086            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3087        );
3088        Ok(vec![control_response_body_frame(
3089            &frame,
3090            &reply,
3091            "ScopeServedByReply",
3092        )?])
3093    }
3094
3095    fn handle_operator_confirm(
3096        &self,
3097        ctx: &RouteCtx,
3098        frame: Frame,
3099    ) -> Result<Vec<Frame>, RouterError> {
3100        use crate::operator_confirm::{audit, Outcome};
3101        let registration = self
3102            .registry
3103            .get_module_by_connection(ctx.connection_id)
3104            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3105        let Some(registration) = registration else {
3106            let outcome = Outcome::refusal("not_registered");
3107            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3108            return Ok(vec![outcome.frame(&frame)]);
3109        };
3110        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3111            Ok(request) => request,
3112            Err(_) => {
3113                let outcome = Outcome::refusal("invalid_control_body");
3114                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3115                return Ok(vec![outcome.frame(&frame)]);
3116            }
3117        };
3118        let module_id = registration.manifest.module_id;
3119        // Read the launch nonce (under its own lock) before taking the forwarding
3120        // table's lock below: holding forwarding while waiting on another daemon
3121        // lock risks a lock-order deadlock with paths that take them the other way.
3122        let nonce = self
3123            .hello_launch_nonces
3124            .lock()
3125            .unwrap_or_else(|p| p.into_inner())
3126            .nonce(ctx.connection_id)
3127            .map(str::to_owned);
3128        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3129            self.supervisor
3130                .spawned_consumer_authorized(&module_id, nonce)
3131        });
3132        let confirms = self.forwarding.operator_confirms();
3133        self.forwarding
3134            .with_operator_route(
3135                ctx.connection_id,
3136                request.route_channel,
3137                request.route_epoch,
3138                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3139            )
3140            .map_err(RouterError::Forwarding)
3141    }
3142
3143    /// `scope.sync`: the owner is the module registered on this connection.
3144    /// A connection with no registration (every client connection, `direct`
3145    /// included) is refused `not_registered` before the table is consulted.
3146    fn handle_scope_sync(
3147        &self,
3148        connection_id: ConnectionId,
3149        frame: Frame,
3150        generation: u64,
3151        scopes: Vec<ScopeRecord>,
3152    ) -> Result<Vec<Frame>, RouterError> {
3153        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3154    }
3155
3156    fn handle_scope_change(
3157        &self,
3158        connection_id: ConnectionId,
3159        frame: Frame,
3160        generation: u64,
3161        scopes: Vec<ScopeRecord>,
3162        end: Option<Vec<ScopeEnd>>,
3163    ) -> Result<Vec<Frame>, RouterError> {
3164        let is_apply = end.is_some();
3165        let Some(registration) = self
3166            .registry
3167            .get_module_by_connection(connection_id)
3168            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3169        else {
3170            return Ok(vec![control_error_frame(
3171                &frame,
3172                "not_registered",
3173                "scope.sync and scope.apply require an active module registration owned by this connection",
3174            )?]);
3175        };
3176        let owner = registration.manifest.module_id;
3177        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3178        let is_current_launch = |connection: ConnectionId| {
3179            self.hello_launch_nonces
3180                .lock()
3181                .unwrap_or_else(|poisoned| poisoned.into_inner())
3182                .presented(connection, current_nonce.as_deref())
3183        };
3184        // Lock order is the scope table, then the forwarding table: the new
3185        // tags are published, and the routes the change closes are selected,
3186        // while the scope table is still write-locked, so no admission can read
3187        // a record whose tag is not yet published.
3188        let mut table = self
3189            .scopes
3190            .write()
3191            .unwrap_or_else(|poisoned| poisoned.into_inner());
3192        let outcome = match end {
3193            Some(end) => table.apply(
3194                &owner,
3195                connection_id,
3196                is_current_launch,
3197                generation,
3198                scopes,
3199                end,
3200            ),
3201            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3202        };
3203        let drained = match &outcome {
3204            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3205            Err(_) => Vec::new(),
3206        };
3207        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3208        drop(table);
3209        if has_deadlines {
3210            self.scope_deadline_added.notify_one();
3211        }
3212        match outcome {
3213            Ok(applied) => {
3214                let counts = ScopeOutcomeCounts::of(&applied.results);
3215                info!(
3216                    owner = %owner,
3217                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3218                    generation,
3219                    records = applied.results.len(),
3220                    created = counts.created,
3221                    replaced = counts.replaced,
3222                    updated = counts.updated,
3223                    unchanged = counts.unchanged,
3224                    refused = counts.refused,
3225                    ended = applied.ended.len(),
3226                    tag_changes = applied.tag_changes.len(),
3227                    routes_closed = drained.len(),
3228                    "scope change accepted"
3229                );
3230                // An accepted scope change can still refuse individual records, and the
3231                // owner is the only party that sees the reply. Name them here so
3232                // an operator can tell a refused session from a missing one
3233                // without the owner's logs. Capped so a call that refuses
3234                // thousands cannot flood the log; the count above is complete.
3235                for refused in applied
3236                    .results
3237                    .iter()
3238                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3239                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3240                {
3241                    warn!(
3242                        owner = %owner,
3243                        generation,
3244                        scope_ref = %refused.scope_ref,
3245                        scope_epoch = refused.scope_epoch,
3246                        code = refused.code.as_deref().unwrap_or(""),
3247                        "scope record refused"
3248                    );
3249                }
3250                self.close_scope_drained_routes(drained);
3251                let response = if is_apply {
3252                    ModuleControlResponseToModule::ScopeApply {
3253                        generation,
3254                        results: applied.results,
3255                        end_results: applied.end_results,
3256                        ended: applied.ended,
3257                    }
3258                } else {
3259                    ModuleControlResponseToModule::ScopeSync {
3260                        generation,
3261                        results: applied.results,
3262                        ended: applied.ended,
3263                    }
3264                };
3265                Ok(vec![control_response_body_frame(
3266                    &frame,
3267                    &response,
3268                    "scope change response",
3269                )?])
3270            }
3271            Err(refusal) => {
3272                info!(
3273                    owner = %owner,
3274                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3275                    generation,
3276                    code = refusal.code,
3277                    "scope change refused"
3278                );
3279                Ok(vec![control_error_frame(
3280                    &frame,
3281                    refusal.code,
3282                    refusal.message,
3283                )?])
3284            }
3285        }
3286    }
3287
3288    /// Tell both ends of each route a scope change closed. The module gets a
3289    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3290    /// the client's route handle. The client also gets `route.closed` with the
3291    /// scope reason, one push per module and reason, so it can tell a revoked
3292    /// route from an ordinary close and not reopen it.
3293    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3294        if drained.is_empty() {
3295            return;
3296        }
3297        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3298            BTreeMap::new();
3299        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3300        for route in drained {
3301            warn!(
3302                module_id = %route.module_id,
3303                reason = ?route.reason,
3304                client_connection_id = route.client.connection_id.get(),
3305                route_channel = route.client.channel,
3306                "closing route because its scope changed"
3307            );
3308            pushes
3309                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3310                .or_insert_with(|| (route.reason, Vec::new()))
3311                .1
3312                .push(EndpointRoute {
3313                    goodbye_target: route.client.clone(),
3314                    principal: Principal::Unverified,
3315                    bound_at: Instant::now(),
3316                    draining: false,
3317                    drain_reason: None,
3318                });
3319            goodbyes.push(route.module);
3320            goodbyes.push(route.client);
3321        }
3322        for ((module_id, _), (reason, routes)) in pushes {
3323            send_route_control_pushes(
3324                &self.forwarding,
3325                routes,
3326                ClientControlPush::RouteClosed {
3327                    module_id,
3328                    channels: Vec::new(),
3329                    reason,
3330                    drained: false,
3331                    abandoned: 0,
3332                    excluded_subscriptions: 0,
3333                    terminal: Some(false),
3334                },
3335            );
3336        }
3337        self.emit_route_goodbyes(goodbyes);
3338    }
3339
3340    /// `scope.describe`: any registered module may read any scope, because a
3341    /// provider must read the scope a route it serves is stamped with.
3342    fn handle_scope_describe(
3343        &self,
3344        connection_id: ConnectionId,
3345        frame: Frame,
3346        owner: Principal,
3347        scope_ref: String,
3348    ) -> Result<Vec<Frame>, RouterError> {
3349        let registered = self
3350            .registry
3351            .get_module_by_connection(connection_id)
3352            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3353        let Some(registered) = registered else {
3354            return Ok(vec![control_error_frame(
3355                &frame,
3356                "not_registered",
3357                "scope.describe requires an active module registration owned by this connection",
3358            )?]);
3359        };
3360        let mut description = self
3361            .scopes
3362            .read()
3363            .unwrap_or_else(|poisoned| poisoned.into_inner())
3364            .describe(&owner, &scope_ref);
3365        // Describe readers also decode strict ScopeAttributes. Attribution is
3366        // safe to omit unless the requesting module declares it understands it.
3367        if !registered
3368            .manifest
3369            .capabilities
3370            .as_ref()
3371            .is_some_and(|caps| {
3372                caps.provides
3373                    .iter()
3374                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3375            })
3376        {
3377            if let Some(stamp) = description.stamp.as_mut() {
3378                stamp.attributes.on_behalf_of_agent = None;
3379            }
3380        }
3381        let owner_configured = match &owner {
3382            // Ask whether the owner is configured (`is_configured`), not
3383            // whether it is on the roster (`get(..).is_some()`): a supervised
3384            // module's process can register and describe a scope before the
3385            // supervisor has put it on the roster.
3386            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3387            _ => false,
3388        };
3389        let response = ModuleControlResponseToModule::ScopeDescribe {
3390            status: description.status,
3391            scope_epoch: description.scope_epoch,
3392            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3393            owner_synced: description.owner_synced,
3394            owner_configured,
3395            scope: description.stamp,
3396        };
3397        Ok(vec![control_response_body_frame(
3398            &frame,
3399            &response,
3400            "ModuleControlResponseToModule::ScopeDescribe",
3401        )?])
3402    }
3403
3404    fn handle_catalog_update(
3405        &self,
3406        connection_id: ConnectionId,
3407        frame: Frame,
3408        provides: Vec<ProviderRole>,
3409        capabilities: Option<CapabilityDeclarations>,
3410        ready: Option<bool>,
3411    ) -> Result<Vec<Frame>, RouterError> {
3412        self.refresh_capability_requirements();
3413        let Some(registration) = self
3414            .registry
3415            .get_module_by_connection(connection_id)
3416            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3417        else {
3418            return Ok(vec![control_error_frame(
3419                &frame,
3420                "not_registered",
3421                "catalog.update requires an active module registration owned by this connection",
3422            )?]);
3423        };
3424
3425        if let Some(message) =
3426            catalog_update_frozen_field_message(&registration.manifest, &provides)
3427        {
3428            return Ok(vec![control_error_frame(
3429                &frame,
3430                "catalog_update_frozen_field",
3431                message,
3432            )?]);
3433        }
3434
3435        let mut candidate = registration.manifest.clone();
3436        candidate.provides = provides.clone();
3437        candidate.capabilities = capabilities
3438            .clone()
3439            .or_else(|| registration.manifest.capabilities.clone());
3440        if let Err(err) = candidate.validate_capability_grammar() {
3441            return Ok(vec![control_error_frame(
3442                &frame,
3443                "invalid_capability_grammar",
3444                err.to_string(),
3445            )?]);
3446        }
3447
3448        // Updates must honor the same reserved owner as initial registration;
3449        // otherwise an empty HELLO could acquire the claim after admission.
3450        let mut conflicts = self
3451            .capability_evaluator
3452            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3453        if let Some(conflict) = conflicts.first() {
3454            let message = format!(
3455                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3456                conflict.capability, conflict.claimants[0], candidate.module_id
3457            );
3458            for conflict in &mut conflicts {
3459                conflict.source = DuplicateClaimSource::CatalogUpdate;
3460            }
3461            log_duplicate_claim_events(conflicts);
3462            return Ok(vec![control_error_frame(
3463                &frame,
3464                "reserved_capability",
3465                message,
3466            )?]);
3467        }
3468
3469        let updated = self
3470            .registry
3471            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3472            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3473        if updated.is_none() {
3474            return Ok(vec![control_error_frame(
3475                &frame,
3476                "not_registered",
3477                "catalog.update requires an active module registration owned by this connection",
3478            )?]);
3479        }
3480        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3481            log_duplicate_claim_events(
3482                self.capability_evaluator
3483                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3484            );
3485        }
3486        if capability_census_trigger(
3487            registration.manifest.capabilities.as_ref(),
3488            updated
3489                .as_ref()
3490                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3491        ) {
3492            self.enforce_capability_denies();
3493        }
3494        self.refresh_capability_requirements();
3495
3496        let response = ModuleControlResponseToModule::CatalogUpdate {};
3497        control_response_body_frame(
3498            &frame,
3499            &response,
3500            "ModuleControlResponseToModule::CatalogUpdate",
3501        )
3502        .map(|frame| vec![frame])
3503    }
3504
3505    pub(crate) fn with_activity_ledger_config(
3506        mut self,
3507        run_dir: &Path,
3508        incarnation: String,
3509        config: crate::activity::ActivityConfig,
3510    ) -> Self {
3511        self.activity = Some(crate::activity::ActivityLedger::open_config(
3512            run_dir.join("activity.jsonl"),
3513            incarnation,
3514            config,
3515        ));
3516        self.supervisor.set_activity(self.activity.clone());
3517        self
3518    }
3519
3520    pub(crate) async fn configure_activity(
3521        &self,
3522        config: crate::activity::ActivityConfig,
3523    ) -> Result<(), &'static str> {
3524        match &self.activity {
3525            Some(ledger) => ledger.configure(config).await,
3526            None => Ok(()),
3527        }
3528    }
3529
3530    fn cancel_activity_subscription(
3531        &self,
3532        ctx: &RouteCtx,
3533        frame: &Frame,
3534    ) -> Result<bool, RouterError> {
3535        let task = self
3536            .activity_streams
3537            .lock()
3538            .unwrap_or_else(|p| p.into_inner())
3539            .remove(&(ctx.connection_id, frame.header.corr));
3540        let Some(task) = task else {
3541            return Ok(false);
3542        };
3543        task.abort();
3544        let end = Frame::build_with_version(
3545            response_version(frame),
3546            FrameType::StreamEnd,
3547            control_flags(),
3548            0,
3549            0,
3550            frame.header.corr,
3551            Vec::new(),
3552        )
3553        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3554        let sink = ctx.egress.clone();
3555        tokio::spawn(async move {
3556            let _ = sink.send(end).await;
3557        });
3558        Ok(true)
3559    }
3560
3561    async fn handle_activity_read(
3562        &self,
3563        ctx: &RouteCtx,
3564        frame: Frame,
3565    ) -> Result<Vec<Frame>, RouterError> {
3566        use subc_control::{
3567            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3568            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3569        };
3570        #[derive(Deserialize)]
3571        struct Identity {
3572            #[serde(default)]
3573            consumer_identity: Option<ConsumerIdentity>,
3574        }
3575        // A registered module connection already identifies its caller. Ignore
3576        // consumer_identity there so a module cannot borrow a mirror's identity.
3577        // Only separate consumer connections use the Direct/nonce trust boundary.
3578        let principal = match self
3579            .forwarding
3580            .module_id_for_connection(ctx.connection_id)
3581            .map_err(RouterError::Forwarding)?
3582        {
3583            Some(module_id) => Ok(Principal::Reserved { module_id }),
3584            None => {
3585                let identity: Identity = match serde_json::from_slice(&frame.body) {
3586                    Ok(identity) => identity,
3587                    Err(_) => {
3588                        return Ok(vec![control_error_frame(
3589                            &frame,
3590                            ACTIVITY_MALFORMED,
3591                            "malformed activity read",
3592                        )?])
3593                    }
3594                };
3595                self.route_open_principal(&frame, identity.consumer_identity)?
3596            }
3597        };
3598        let admitted = matches!(principal, Ok(Principal::Direct))
3599            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3600        if !admitted {
3601            return Ok(vec![control_error_frame(
3602                &frame,
3603                ACTIVITY_READ_FORBIDDEN,
3604                "activity reads require an operator or trusted mirror principal",
3605            )?]);
3606        }
3607        let Some(ledger) = &self.activity else {
3608            return Ok(vec![control_error_frame(
3609                &frame,
3610                ACTIVITY_UNAVAILABLE,
3611                "activity ledger unavailable",
3612            )?]);
3613        };
3614        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3615            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3616        {
3617            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3618                Ok(request) => ledger.query(request).await,
3619                Err(_) => Err(ACTIVITY_MALFORMED),
3620            };
3621            return match result {
3622                Ok(reply) => Ok(vec![control_response_body_frame(
3623                    &frame,
3624                    &reply,
3625                    ops::ACTIVITY_QUERY,
3626                )?]),
3627                Err(code) => Ok(vec![control_error_frame(
3628                    &frame,
3629                    code,
3630                    "activity query refused",
3631                )?]),
3632            };
3633        }
3634        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3635            Ok(request) => request,
3636            Err(_) => {
3637                return Ok(vec![control_error_frame(
3638                    &frame,
3639                    ACTIVITY_MALFORMED,
3640                    "malformed activity subscription",
3641                )?])
3642            }
3643        };
3644        let mut subscription = match ledger.subscribe(request.since).await {
3645            Ok(subscription) => subscription,
3646            Err(code) => {
3647                return Ok(vec![control_error_frame(
3648                    &frame,
3649                    code,
3650                    "activity subscription refused",
3651                )?])
3652            }
3653        };
3654        let key = (ctx.connection_id, frame.header.corr);
3655        let sink = ctx.egress.clone();
3656        let mut streams = self
3657            .activity_streams
3658            .lock()
3659            .unwrap_or_else(|p| p.into_inner());
3660        streams.retain(|_, task| !task.is_finished());
3661        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3662            return Ok(vec![control_error_frame(
3663                &frame,
3664                "duplicate_subscription",
3665                "activity subscription correlation id already in use",
3666            )?]);
3667        }
3668        let task = tokio::spawn(async move {
3669            loop {
3670                let entry = if let Some(entry) = subscription.replay.pop_front() {
3671                    Some(entry)
3672                } else {
3673                    subscription.receiver.recv().await
3674                };
3675                let Some(entry) = entry else {
3676                    break;
3677                };
3678                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3679                    return;
3680                };
3681                let Ok(next) = Frame::build_with_version(
3682                    response_version(&frame),
3683                    FrameType::StreamData,
3684                    control_flags(),
3685                    0,
3686                    0,
3687                    frame.header.corr,
3688                    body,
3689                ) else {
3690                    return;
3691                };
3692                if sink.send(next).await.is_err() {
3693                    return;
3694                }
3695            }
3696            // Dropping a full subscriber's sender drains its accepted entries
3697            // before the terminal error, preserving their order on the wire.
3698            if let Some(code) = subscription.terminal_code() {
3699                if let Ok(end) =
3700                    control_error_frame(&frame, code, "activity subscriber fell behind")
3701                {
3702                    let _ = sink.send(end).await;
3703                }
3704            }
3705        });
3706        streams.insert(key, task.abort_handle());
3707        Ok(Vec::new())
3708    }
3709
3710    async fn handle_activity_record(
3711        &self,
3712        ctx: &RouteCtx,
3713        frame: Frame,
3714    ) -> Result<Vec<Frame>, RouterError> {
3715        use subc_control::{
3716            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3717            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3718        };
3719        let result = async {
3720            let module = self
3721                .forwarding
3722                .module_id_for_connection(ctx.connection_id)
3723                .ok()
3724                .flatten()
3725                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3726            let request: ActivityRecordRequest =
3727                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3728            crate::activity::validate(&module, &request)?;
3729            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3730            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3731            let at_ms = std::time::SystemTime::now()
3732                .duration_since(std::time::UNIX_EPOCH)
3733                .unwrap_or_default()
3734                .as_millis() as u64;
3735            ledger.record(module, request, actor, at_ms).await
3736        }
3737        .await;
3738        match result {
3739            Ok(entry) => Ok(vec![control_response_body_frame(
3740                &frame,
3741                &ActivityRecordResponse { seq: entry.seq },
3742                "activity.record",
3743            )?]),
3744            Err(code) => Ok(vec![control_error_frame(
3745                &frame,
3746                code,
3747                "activity recording refused",
3748            )?]),
3749        }
3750    }
3751
3752    fn activity_actor(
3753        &self,
3754        connection: ConnectionId,
3755        module: &str,
3756        on: Option<&subc_control::ActivityOn>,
3757    ) -> Result<subc_control::ActivityActor, &'static str> {
3758        use crate::forwarding::{DataRoute, DataRouteState};
3759        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3760        let principal = |id: &str| Principal::Reserved {
3761            module_id: id.to_owned(),
3762        };
3763        let mut actor = ActivityActor {
3764            principal: principal(module),
3765            scope_owner: None,
3766            agent_id: None,
3767            run_id: None,
3768            flow_id: None,
3769            project_root: None,
3770        };
3771        let scope = match on {
3772            None => return Ok(actor),
3773            Some(ActivityOn::Scope(scope)) => {
3774                let selector = subc_protocol::scope::ScopeSelector::new(
3775                    principal(module),
3776                    scope.scope_ref.clone(),
3777                    Some(scope.epoch),
3778                );
3779                self.scopes
3780                    .read()
3781                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3782                    .attribute(&principal(module), module, &selector, true)
3783                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3784            }
3785            Some(ActivityOn::Route(route)) => {
3786                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3787                    .forwarding
3788                    .lookup_data_route(connection, route.channel, route.epoch)
3789                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3790                else {
3791                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3792                };
3793                if binding.module_id != module {
3794                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3795                }
3796                actor.principal = binding.principal.clone();
3797                actor.project_root = binding
3798                    .project_root
3799                    .as_ref()
3800                    .map(|root| root.as_path().to_path_buf());
3801                let Some(bound) = &binding.scope else {
3802                    return Ok(actor);
3803                };
3804                let owner = principal(&bound.owner);
3805                let selector = subc_protocol::scope::ScopeSelector::new(
3806                    owner.clone(),
3807                    bound.scope_ref.clone(),
3808                    Some(bound.tag.scope_epoch),
3809                );
3810                let admitted = self
3811                    .scopes
3812                    .read()
3813                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3814                    .attribute(&owner, module, &selector, true)
3815                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3816                if admitted.tag != bound.tag {
3817                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3818                }
3819                admitted
3820            }
3821        };
3822        actor.scope_owner = Some(scope.stamp.owner);
3823        actor.agent_id = scope.stamp.attributes.agent_id;
3824        actor.run_id = scope.stamp.attributes.run_id;
3825        actor.flow_id = scope.stamp.attributes.flow_id;
3826        Ok(actor)
3827    }
3828
3829    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3830        self.refresh_capability_requirements();
3831        // A bare connection count is ambiguous between many clients holding a
3832        // route each and one client accumulating hundreds, so publish the
3833        // concentration alongside it. Route state is best-effort here: a
3834        // diagnostic endpoint must still answer if the forwarding lock is
3835        // contended.
3836        let mut counters = self.counters.snapshot();
3837        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3838            obj.insert(
3839                "activity_unparseable_lines".into(),
3840                ledger.read_errors().into(),
3841            );
3842            obj.insert(
3843                "activity_lifecycle_append_failures".into(),
3844                ledger.lifecycle_failures().into(),
3845            );
3846            obj.insert(
3847                "activity_started_digest_skips".into(),
3848                ledger.lifecycle_digest_skips().into(),
3849            );
3850        }
3851        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3852            self.forwarding.client_route_concentration(),
3853            counters.as_object_mut(),
3854        ) {
3855            obj.insert(
3856                "client_connections_with_routes".into(),
3857                connections_with_routes.into(),
3858            );
3859            obj.insert("max_routes_on_one_connection".into(), max.into());
3860        }
3861        // A module that is being fast-refused and a module that is fine look
3862        // identical from a client that retries and succeeds, so name the open
3863        // breakers here. This rides the existing free-form counters object
3864        // rather than a new wire field, so no sibling that deserializes
3865        // `ServerDescribe` has to be rebuilt to keep reading it.
3866        if let (Some(open_breakers), Some(obj)) = (
3867            self.route_bind_breakers.open_snapshot(),
3868            counters.as_object_mut(),
3869        ) {
3870            obj.insert("route_bind_breakers_open".into(), open_breakers);
3871        }
3872        let response = ClientControlResponse::ServerDescribe {
3873            protocol_ver: PROTOCOL_VERSION,
3874            subc_ops: subc_ops(),
3875            capabilities: self.describe_capabilities(),
3876            connected_clients: self.connected_clients.count(),
3877            counters: Some(counters),
3878            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3879            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3880            capability_requirements: self.capability_requirement_statuses(),
3881            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3882            config_path: self
3883                .rescan
3884                .as_ref()
3885                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3886        };
3887        Ok(vec![control_response_body_frame(
3888            &frame,
3889            &response,
3890            "ClientControlResponse::ServerDescribe",
3891        )?])
3892    }
3893
3894    fn handle_catalog_list(
3895        &self,
3896        frame: Frame,
3897        module_id: Option<String>,
3898    ) -> Result<Vec<Frame>, RouterError> {
3899        let (generation, modules) = self.registry.list_modules().map_err(|err| {
3900            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3901        })?;
3902        let entries = modules
3903            .into_iter()
3904            .filter(|registration| {
3905                module_id
3906                    .as_deref()
3907                    .map(|wanted| registration.manifest.module_id == wanted)
3908                    .unwrap_or(true)
3909            })
3910            .map(|registration| {
3911                let not_ready = self.not_ready_reason(&registration);
3912                let roles = registration.manifest.provides;
3913                CatalogEntry::new(
3914                    registration.manifest.module_id,
3915                    roles,
3916                    registration.control_ops,
3917                )
3918                .with_ready(not_ready.is_none())
3919                .with_not_ready(not_ready)
3920                .with_module_version(Some(registration.manifest.module_version))
3921                .with_capabilities(registration.manifest.capabilities)
3922                .with_self_signals(registration.manifest.self_signals)
3923                .with_events(registration.manifest.events)
3924                .with_operation_delivery(registration.manifest.operation_delivery)
3925            })
3926            .collect();
3927        let response = ClientControlResponse::CatalogList {
3928            generation,
3929            modules: entries,
3930            subc_ops: subc_ops(),
3931        };
3932        Ok(vec![control_response_body_frame(
3933            &frame,
3934            &response,
3935            "ClientControlResponse::CatalogList",
3936        )?])
3937    }
3938
3939    fn route_open_principal(
3940        &self,
3941        frame: &Frame,
3942        consumer_identity: Option<ConsumerIdentity>,
3943    ) -> Result<Result<Principal, Frame>, RouterError> {
3944        let Some(consumer_identity) = consumer_identity else {
3945            return Ok(Ok(Principal::Direct));
3946        };
3947
3948        if self.supervisor.spawned_consumer_authorized(
3949            &consumer_identity.module_id,
3950            &consumer_identity.launch_nonce,
3951        ) {
3952            return Ok(Ok(Principal::Reserved {
3953                module_id: consumer_identity.module_id,
3954            }));
3955        }
3956
3957        Ok(Err(control_error_frame(
3958            frame,
3959            "bad_consumer_identity",
3960            format!(
3961                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3962                consumer_identity.module_id
3963            ),
3964        )?))
3965    }
3966
3967    /// Ordinary `route.open` refusals go through here; admission and breaker
3968    /// refusals log separately with their capacity or breaker state. The daemon can
3969    /// attest which code it sent: without the event, a client's "the daemon
3970    /// refused me" and the daemon's own view could only be reconciled by
3971    /// argument. Malformed input (`invalid_project_root`) does not come here;
3972    /// rejecting a request that was never a valid open is not a refusal of one.
3973    fn route_open_refusal_frame(
3974        &self,
3975        ctx: &RouteCtx,
3976        frame: &Frame,
3977        module_id: &str,
3978        reason: &'static str,
3979        code: &'static str,
3980        message: impl Into<String>,
3981    ) -> Result<Frame, RouterError> {
3982        self.observe_route_open_refusal(ctx, module_id, reason, code);
3983        control_error_frame(frame, code, message.into())
3984    }
3985
3986    /// Refuse a `route.open` because the target module's bind-relay breaker is
3987    /// open, without attempting the relay.
3988    ///
3989    /// The wire code is `module_timeout`, which is the truth (the module has
3990    /// not been answering binds) and which both SDKs already classify as
3991    /// retryable with capped backoff. Reusing it is what keeps this change out
3992    /// of both SDKs; the daemon-side distinction lives in the counter key
3993    /// instead.
3994    ///
3995    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
3996    /// While a breaker is open this fires on every open to that module, and the
3997    /// stall written up in `docs/designs/route-open-head-of-line.md` already
3998    /// produced 261 lines about a single module inside 3000 lines of daemon
3999    /// log. The rare transitions are logged at warn/info instead and the volume
4000    /// is carried by the counter, so the evidence survives without the flood.
4001    /// The debug line keeps a per-refusal record reachable for whoever turns
4002    /// the level up.
4003    fn route_open_breaker_refusal_frame(
4004        &self,
4005        ctx: &RouteCtx,
4006        frame: &Frame,
4007        module_id: &str,
4008        consecutive_timeouts: u32,
4009        retry_in: Duration,
4010        probe_in_flight: bool,
4011    ) -> Result<Frame, RouterError> {
4012        self.counters
4013            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4014        debug!(
4015            target: "control",
4016            code = "module_timeout",
4017            module_id = ?module_id,
4018            connection_id = ctx.connection_id.get(),
4019            consecutive_timeouts,
4020            retry_in_ms = retry_in.as_millis() as u64,
4021            probe_in_flight,
4022            "route.open refused by open bind-relay breaker"
4023        );
4024        // Say what a caller can act on. An open bind-relay breaker means the
4025        // module timed out accepting several new routes in a row. The module
4026        // is still running and its established routes keep working; only new
4027        // route.open requests are refused until the cooldown ends and one
4028        // test route (the probe) gets through. A message that only counts
4029        // failed relays reads as "the module is down" to a worker that sees it.
4030        let detail = if probe_in_flight {
4031            "one test route is already being tried; retry once it settles".to_string()
4032        } else {
4033            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4034        };
4035        control_error_frame(
4036            frame,
4037            "module_timeout",
4038            format!(
4039                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4040                 timed out in a row); {detail}; its established routes are unaffected"
4041            ),
4042        )
4043    }
4044
4045    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4046    /// requester's bytes (an unknown target is whatever the client sent) and
4047    /// is Debug-formatted so control characters land in the log escaped
4048    /// rather than as terminal sequences for whoever tails it.
4049    ///
4050    /// `reason` names the check that refused, because one wire code has
4051    /// several senders: after a module registers, `target_unavailable` can
4052    /// come from a missing role, an inactive registration, a supervisor that
4053    /// has not marked the process live, a missing forwarding connection, or a
4054    /// failed relay, and a log that records only the code cannot say which of
4055    /// them fired. It is a static, daemon-chosen label per branch, so it is
4056    /// safe to print plainly and stays a closed set.
4057    fn observe_route_open_refusal(
4058        &self,
4059        ctx: &RouteCtx,
4060        module_id: &str,
4061        reason: &'static str,
4062        code: &'static str,
4063    ) {
4064        self.counters.increment_route_open_refused(code);
4065        info!(
4066            target: "control",
4067            code,
4068            reason,
4069            module_id = ?module_id,
4070            connection_id = ctx.connection_id.get(),
4071            "route.open refused"
4072        );
4073        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4074            self.route_outages.record_not_serving(module_id, reason);
4075        }
4076    }
4077
4078    /// Record an ACCEPTED route.open.
4079    ///
4080    /// Refusals have been logged and counted since the attestation work; accepts
4081    /// were invisible, so the daemon knew every principal it stamped and wrote
4082    /// none of them down. The party that attests the identity was the only party
4083    /// not recording it, which left a credential vault unable to name the sender
4084    /// of a call that reached it (claustrum #43) and left the launch-nonce
4085    /// concurrency question unanswerable from the outside.
4086    ///
4087    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4088    /// `code`/`module_id`/`connection_id` returns both directions of the same
4089    /// decision rather than two shapes a reader has to join by hand.
4090    ///
4091    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4092    /// and the difference carries information rather than being an
4093    /// inconsistency. This line is only reachable after a successful bind to a
4094    /// REGISTERED module, so the value has already passed HELLO validation
4095    /// including the path-hazard refusal and cannot contain control bytes. A
4096    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4097    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4098    ///
4099    /// Bare is also what every other daemon line already emits (`module
4100    /// registered`, `configured module supervised`). Shipping `?module_id` here
4101    /// made this instrument the only one in the file whose ids did not answer
4102    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4103    /// line whose whole purpose is being grepped beside its sibling.
4104    ///
4105    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4106    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4107    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4108    /// one are recorded identically. A test written against that harness passes
4109    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4110    /// green assertion that cannot fail. The same limit applies to the escaping
4111    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4112    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4113    /// Fencing either needs the real formatter, not the capture layer.
4114    ///
4115    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4116    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4117    /// to record. The ephemeral port would decay within minutes and answer only a
4118    /// live question. The identity question is instead answered by counting
4119    /// distinct live connections presenting one module's `consumer_identity` --
4120    /// "is anyone else holding this secret" rather than "is this the right
4121    /// process".
4122    fn observe_route_open_accept(
4123        &self,
4124        ctx: &RouteCtx,
4125        module_id: &str,
4126        principal: &str,
4127        scope: &str,
4128    ) {
4129        self.route_outages.record_accepted(module_id);
4130        self.counters.increment_route_open_accepted(principal);
4131        info!(
4132            target: "control",
4133            principal,
4134            module_id,
4135            connection_id = ctx.connection_id.get(),
4136            scope = %scope,
4137            "route.open accepted"
4138        );
4139    }
4140
4141    fn supervised_absent_route_open_refusal_frame(
4142        &self,
4143        ctx: &RouteCtx,
4144        frame: &Frame,
4145        module_id: &str,
4146        code: &'static str,
4147        status: &crate::supervise::ModuleStatus,
4148    ) -> Result<Frame, RouterError> {
4149        self.counters.increment_route_open_refused(code);
4150        info!(
4151            target: "control",
4152            code,
4153            reason = "supervised_not_registered",
4154            module_id = ?module_id,
4155            connection_id = ctx.connection_id.get(),
4156            state = %status.state,
4157            enabled = status.enabled,
4158            live = status.live,
4159            "route.open refused"
4160        );
4161        // A supervised module whose process has not registered is not
4162        // serving, whatever the reason; the supervisor knows this id, so it is
4163        // safe to track.
4164        self.route_outages
4165            .record_not_serving(module_id, "supervised_not_registered");
4166        let message = if let Some(pid) = status.waiting_for_previous_pid {
4167            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4168                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4169            })
4170        } else {
4171            format!(
4172                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4173                status.state, status.enabled, status.live
4174            )
4175        };
4176        control_error_frame(frame, code, message)
4177    }
4178
4179    async fn handle_route_open(
4180        &self,
4181        ctx: &RouteCtx,
4182        frame: Frame,
4183        request: RouteOpenRequest,
4184        timings: &mut RouteOpenTimings,
4185    ) -> Result<Vec<Frame>, RouterError> {
4186        let RouteOpenRequest {
4187            target,
4188            mut identity,
4189            consumer_identity,
4190            consumer_capabilities,
4191            role_versions,
4192            admission_facts,
4193            scope,
4194        } = request;
4195        let target_module_id = target_module_id(&target).to_string();
4196        timings.module_id.clone_from(&target_module_id);
4197        if self
4198            .registry
4199            .get_module_by_connection(ctx.connection_id)
4200            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4201            .is_some()
4202        {
4203            return Ok(vec![control_error_frame(
4204                &frame,
4205                "invalid_request",
4206                "module connections cannot open client routes",
4207            )?]);
4208        }
4209        debug!(
4210            connection_id = ctx.connection_id.get(),
4211            corr = frame.header.corr,
4212            module_id = %target_module_id,
4213            "handling route.open"
4214        );
4215
4216        // A malformed declaration is refused first, before anything about the
4217        // target is looked up: the same body would be refused against any
4218        // module, so the caller learns nothing by retrying or waiting. An empty
4219        // map declares nothing and travels as no field at all, so a provider
4220        // only ever sees a missing field or a non-empty one.
4221        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4222        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4223            self.observe_route_open_refusal(
4224                ctx,
4225                &target_module_id,
4226                "invalid_role_versions",
4227                error_codes::INVALID_REQUEST,
4228            );
4229            return Ok(vec![control_error_body_frame(
4230                &frame,
4231                ErrorBody {
4232                    code: error_codes::INVALID_REQUEST.to_string(),
4233                    message: error.to_string(),
4234                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4235                },
4236            )?]);
4237        }
4238
4239        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4240            if status.waiting_for_previous_pid.is_some() {
4241                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4242                    ctx,
4243                    &frame,
4244                    &target_module_id,
4245                    "module_warming",
4246                    &status,
4247                )?]);
4248            }
4249        }
4250
4251        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4252        // opposite. Below, a caller learns whether a module is unregistered,
4253        // supervised-but-down (with state/enabled/live), or registered without the
4254        // requested role. Elsewhere that is an enumeration leak: a probe learning
4255        // the shape of a fleet it cannot otherwise see.
4256        //
4257        // It is not one here, and the reason is the ACCESS MODEL rather than
4258        // anything about these errors. Reaching route.open requires the
4259        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4260        // connection file, so any caller who completes it already runs as this
4261        // user -- and can read subc.jsonc for the module list and `ck module
4262        // status` for live state. The reply discloses nothing the caller cannot
4263        // read more easily from disk, while the precision is load-bearing:
4264        // `unknown_module` is retryable and a missing role is not.
4265        //
4266        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4267        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4268        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4269        let Some(registration) = self
4270            .registry
4271            .get_module(&target_module_id)
4272            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4273        else {
4274            if let Some((status, warming)) =
4275                self.supervisor_status(&target_module_id, frame.header.corr)?
4276            {
4277                // BEFORE the two availability codes below, because for a module
4278                // that speaks no subc wire both of them are false comfort: they
4279                // say "not right now" and are retried, and this module will
4280                // never register no matter how long the caller waits. The
4281                // absence here is the declaration being honoured, not a module
4282                // that is late.
4283                if status.protocol == ModuleProtocol::None {
4284                    return Ok(vec![self.route_open_refusal_frame(
4285                        ctx,
4286                        &frame,
4287                        &target_module_id,
4288                        "protocol_none",
4289                        error_codes::MODULE_NO_PROTOCOL,
4290                        format!(
4291                            "module_id '{target_module_id}' is declared protocol: none; \
4292                             it speaks no subc wire and serves no routes"
4293                        ),
4294                    )?]);
4295                }
4296                let code = if warming {
4297                    "module_warming"
4298                } else {
4299                    "target_unavailable"
4300                };
4301                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4302                    ctx,
4303                    &frame,
4304                    &target_module_id,
4305                    code,
4306                    &status,
4307                )?]);
4308            }
4309            if let Some(removed_ago_ms) =
4310                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4311            {
4312                return Ok(vec![self.route_open_refusal_frame(
4313                    ctx,
4314                    &frame,
4315                    &target_module_id,
4316                    "removed",
4317                    error_codes::MODULE_REMOVED,
4318                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4319                )?]);
4320            }
4321            return Ok(vec![self.route_open_refusal_frame(
4322                ctx,
4323                &frame,
4324                &target_module_id,
4325                "not_registered",
4326                error_codes::UNKNOWN_MODULE,
4327                format!("module_id '{target_module_id}' is not registered"),
4328            )?]);
4329        };
4330
4331        // Best-effort only: registry readiness and forwarding reservation use
4332        // different locks, so a module can flip readiness between this read and
4333        // the relay. Modules must still tolerate an `on_bind` while not ready.
4334        if !registration.ready {
4335            self.counters
4336                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4337            info!(
4338                target: "control",
4339                code = error_codes::MODULE_WARMING,
4340                module_id = ?target_module_id,
4341                connection_id = ctx.connection_id.get(),
4342                reason = "declared_not_ready",
4343                "route.open refused"
4344            );
4345            // The module is registered but says it cannot take work, which is
4346            // an outage from the caller's side even though its process is up.
4347            self.route_outages
4348                .record_not_serving(&target_module_id, "declared_not_ready");
4349            return Ok(vec![control_error_body_frame(
4350                &frame,
4351                ErrorBody {
4352                    code: error_codes::MODULE_WARMING.to_string(),
4353                    message: format!(
4354                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4355                    ),
4356                    detail: Some(serde_json::json!({
4357                        "reason": "declared_not_ready"
4358                    })),
4359                },
4360            )?]);
4361        }
4362
4363        // Effective readiness, second half: a module that declares a capability
4364        // `need: required` is not routable while that capability has no
4365        // registered provider. It is enforced HERE, as a retryable routing
4366        // refusal, and deliberately not as spawn ordering or a boot block. The
4367        // module is still started and registered and can make its own calls;
4368        // spawn ordering is a promise that cannot be kept once a provider
4369        // crashes at runtime, and refusing to boot would stop the whole
4370        // machine, including the tools needed to fix its configuration.
4371        //
4372        // "Provided" is the evaluator's verdict, which counts a provider as
4373        // soon as it has REGISTERED, not once it is ready. Two modules that
4374        // require each other's capabilities are therefore both routable once
4375        // both register; counting readiness instead would deadlock them.
4376        //
4377        // Only new opens are refused. Routes already bound when a provider
4378        // goes away stay bound: nothing here tears them down, and the module
4379        // answers them as it can. Like the readiness read above this is
4380        // best-effort against a provider registering or leaving concurrently.
4381        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4382            self.counters
4383                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4384            info!(
4385                target: "control",
4386                code = error_codes::MODULE_WARMING,
4387                module_id = ?target_module_id,
4388                connection_id = ctx.connection_id.get(),
4389                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4390                capability = %capability,
4391                "route.open refused"
4392            );
4393            return Ok(vec![control_error_body_frame(
4394                &frame,
4395                ErrorBody {
4396                    code: error_codes::MODULE_WARMING.to_string(),
4397                    message: format!(
4398                        "module_id '{target_module_id}' requires capability '{capability}', \
4399                         which no registered module provides; retry"
4400                    ),
4401                    detail: Some(serde_json::json!({
4402                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4403                        "capability": capability,
4404                    })),
4405                },
4406            )?]);
4407        }
4408
4409        if !target_has_required_role(&target, &registration.manifest.provides) {
4410            return Ok(vec![self.route_open_refusal_frame(
4411                ctx,
4412                &frame,
4413                &target_module_id,
4414                "role_not_provided",
4415                "target_unavailable",
4416                format!("module_id '{target_module_id}' does not provide the requested target"),
4417            )?]);
4418        }
4419
4420        if registration.state != ChannelState::Active {
4421            return Ok(vec![self.route_open_refusal_frame(
4422                ctx,
4423                &frame,
4424                &target_module_id,
4425                "registration_not_active",
4426                "target_unavailable",
4427                format!("module_id '{target_module_id}' is not active"),
4428            )?]);
4429        }
4430
4431        if self
4432            .forwarding
4433            .module_is_draining(&target_module_id)
4434            .map_err(RouterError::Forwarding)?
4435        {
4436            return Ok(vec![self.route_open_refusal_frame(
4437                ctx,
4438                &frame,
4439                &target_module_id,
4440                "reloading",
4441                "module_reloading",
4442                format!("module_id '{target_module_id}' is reloading"),
4443            )?]);
4444        }
4445
4446        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4447            process_liveness.process_live(&target_module_id) == Some(false)
4448        }) {
4449            // A module the supervisor is restarting or reloading can still hold
4450            // a registration: the old process before its connection closes, or
4451            // a new one that registered while the supervisor was draining. The
4452            // forwarding table does not see that as draining, but the consumer
4453            // should still be told to retry soon, exactly as for the drain
4454            // above, rather than that the target is unavailable.
4455            if process_liveness.process_replacing(&target_module_id) {
4456                return Ok(vec![self.route_open_refusal_frame(
4457                    ctx,
4458                    &frame,
4459                    &target_module_id,
4460                    "reloading",
4461                    "module_reloading",
4462                    format!("module_id '{target_module_id}' is reloading"),
4463                )?]);
4464            }
4465            return Ok(vec![self.route_open_refusal_frame(
4466                ctx,
4467                &frame,
4468                &target_module_id,
4469                "supervisor_not_live",
4470                "target_unavailable",
4471                format!("module_id '{target_module_id}' is not live"),
4472            )?]);
4473        }
4474
4475        if !self
4476            .forwarding
4477            .has_live_module_connection(&target_module_id)
4478            .map_err(RouterError::Forwarding)?
4479        {
4480            return Ok(vec![self.route_open_refusal_frame(
4481                ctx,
4482                &frame,
4483                &target_module_id,
4484                "no_forwarding_connection",
4485                "target_unavailable",
4486                format!("module_id '{target_module_id}' has no live forwarding connection"),
4487            )?]);
4488        }
4489
4490        if let Some(error) =
4491            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4492        {
4493            self.observe_route_open_refusal(
4494                ctx,
4495                &target_module_id,
4496                "op_not_allowed",
4497                "op_not_allowed",
4498            );
4499            return Ok(vec![error]);
4500        }
4501
4502        let principal = match self.route_open_principal(&frame, consumer_identity)? {
4503            Ok(principal) => principal,
4504            Err(error) => {
4505                self.observe_route_open_refusal(
4506                    ctx,
4507                    &target_module_id,
4508                    "bad_consumer_identity",
4509                    "bad_consumer_identity",
4510                );
4511                return Ok(vec![error]);
4512            }
4513        };
4514        timings.principal = principal_label(&principal);
4515
4516        // This is attested, control-plane policy for supervised module origins.
4517        // Keep it before route reservation and out of the opaque forwarding hot
4518        // path: data frames must never acquire a per-frame capability check.
4519        if let Principal::Reserved {
4520            module_id: opening_module_id,
4521        } = &principal
4522        {
4523            if let Some(opening_registration) = self
4524                .registry
4525                .get_module(opening_module_id)
4526                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4527            {
4528                if let Some(capability) =
4529                    denied_capability(&opening_registration.manifest, &registration.manifest)
4530                {
4531                    warn!(
4532                        opening_module_id,
4533                        target_module_id,
4534                        capability,
4535                        "refusing route.open because an attested capability deny edge matches"
4536                    );
4537                    return Ok(vec![self.route_open_refusal_frame(
4538                        ctx,
4539                        &frame,
4540                        &target_module_id,
4541                        "capability_deny_edge",
4542                        "capability_forbidden",
4543                        format!(
4544                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4545                        ),
4546                    )?]);
4547                }
4548            }
4549        }
4550
4551        if admission_facts.is_some() {
4552            let carrier_matches = matches!(
4553                &principal,
4554                Principal::Reserved { module_id }
4555                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4556            );
4557            if !carrier_matches {
4558                return Ok(vec![self.route_open_refusal_frame(
4559                    ctx,
4560                    &frame,
4561                    &target_module_id,
4562                    "admission_facts_carrier_not_permitted",
4563                    "admission_facts_not_permitted",
4564                    "admission facts may only be carried by the configured reserved module",
4565                )?]);
4566            }
4567
4568            let target_allowed = self
4569                .admission_facts_targets
4570                .as_ref()
4571                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4572            if !target_allowed {
4573                return Ok(vec![self.route_open_refusal_frame(
4574                    ctx,
4575                    &frame,
4576                    &target_module_id,
4577                    "admission_facts_target_not_listed",
4578                    "admission_facts_target_not_allowed",
4579                    format!(
4580                        "admission facts are not permitted for target module_id '{target_module_id}'"
4581                    ),
4582                )?]);
4583            }
4584
4585            // Keep the value opaque to subc. The downstream admission validator owns
4586            // schema and semantic checks; this daemon only enforces carrier authority
4587            // and the configured destination allowlist.
4588        }
4589
4590        // Scope admission, on the attested principal above and never on the
4591        // request body. The tag read here travels with the pending bind and is
4592        // compared with the published one at commit, so a sync between here
4593        // and the module's ack refuses the open instead of binding a stamp
4594        // that is no longer true.
4595        let (bound_scope, mut scope_stamp) = match &scope {
4596            None => (None, None),
4597            Some(selector) => {
4598                let owner_configured = match &selector.owner {
4599                    // A reserved owner counts as configured from before its
4600                    // process is spawned (see `SupervisorHandle::is_configured`).
4601                    // So an owner that has not synced its scopes yet is refused
4602                    // as retryable (`scope_not_synced`), not as one that will
4603                    // never sync.
4604                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4605                    _ => false,
4606                };
4607                let mut scopes = self
4608                    .scopes
4609                    .write()
4610                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4611                let admitted = scopes
4612                    .admit(&principal, &target_module_id, selector, owner_configured)
4613                    .and_then(|admission| {
4614                        crate::scopes::check_target_flow_support(
4615                            &admission.stamp,
4616                            &target_module_id,
4617                            registration.manifest.capabilities.as_ref(),
4618                        )?;
4619                        crate::scopes::check_target_agent_run_support(
4620                            &admission.stamp,
4621                            &target_module_id,
4622                            registration.manifest.capabilities.as_ref(),
4623                        )?;
4624                        let bound = scopes.reserve_bind(
4625                            &admission,
4626                            &principal,
4627                            ctx.connection_id,
4628                            selector.hold_presence,
4629                        );
4630                        Ok((admission, bound))
4631                    });
4632                match admitted {
4633                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4634                    Err(refusal) => {
4635                        return Ok(vec![self.route_open_refusal_frame(
4636                            ctx,
4637                            &frame,
4638                            &target_module_id,
4639                            refusal.code,
4640                            refusal.code,
4641                            refusal.message,
4642                        )?]);
4643                    }
4644                }
4645            }
4646        };
4647
4648        // Bind admits a root that no longer exists on disk, because refusing here
4649        // closes the only exit from a paused run: cancel needs a bound route, and a
4650        // renamed or reclaimed directory makes that route unopenable forever. The
4651        // run itself is intact and still addressable by its recorded identity.
4652        //
4653        // This does NOT relax the rule the strict constructor protects. That rule is
4654        // that no root is ever aliased into NEW durable state -- a missing component
4655        // can reappear as a symlink elsewhere, which would move the identity and
4656        // split a session's history across two of them. The engine now refuses the
4657        // two operations that create such state (send and import) at admission,
4658        // which is a narrower way to hold the same invariant: reads and terminations
4659        // are admitted, writes are not. That refusal had to ship before this line
4660        // changed, or there is an interval where a send commits under a provisional
4661        // identity -- the exact failure the original policy existed to prevent.
4662        //
4663        // Resolution follows realpath rather than lexical cleanup: the longest
4664        // existing ancestor is canonicalized and the missing tail re-appended, so a
4665        // live root is unchanged and a vanished leaf keeps the identity it was
4666        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4667        // same caller the moment the directory vanished, which strands the run more
4668        // quietly than refusing it.
4669        timings.enter(RouteOpenPhase::ProjectRoot);
4670        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4671            Ok(project_root) => project_root,
4672            Err(err) => {
4673                return Ok(vec![control_error_frame(
4674                    &frame,
4675                    "invalid_project_root",
4676                    err.to_string(),
4677                )?])
4678            }
4679        };
4680        timings.enter(RouteOpenPhase::Admission);
4681        identity.project_root = project_root.as_path().to_path_buf();
4682
4683        // Last gate before any relay work, and deliberately after the cheap
4684        // registry and availability checks above: those name a more precise
4685        // condition (unknown, removed, reloading) and a caller is better served
4686        // by the precise code than by this one.
4687        //
4688        // Everything below this point costs an egress permit, a reserved handle
4689        // pair and, if the module does not answer, the whole relay budget. The
4690        // reader no longer waits for that budget, so cap each target explicitly;
4691        // serial dispatch used to provide the accidental cap of one relay per
4692        // connection. Admission is a mutex-protected count and never waits.
4693        let _concurrency_guard = match self
4694            .route_bind_concurrency
4695            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4696        {
4697            Ok(guard) => guard,
4698            Err(in_flight) => {
4699                return Ok(vec![self.route_open_target_capacity_refusal(
4700                    ctx,
4701                    &frame,
4702                    &target_module_id,
4703                    in_flight,
4704                )?]);
4705            }
4706        };
4707
4708        // A module that has already burned the whole budget `threshold` times
4709        // in a row does not get to charge it again until a probe says it recovered.
4710        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4711            RouteBindAdmission::Admitted { guard, probe } => {
4712                if probe {
4713                    info!(
4714                        module_id = %target_module_id,
4715                        connection_id = ctx.connection_id.get(),
4716                        "route.bind breaker half-open: admitting one probe"
4717                    );
4718                }
4719                guard
4720            }
4721            RouteBindAdmission::Refused {
4722                consecutive_timeouts,
4723                retry_in,
4724                probe_in_flight,
4725            } => {
4726                return Ok(vec![self.route_open_breaker_refusal_frame(
4727                    ctx,
4728                    &frame,
4729                    &target_module_id,
4730                    consecutive_timeouts,
4731                    retry_in,
4732                    probe_in_flight,
4733                )?]);
4734            }
4735        };
4736
4737        // Resolve the per-module budget here so the wait matches the operator's
4738        // intent for this specific target. A per-module override in
4739        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4740        // daemons) wins over the daemon-wide default.
4741        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4742        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4743        timings.enter(RouteOpenPhase::EgressReservation);
4744        let pending = match self
4745            .forwarding
4746            .begin_route_bind_relay_for(
4747                ctx.connection_id,
4748                ctx.egress.clone(),
4749                response_version(&frame),
4750                frame.header.corr,
4751                &target_module_id,
4752                principal.clone(),
4753                bound_scope,
4754                Some(project_root),
4755                relay_deadline,
4756            )
4757            .await
4758        {
4759            Ok(pending) => pending,
4760            Err(err) => {
4761                return Ok(vec![self.route_open_refusal_frame(
4762                    ctx,
4763                    &frame,
4764                    &target_module_id,
4765                    "relay_reservation_failed",
4766                    forwarding_error_code(&err),
4767                    err.to_string(),
4768                )?])
4769            }
4770        };
4771        timings.enter(RouteOpenPhase::Admission);
4772        let crate::forwarding::PendingRouteBindRelay {
4773            endpoint,
4774            module_sink,
4775            negotiated_ver,
4776            client_channel,
4777            client_epoch,
4778            module_channel,
4779            module_epoch,
4780            corr: relay_corr,
4781            receiver,
4782        } = pending;
4783        let mut reservation =
4784            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4785
4786        // Reserving egress can wait while a module reconnects or a swap cuts
4787        // over. Check the relay's captured connection, not the earlier by-id
4788        // lookup: the original module's flow/run capabilities cannot authorize
4789        // its replacement. The captured sink stays bound to that connection.
4790        let relay_registration = self
4791            .registry
4792            .get_module_by_connection(endpoint.connection_id)
4793            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4794        let relay_capabilities = relay_registration
4795            .as_ref()
4796            .and_then(|registration| registration.manifest.capabilities.as_ref());
4797        if let Some(selector) = &scope {
4798            // Egress reservation may wait across a scope update. Recheck the
4799            // proof against the current record before sending the captured bind.
4800            if let Err(refusal) = self
4801                .scopes
4802                .read()
4803                .unwrap_or_else(|poisoned| poisoned.into_inner())
4804                .check_bind_proof(&principal, selector)
4805            {
4806                reservation.release_and_disarm();
4807                return Ok(vec![self.route_open_refusal_frame(
4808                    ctx,
4809                    &frame,
4810                    &target_module_id,
4811                    refusal.code,
4812                    refusal.code,
4813                    refusal.message,
4814                )?]);
4815            }
4816        }
4817        if let Some(stamp) = scope_stamp
4818            .as_ref()
4819            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4820        {
4821            if let Err(refusal) = crate::scopes::check_target_flow_support(
4822                stamp,
4823                &target_module_id,
4824                relay_capabilities,
4825            )
4826            .and_then(|()| {
4827                crate::scopes::check_target_agent_run_support(
4828                    stamp,
4829                    &target_module_id,
4830                    relay_capabilities,
4831                )
4832            }) {
4833                reservation.release_and_disarm();
4834                return Ok(vec![self.route_open_refusal_frame(
4835                    ctx,
4836                    &frame,
4837                    &target_module_id,
4838                    refusal.code,
4839                    refusal.code,
4840                    refusal.message,
4841                )?]);
4842            }
4843        }
4844
4845        // Older readers may reject unknown fields. Select optional fields from
4846        // the captured endpoint's declaration, including on unscoped binds.
4847        if !relay_capabilities.is_some_and(|caps| {
4848            caps.provides
4849                .iter()
4850                .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4851        }) {
4852            if let Some(stamp) = scope_stamp.as_mut() {
4853                stamp.attributes.on_behalf_of_agent = None;
4854            }
4855        }
4856        let call_mode = if relay_capabilities
4857            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4858        {
4859            Some(derive_call_mode(scope_stamp.as_ref()))
4860        } else {
4861            if let Some(stamp) = scope_stamp.as_mut() {
4862                stamp.expires_at_ms = None;
4863            }
4864            None
4865        };
4866
4867        debug!(
4868            connection_id = ctx.connection_id.get(),
4869            client_channel,
4870            client_epoch,
4871            module_channel,
4872            module_epoch,
4873            "reserved route handle pair"
4874        );
4875        // Rendered BEFORE the move into the relay, because the accept arm below
4876        // is where it is logged and the principal is gone by then.
4877        let route_principal_label = principal_label(&principal);
4878        let scope_label = scope_stamp
4879            .as_ref()
4880            .map_or_else(|| "none".to_string(), route_open_scope_label);
4881        let relay = ModuleControlRequest::RouteBind {
4882            route_channel: module_channel,
4883            epoch: module_epoch,
4884            target,
4885            identity,
4886            principal: Some(principal),
4887            consumer_capabilities,
4888            role_versions,
4889            admission_facts,
4890            scope: scope_stamp,
4891            call_mode,
4892        };
4893        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4894            RouterError::backend(
4895                0,
4896                frame.header.corr,
4897                format!("failed to encode route.bind request: {err}"),
4898            )
4899        })?;
4900        let relay_frame = Frame::build_with_version(
4901            negotiated_ver,
4902            FrameType::Request,
4903            control_flags(),
4904            0,
4905            0,
4906            relay_corr,
4907            relay_body,
4908        )
4909        .map_err(RouterError::FrameBuild)?;
4910
4911        timings.enter(RouteOpenPhase::BindSend);
4912        let sent = module_sink.send(relay_frame).await;
4913        timings.enter(RouteOpenPhase::Admission);
4914        if let Err(err) = sent {
4915            reservation.release_and_disarm();
4916            return Ok(vec![self.route_open_refusal_frame(
4917                ctx,
4918                &frame,
4919                &target_module_id,
4920                "relay_send_failed",
4921                "target_unavailable",
4922                err.to_string(),
4923            )?]);
4924        }
4925
4926        #[cfg(test)]
4927        if let Some(hold) = &self.route_bind_relay_sent_hold {
4928            // First wait: tell the test the bind is queued. Second wait: stay
4929            // here until the test has driven whatever it wants to happen
4930            // between queueing the bind and marking it enqueued.
4931            hold.wait().await;
4932            hold.wait().await;
4933        }
4934
4935        // A missing relay is not by itself an abandoned bind: the module may
4936        // already have acked it and the route may be committed. Only a relay
4937        // dropped without any module reply needs this GOODBYE; one a reply
4938        // settled has already had its GOODBYE decided by the reply path.
4939        if self
4940            .forwarding
4941            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4942            .map_err(RouterError::Forwarding)?
4943            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4944        {
4945            self.send_abandoned_route_bind_goodbye(
4946                &module_sink,
4947                negotiated_ver,
4948                module_channel,
4949                module_epoch,
4950            );
4951        }
4952
4953        timings.enter(RouteOpenPhase::BindAckWait);
4954        let outcome = timeout_at(relay_deadline, receiver).await;
4955        timings.enter(RouteOpenPhase::Admission);
4956        match outcome {
4957            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4958                reservation.disarm();
4959                if breaker.record_accepted() {
4960                    info!(
4961                        module_id = %target_module_id,
4962                        "route.bind breaker closed: the probe was accepted"
4963                    );
4964                }
4965                self.observe_route_open_accept(
4966                    ctx,
4967                    &target_module_id,
4968                    &route_principal_label,
4969                    &scope_label,
4970                );
4971                Ok(Vec::new())
4972            }
4973            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4974                reservation.release_and_disarm();
4975                // A module that says no in microseconds is healthy. Rejection
4976                // is a different condition with its own refusal and must not
4977                // move the breaker.
4978                breaker.record_inconclusive();
4979                // The daemon's own commit re-check refused the bind because the
4980                // scope ended or changed after admission. The module accepted;
4981                // counting it as a module rejection would blame the module.
4982                let scope_code = match body.code.as_str() {
4983                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4984                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4985                    _ => None,
4986                };
4987                if let Some(code) = scope_code {
4988                    self.observe_route_open_refusal(
4989                        ctx,
4990                        &target_module_id,
4991                        "scope_changed_before_commit",
4992                        code,
4993                    );
4994                    return Ok(vec![control_error_body_frame(&frame, body)?]);
4995                }
4996                self.counters
4997                    .increment_route_open_refused("module_rejected");
4998                info!(
4999                    target: "control",
5000                    code = "module_rejected",
5001                    module_code = ?body.code,
5002                    module_id = ?target_module_id,
5003                    connection_id = ctx.connection_id.get(),
5004                    "route.open refused"
5005                );
5006                Ok(vec![control_error_body_frame(&frame, body)?])
5007            }
5008            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5009                reservation.release_and_disarm();
5010                breaker.record_inconclusive();
5011                // Fires when the module's connection closes while a relayed
5012                // bind is pending -- typically a caller racing a module restart
5013                // whose bind was relayed BEFORE the drain mark went up. Logged
5014                // because the caller sees only its own error and the fleet has
5015                // already spent one diagnosis round unable to tell this arm
5016                // from a relay timeout without daemon-side evidence.
5017                tracing::warn!(
5018                    module_id = %target_module_id,
5019                    "route.bind relay abandoned: {message}"
5020                );
5021                Ok(vec![self.route_open_refusal_frame(
5022                    ctx,
5023                    &frame,
5024                    &target_module_id,
5025                    "relay_abandoned",
5026                    "target_unavailable",
5027                    message,
5028                )?])
5029            }
5030            Ok(Err(_)) => {
5031                reservation.release_and_disarm();
5032                breaker.record_inconclusive();
5033                Ok(vec![self.route_open_refusal_frame(
5034                    ctx,
5035                    &frame,
5036                    &target_module_id,
5037                    "relay_waiter_canceled",
5038                    "target_unavailable",
5039                    "route.bind relay waiter was canceled before the module responded",
5040                )?])
5041            }
5042            Err(_) => {
5043                reservation.release_and_disarm();
5044                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5045                // answer at all is the one condition a fast refusal can
5046                // usefully stand in for; every other arm already answered.
5047                if let Some(opened) = breaker.record_timeout(
5048                    self.route_bind_breaker_threshold,
5049                    self.route_bind_breaker_cooldown,
5050                ) {
5051                    warn!(
5052                        module_id = %target_module_id,
5053                        consecutive_timeouts = opened.consecutive_timeouts,
5054                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5055                        reopened_after_probe = opened.reopened_after_probe,
5056                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5057                    );
5058                }
5059                // The generous budget just burned to no answer: the module is
5060                // registered and its connection is up, but its bind handler sat
5061                // on the ack for the full budget (warm-on-bind, cold configure,
5062                // or a wedged handler). Every earlier unavailability shape
5063                // fast-refuses BEFORE the relay, so this arm firing means the
5064                // slowness is module-side -- log it so the per-module timeline
5065                // is reconstructable without client audit rows.
5066                tracing::warn!(
5067                    module_id = %target_module_id,
5068                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5069                    "route.bind relay timed out: module did not ack within budget"
5070                );
5071                Ok(vec![self.route_open_refusal_frame(
5072                    ctx,
5073                    &frame,
5074                    &target_module_id,
5075                    "relay_timed_out",
5076                    "module_timeout",
5077                    format!(
5078                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5079                        route_bind_relay_timeout
5080                    ),
5081                )?])
5082            }
5083        }
5084    }
5085
5086    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5087        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5088            snapshot: self.supervisor.spawn_snapshot(),
5089        };
5090        Ok(vec![control_response_body_frame(
5091            &frame,
5092            &response,
5093            "ClientControlResponse::SupervisorSpawnSnapshot",
5094        )?])
5095    }
5096
5097    fn handle_supervisor_spawn_subscribe(
5098        &self,
5099        ctx: &RouteCtx,
5100        frame: Frame,
5101        since: Option<SpawnCursor>,
5102    ) -> Result<Vec<Frame>, RouterError> {
5103        match self.supervisor.subscribe_spawns(
5104            ctx.connection_id,
5105            frame.header.corr,
5106            response_version(&frame),
5107            since,
5108            ctx.egress.clone(),
5109        ) {
5110            Ok(()) => Ok(Vec::new()),
5111            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5112                Ok(vec![control_error_body_frame(
5113                    &frame,
5114                    ErrorBody {
5115                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5116                        message: "spawn cursor belongs to a different daemon incarnation"
5117                            .to_string(),
5118                        detail: Some(serde_json::json!({
5119                            "current_daemon_incarnation": current
5120                        })),
5121                    },
5122                )?])
5123            }
5124            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5125                &frame,
5126                ErrorBody {
5127                    code: "spawn_cursor_too_old".to_string(),
5128                    message: "spawn cursor predates the retained event ring".to_string(),
5129                    detail: Some(serde_json::json!({
5130                        "oldest_retained_cursor": oldest
5131                    })),
5132                },
5133            )?]),
5134            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5135                0,
5136                frame.header.corr,
5137                format!("failed to open supervisor spawn subscription: {error}"),
5138            )),
5139        }
5140    }
5141
5142    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5143        let generation = self
5144            .registry
5145            .generation()
5146            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5147        let mut modules = Vec::new();
5148        for module in self.supervisor.list() {
5149            let status = module.status_for_control("list").map_err(|err| {
5150                RouterError::backend(
5151                    0,
5152                    frame.header.corr,
5153                    format!("failed to read supervisor status: {err}"),
5154                )
5155            })?;
5156            let (configured, _) = module.configuration().map_err(|err| {
5157                RouterError::backend(
5158                    0,
5159                    frame.header.corr,
5160                    format!("failed to read module configuration: {err}"),
5161                )
5162            })?;
5163            // Status and configuration snapshots release their locks before the image probe awaits.
5164            let image = module.running_image_agreement().await;
5165            // Read per request so the figure is current when the operator asks;
5166            // the daemon samples nothing in between.
5167            let resources = Some(module.child_resource_usage());
5168            let mut verdict =
5169                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5170            verdict.spec = Some(launch_spec_agreement(
5171                &configured,
5172                status.process_alive,
5173                status.launch_spec_digest.as_ref(),
5174            ));
5175            let pending_reload = Some(verdict);
5176            // Keep the retired policy field on the wire for one release so
5177            // existing status consumers still receive the platform policy.
5178            modules.push(
5179                SupervisorEntry::new(
5180                    status.module_id,
5181                    status.state.to_string(),
5182                    status.enabled,
5183                    status.live,
5184                    status.health.status,
5185                )
5186                .with_launch_nonce_env(Some(!cfg!(unix)))
5187                .with_launch_nonce_source(status.launch_nonce_source)
5188                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5189                .with_protocol(status.protocol)
5190                .with_pending_reload(pending_reload)
5191                .with_last_probe_ms(status.health.last_probe_ms)
5192                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5193                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5194                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5195                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5196                .with_restart_count(Some(status.restart_count))
5197                .with_max_restarts(Some(status.max_restarts))
5198                .with_lifetime_restarts(Some(status.lifetime_restarts))
5199                .with_spawn_generation(Some(status.spawn_generation))
5200                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5201                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5202                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5203                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5204                .with_resources(resources),
5205            );
5206        }
5207        let response = ClientControlResponse::SupervisorList {
5208            generation,
5209            modules,
5210        };
5211        Ok(vec![control_response_body_frame(
5212            &frame,
5213            &response,
5214            "ClientControlResponse::SupervisorList",
5215        )?])
5216    }
5217
5218    fn handle_supervisor_stderr_tail(
5219        &self,
5220        frame: Frame,
5221        module_id: String,
5222        max_lines: Option<u32>,
5223        max_bytes: Option<u32>,
5224    ) -> Result<Vec<Frame>, RouterError> {
5225        let Some(module) = self.supervisor.get(&module_id) else {
5226            return Ok(vec![control_error_frame(
5227                &frame,
5228                "unknown_module",
5229                format!("module_id '{module_id}' is not supervised"),
5230            )?]);
5231        };
5232
5233        let snapshot = module.stderr_tail(
5234            max_lines.map(|value| value as usize),
5235            max_bytes.map(|value| value as usize),
5236        );
5237
5238        let response = ClientControlResponse::SupervisorStderrTail {
5239            module_id,
5240            tail: StderrTail {
5241                capture: match snapshot.capture {
5242                    CaptureState::Captured => StderrCaptureState::Captured,
5243                    CaptureState::Incomplete { reason } => {
5244                        StderrCaptureState::Incomplete { reason }
5245                    }
5246                    CaptureState::NotCaptured { reason } => {
5247                        StderrCaptureState::NotCaptured { reason }
5248                    }
5249                },
5250                entries: snapshot
5251                    .entries
5252                    .into_iter()
5253                    .map(|entry| match entry {
5254                        TailEntry::Line {
5255                            text,
5256                            truncated,
5257                            at_ms,
5258                        } => StderrTailEntry::Line {
5259                            text,
5260                            truncated,
5261                            at_ms,
5262                        },
5263                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5264                    })
5265                    .collect(),
5266                dropped_lines: snapshot.dropped_lines,
5267            },
5268        };
5269        Ok(vec![control_response_body_frame(
5270            &frame,
5271            &response,
5272            "ClientControlResponse::SupervisorStderrTail",
5273        )?])
5274    }
5275
5276    async fn handle_supervisor_terminals(
5277        &self,
5278        frame: Frame,
5279        module_id: String,
5280    ) -> Result<Vec<Frame>, RouterError> {
5281        let Some(module) = self.supervisor.get(&module_id) else {
5282            return Ok(vec![control_error_frame(
5283                &frame,
5284                "unknown_module",
5285                format!("module_id '{module_id}' is not supervised"),
5286            )?]);
5287        };
5288
5289        // The journal read runs on a blocking thread: it can be megabytes of
5290        // file I/O and must not occupy a runtime worker.
5291        let terminals = module
5292            .read_durable_terminal_history()
5293            .await
5294            .map_err(|error| {
5295                RouterError::backend(
5296                    0,
5297                    frame.header.corr,
5298                    format!("failed to read terminal history: {error}"),
5299                )
5300            })?;
5301        let response = ClientControlResponse::SupervisorTerminals {
5302            module_id,
5303            terminals,
5304        };
5305        Ok(vec![control_response_body_frame(
5306            &frame,
5307            &response,
5308            "ClientControlResponse::SupervisorTerminals",
5309        )?])
5310    }
5311
5312    fn handle_supervisor_routes(
5313        &self,
5314        frame: Frame,
5315        module_id: Option<String>,
5316    ) -> Result<Vec<Frame>, RouterError> {
5317        let modules = self
5318            .forwarding
5319            .route_census(module_id.as_deref())
5320            .map_err(RouterError::Forwarding)?
5321            .into_iter()
5322            .map(|(module_id, routes)| SupervisorRouteModule {
5323                module_id,
5324                routes: routes
5325                    .into_iter()
5326                    .map(|route| SupervisorRoute {
5327                        consumer: match route.principal {
5328                            Principal::Reserved { module_id } => {
5329                                SupervisorRouteConsumer::Reserved { module_id }
5330                            }
5331                            Principal::Direct | Principal::Unverified => {
5332                                SupervisorRouteConsumer::Direct {
5333                                    connection_id: route.goodbye_target.connection_id.get(),
5334                                }
5335                            }
5336                        },
5337                        age_ms: Instant::now()
5338                            .saturating_duration_since(route.bound_at)
5339                            .as_millis()
5340                            .try_into()
5341                            .unwrap_or(u64::MAX),
5342                        draining: route.draining,
5343                        drain_reason: route.drain_reason,
5344                    })
5345                    .collect(),
5346            })
5347            .collect();
5348        let response = ClientControlResponse::SupervisorRoutes { modules };
5349        Ok(vec![control_response_body_frame(
5350            &frame,
5351            &response,
5352            "ClientControlResponse::SupervisorRoutes",
5353        )?])
5354    }
5355
5356    async fn handle_supervisor_provenance(
5357        &self,
5358        frame: Frame,
5359        module_id: Option<String>,
5360    ) -> Result<Vec<Frame>, RouterError> {
5361        let mut selected = if let Some(module_id) = module_id {
5362            let Some(module) = self.supervisor.get(&module_id) else {
5363                return Ok(vec![control_error_frame(
5364                    &frame,
5365                    "unknown_module",
5366                    format!("module_id '{module_id}' is not supervised"),
5367                )?]);
5368            };
5369            vec![module]
5370        } else {
5371            self.supervisor.list()
5372        };
5373
5374        let mut modules = Vec::with_capacity(selected.len());
5375        for module in selected.drain(..) {
5376            let (status, observed_image) = module
5377                .status_and_running_image_agreement()
5378                .await
5379                .map_err(|err| {
5380                    RouterError::backend(
5381                        0,
5382                        frame.header.corr,
5383                        format!("failed to read supervisor status: {err}"),
5384                    )
5385                })?;
5386            let module_declared = self
5387                .registry
5388                .get_module(&status.module_id)
5389                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5390                .and_then(|registration| registration.manifest.provenance)
5391                .map(|build| ModuleDeclaredProvenance::Reported { build })
5392                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5393            #[cfg(test)]
5394            let running_image = match &self.provenance_probe_override {
5395                Some(result) => result.clone(),
5396                None => observed_image,
5397            };
5398            #[cfg(not(test))]
5399            let running_image = observed_image;
5400            modules.push(SupervisorModuleProvenance {
5401                module_id: status.module_id,
5402                module_declared,
5403                daemon_observed: SupervisorObservedProcess {
5404                    launch_nonce_source: status.launch_nonce_source,
5405                    pid: status.pid,
5406                    spawned_at_ms: status.spawned_at_ms,
5407                    spawned_from: status.spawned_from,
5408                    running_image,
5409                },
5410            });
5411        }
5412        let daemon = SupervisorDaemonProvenance {
5413            daemon_build: self.daemon_provenance.build.clone(),
5414            daemon_observed: DaemonObservedProcess {
5415                pid: self.daemon_provenance.pid,
5416                started_at_ms: self
5417                    .daemon_provenance
5418                    .start_clock
5419                    .map(|clock| clock.started_at_ms())
5420                    .or(self.daemon_provenance.started_at_ms),
5421                running_image: self
5422                    .daemon_provenance
5423                    .probe
5424                    .observe(
5425                        self.daemon_provenance.pid,
5426                        self.daemon_provenance.executable_path.as_deref(),
5427                        self.daemon_provenance.executable_identity.clone(),
5428                        self.daemon_provenance.process_start_time,
5429                    )
5430                    .await,
5431            },
5432        };
5433        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5434        Ok(vec![control_response_body_frame(
5435            &frame,
5436            &response,
5437            "ClientControlResponse::SupervisorProvenance",
5438        )?])
5439    }
5440
5441    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5442        self.refresh_capability_requirements();
5443        let generation = self
5444            .registry
5445            .generation()
5446            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5447        let modules = self
5448            .supervisor
5449            .list()
5450            .into_iter()
5451            .map(|module| {
5452                let status = module.status_for_control("health").map_err(|err| {
5453                    RouterError::backend(
5454                        0,
5455                        frame.header.corr,
5456                        format!("failed to read supervisor health: {err}"),
5457                    )
5458                })?;
5459                let module_id = status.module_id;
5460                let capability_detail = self
5461                    .capability_evaluator
5462                    .required_problem_detail(&module_id);
5463                Ok(SupervisorHealthEntry {
5464                    module_id,
5465                    status: status.health.status,
5466                    detail: append_capability_problem_detail(
5467                        status.health.detail,
5468                        capability_detail,
5469                    ),
5470                    metrics: status.health.metrics,
5471                    consecutive_failures: status.health.consecutive_failures,
5472                    late_answer_count: status.health.late_answer_count,
5473                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5474                    last_action: status.health.last_action,
5475                    last_action_ms: status.health.last_action_ms,
5476                    last_probe_ms: status.health.last_probe_ms,
5477                })
5478            })
5479            .collect::<Result<Vec<_>, RouterError>>()?;
5480        let response = ClientControlResponse::SupervisorHealth {
5481            generation,
5482            modules,
5483        };
5484        Ok(vec![control_response_body_frame(
5485            &frame,
5486            &response,
5487            "ClientControlResponse::SupervisorHealth",
5488        )?])
5489    }
5490
5491    async fn handle_supervisor_restart(
5492        &self,
5493        frame: Frame,
5494        module_id: String,
5495        drain_timeout_ms: Option<u64>,
5496        requester: Principal,
5497    ) -> Result<Vec<Frame>, RouterError> {
5498        let operation_lock = self.supervisor.operation_lock();
5499        let _operation_guard = operation_lock.lock().await;
5500        let Some(module) = self.supervisor.get(&module_id) else {
5501            return Ok(vec![control_error_frame(
5502                &frame,
5503                "unknown_module",
5504                format!("module_id '{module_id}' is not supervised"),
5505            )?]);
5506        };
5507
5508        self.route_outages.mark_operator_action(&module_id);
5509        if let Err(err) = module
5510            .restart_requested(drain_timeout_ms, Some(requester))
5511            .await
5512        {
5513            self.route_outages
5514                .operator_action_ended_unrefused(&module_id);
5515            let (code, message) = match err {
5516                crate::supervise::SuperviseError::Disabled { .. } => {
5517                    ("module_disabled", err.to_string())
5518                }
5519                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5520                    ("swap_in_progress", err.to_string())
5521                }
5522                _ => (
5523                    "target_unavailable",
5524                    format!("failed to restart module_id '{module_id}': {err}"),
5525                ),
5526            };
5527            return Ok(vec![control_error_frame(&frame, code, message)?]);
5528        }
5529
5530        let response = ClientControlResponse::SupervisorAck {
5531            module_id,
5532            applied: true,
5533        };
5534        Ok(vec![control_response_body_frame(
5535            &frame,
5536            &response,
5537            "ClientControlResponse::SupervisorAck",
5538        )?])
5539    }
5540
5541    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5542    /// when the old process has finished draining: a caller whose own lane
5543    /// rides the old process must get its reply before that drain waits on it.
5544    async fn handle_supervisor_swap(
5545        &self,
5546        frame: Frame,
5547        module_id: String,
5548        ready_timeout_ms: Option<u64>,
5549        requester: Principal,
5550    ) -> Result<Vec<Frame>, RouterError> {
5551        // The daemon-wide operation lock is held only to resolve the handle,
5552        // not across the swap. The swap can take its whole readiness budget,
5553        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5554        // holding it here would park an operator's stop behind the swap it is
5555        // meant to abort. A rescan or stop that reaches the module during the
5556        // swap is served by the swap itself (see `supervise_swap`).
5557        let module = {
5558            let operation_lock = self.supervisor.operation_lock();
5559            let _operation_guard = operation_lock.lock().await;
5560            self.supervisor.get(&module_id)
5561        };
5562        let Some(module) = module else {
5563            return Ok(vec![control_error_frame(
5564                &frame,
5565                "unknown_module",
5566                format!("module_id '{module_id}' is not supervised"),
5567            )?]);
5568        };
5569
5570        self.route_outages.mark_operator_action(&module_id);
5571        if let Err(err) = module
5572            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5573            .await
5574        {
5575            self.route_outages
5576                .operator_action_ended_unrefused(&module_id);
5577            use crate::supervise::SuperviseError;
5578            let message = err.to_string();
5579            let error = match err {
5580                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5581                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5582                    code: "swap_refused".to_string(),
5583                    message,
5584                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5585                },
5586                SuperviseError::SwapFailed {
5587                    arm,
5588                    candidate_exit,
5589                    ..
5590                } => ErrorBody {
5591                    code: "swap_failed".to_string(),
5592                    message,
5593                    detail: Some(serde_json::json!({
5594                        "arm": arm.as_str(),
5595                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5596                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5597                    })),
5598                },
5599                _ => ErrorBody::new(
5600                    "target_unavailable",
5601                    format!("failed to swap module_id '{module_id}': {message}"),
5602                ),
5603            };
5604            return Ok(vec![control_error_body_frame(&frame, error)?]);
5605        }
5606        // A completed swap kept the incumbent serving until cutover, so it
5607        // usually opened no outage; a mark left behind would make the next,
5608        // unrelated outage read as requested.
5609        self.route_outages
5610            .operator_action_ended_unrefused(&module_id);
5611
5612        let response = ClientControlResponse::SupervisorAck {
5613            module_id,
5614            applied: true,
5615        };
5616        Ok(vec![control_response_body_frame(
5617            &frame,
5618            &response,
5619            "ClientControlResponse::SupervisorAck",
5620        )?])
5621    }
5622
5623    async fn handle_supervisor_reload(
5624        &self,
5625        frame: Frame,
5626        module_id: String,
5627        requester: Principal,
5628    ) -> Result<Vec<Frame>, RouterError> {
5629        let operation_lock = self.supervisor.operation_lock();
5630        let _operation_guard = operation_lock.lock().await;
5631        let Some(module) = self.supervisor.get(&module_id) else {
5632            return Ok(vec![control_error_frame(
5633                &frame,
5634                "unknown_module",
5635                format!("module_id '{module_id}' is not supervised"),
5636            )?]);
5637        };
5638
5639        self.route_outages.mark_operator_action(&module_id);
5640        if let Err(err) = module.reload_requested(Some(requester)).await {
5641            self.route_outages
5642                .operator_action_ended_unrefused(&module_id);
5643            let (code, message) = match err {
5644                crate::supervise::SuperviseError::Disabled { .. } => {
5645                    ("module_disabled", err.to_string())
5646                }
5647                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5648                    ("swap_in_progress", err.to_string())
5649                }
5650                _ => (
5651                    "reload_failed",
5652                    format!("failed to reload module_id '{module_id}': {err}"),
5653                ),
5654            };
5655            return Ok(vec![control_error_frame(&frame, code, message)?]);
5656        }
5657
5658        let response = ClientControlResponse::SupervisorAck {
5659            module_id,
5660            applied: true,
5661        };
5662        Ok(vec![control_response_body_frame(
5663            &frame,
5664            &response,
5665            "ClientControlResponse::SupervisorAck",
5666        )?])
5667    }
5668
5669    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5670    /// the configuration comes from and whether to apply it; every mode then
5671    /// runs the same comparison, so a preview describes the apply it previews.
5672    async fn handle_supervisor_rescan(
5673        &self,
5674        frame: Frame,
5675        mode: RescanMode,
5676        requester: Principal,
5677    ) -> Result<Vec<Frame>, RouterError> {
5678        let preview = mode.previews();
5679        let Some(context) = self.rescan.clone() else {
5680            return Ok(vec![control_error_frame(
5681                &frame,
5682                "rescan_unavailable",
5683                "the daemon was not started with a reloadable config path".to_string(),
5684            )?]);
5685        };
5686
5687        let operation_lock = self.supervisor.operation_lock();
5688        let _operation_guard = operation_lock.lock().await;
5689        // Loaded under the operation lock, so no other rescan can change the
5690        // staged configuration between a checked apply's comparison and the
5691        // apply itself.
5692        let loaded = match self.load_rescan_config(&mode, &context) {
5693            Ok(config) => config,
5694            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5695        };
5696        // `load` reports a missing file as Ok(None), which is correct at boot
5697        // (no config, nothing to supervise) and catastrophic here: rescan treats
5698        // "not in the config" as "remove it", so an absent file would read as an
5699        // empty module list and retire the entire running fleet. An editor
5700        // writing via write-new-then-rename, or a half-finished edit, is enough
5701        // to open that window. Refuse instead: a config that cannot be read
5702        // carries no instruction to remove anything.
5703        let Some((config, activity_config)) = loaded else {
5704            return Ok(vec![control_error_frame(
5705                &frame,
5706                "invalid_daemon_config",
5707                format!(
5708                    "daemon config not found at {}; refusing to rescan (an absent config would \
5709                     retire every supervised module)",
5710                    context.config_path.display()
5711                ),
5712            )?]);
5713        };
5714        let (
5715            configured_port,
5716            storage_config,
5717            admission_facts_carrier_module_id,
5718            admission_facts_targets,
5719            scope_authority_owners,
5720            privileged_operator_prompt,
5721            modules,
5722            reserved_capabilities,
5723        ) = (
5724            config.port,
5725            config.storage,
5726            config.admission_facts_carrier_module_id,
5727            config.admission_facts_targets,
5728            config.scope_authority_owners,
5729            config.privileged_operator_prompt,
5730            config.modules,
5731            config.reserved_capabilities,
5732        );
5733
5734        // Collect the sections rescan cannot apply, so the REPLY carries them.
5735        //
5736        // The warning below has always been correct and has always gone only to
5737        // the journal -- addressed to whoever reads logs, while the person who
5738        // just edited the config is looking at the CLI. Naming each section
5739        // individually rather than setting a flag: "something outside modules
5740        // changed" sends the operator back to diffing their own file, which is
5741        // the work this is meant to save.
5742        let mut restart_required = Vec::new();
5743        for section in RestartRequiredSection::ALL {
5744            let changed = match section {
5745                RestartRequiredSection::Port => configured_port != context.configured_port,
5746                RestartRequiredSection::Storage => storage_config != context.storage_config,
5747                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5748                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5749                }
5750                RestartRequiredSection::AdmissionFactsTargets => {
5751                    admission_facts_targets != context.admission_facts_targets
5752                }
5753                RestartRequiredSection::ScopeAuthorityOwners => {
5754                    scope_authority_owners != context.scope_authority_owners
5755                }
5756                RestartRequiredSection::PrivilegedOperatorPrompt => {
5757                    privileged_operator_prompt != context.privileged_operator_prompt
5758                }
5759            };
5760            if changed {
5761                restart_required.push(section.label().to_string());
5762            }
5763        }
5764        if !restart_required.is_empty() {
5765            warn!(
5766                config_path = %context.config_path.display(),
5767                sections = %restart_required.join(", "),
5768                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5769            );
5770        }
5771
5772        for configured in &modules {
5773            if let Err(err) = validate_spec(&configured.module_spec()) {
5774                return Ok(vec![control_error_frame(
5775                    &frame,
5776                    "invalid_daemon_config",
5777                    format!("supervisor rescan rejected daemon config: {err}"),
5778                )?]);
5779            }
5780        }
5781
5782        let configured_capabilities = modules
5783            .iter()
5784            .map(|module| (module.module_id.clone(), module.enabled))
5785            .collect::<Vec<_>>();
5786        let preview_capability_warnings = if preview {
5787            let (_, registrations) = self.runtime_capability_snapshot()?;
5788            let current_modules = self
5789                .supervisor
5790                .list()
5791                .into_iter()
5792                .map(|module| module.module_id().to_string())
5793                .collect::<BTreeSet<_>>();
5794            let resulting_modules = configured_capabilities.clone();
5795            let removed = current_modules
5796                .into_iter()
5797                .filter(|module_id| {
5798                    !resulting_modules
5799                        .iter()
5800                        .any(|(configured_id, _)| configured_id == module_id)
5801                })
5802                .collect::<Vec<_>>();
5803            // A candidate preview also names providers the candidate disables.
5804            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
5805            let disabled = match &mode {
5806                RescanMode::Candidate(_) => resulting_modules
5807                    .iter()
5808                    .filter(|(_, enabled)| !enabled)
5809                    .map(|(module_id, _)| module_id.clone())
5810                    .collect::<Vec<_>>(),
5811                _ => Vec::new(),
5812            };
5813            self.capability_evaluator.preview_warnings(
5814                resulting_modules,
5815                &removed,
5816                &disabled,
5817                &registrations,
5818            )
5819        } else {
5820            Vec::new()
5821        };
5822        let reconciled = match &mode {
5823            RescanMode::Candidate(candidate) => {
5824                self.preview_candidate(&context, modules, candidate)
5825            }
5826            _ => {
5827                self.reconcile_supervised_modules(
5828                    &context.supervisor,
5829                    modules,
5830                    preview,
5831                    requester.clone(),
5832                )
5833                .await
5834            }
5835        };
5836        let result = match reconciled {
5837            Ok(result) => result,
5838            Err(message) => {
5839                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5840            }
5841        };
5842        if !preview {
5843            if let Some(ledger) = &self.activity {
5844                ledger.module_rescanned(&result.added, &result.removed, requester);
5845            }
5846            if let Err(code) = self.configure_activity(activity_config).await {
5847                warn!(
5848                    code,
5849                    "activity retention could not be applied during rescan"
5850                );
5851            }
5852            self.capability_evaluator
5853                .configure(configured_capabilities, reserved_capabilities);
5854            self.capability_evaluator.wake_deadline_loop();
5855            self.refresh_capability_requirements();
5856        }
5857        let mut result = result;
5858        result.restart_required = restart_required;
5859        result.capability_warnings = preview_capability_warnings;
5860        let response = ClientControlResponse::SupervisorRescan { result };
5861        Ok(vec![control_response_body_frame(
5862            &frame,
5863            &response,
5864            "ClientControlResponse::SupervisorRescan",
5865        )?])
5866    }
5867
5868    async fn handle_supervisor_release_reserved(
5869        &self,
5870        frame: Frame,
5871        module_id: String,
5872    ) -> Result<Vec<Frame>, RouterError> {
5873        let Some(context) = self.rescan.clone() else {
5874            return Ok(vec![control_error_frame(
5875                &frame,
5876                "release_unavailable",
5877                "reserved-id release requires a daemon started with a reloadable config path",
5878            )?]);
5879        };
5880        let operation_lock = self.supervisor.operation_lock();
5881        let _operation_guard = operation_lock.lock().await;
5882        let loaded = match crate::daemon_config::load(&context.config_path) {
5883            Ok(Some(config)) => config,
5884            Ok(None) => {
5885                return Ok(vec![control_error_frame(
5886                    &frame,
5887                    "invalid_daemon_config",
5888                    format!(
5889                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5890                        context.config_path.display()
5891                    ),
5892                )?])
5893            }
5894            Err(err) => {
5895                return Ok(vec![control_error_frame(
5896                    &frame,
5897                    "invalid_daemon_config",
5898                    format!("unable to verify reserved-id release against daemon config: {err}"),
5899                )?])
5900            }
5901        };
5902        if loaded
5903            .modules
5904            .iter()
5905            .any(|configured| configured.module_id == module_id)
5906        {
5907            return Ok(vec![control_error_frame(
5908                &frame,
5909                "reserved_module_configured",
5910                format!(
5911                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5912                ),
5913            )?]);
5914        }
5915        if !self.supervisor.release_retained_reserved_gate(&module_id) {
5916            return Ok(vec![control_error_frame(
5917                &frame,
5918                "reserved_gate_not_retained",
5919                format!(
5920                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5921                ),
5922            )?]);
5923        }
5924
5925        let response = ClientControlResponse::SupervisorAck {
5926            module_id,
5927            applied: true,
5928        };
5929        Ok(vec![control_response_body_frame(
5930            &frame,
5931            &response,
5932            "ClientControlResponse::SupervisorAck",
5933        )?])
5934    }
5935
5936    /// Compare the configured module set with the supervised one, changing
5937    /// nothing. The staged spec each supervised module holds is what the
5938    /// configured entry is compared with.
5939    fn plan_rescan(
5940        &self,
5941        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5942    ) -> Result<RescanPlan, String> {
5943        let mut current = BTreeMap::new();
5944        for module in self.supervisor.list() {
5945            let (spec, health) = module.configuration().map_err(|err| {
5946                format!(
5947                    "failed to read configuration for module_id '{}': {err}",
5948                    module.module_id()
5949                )
5950            })?;
5951            let enabled = module
5952                .status()
5953                .map_err(|err| {
5954                    format!(
5955                        "failed to read status for module_id '{}': {err}",
5956                        module.module_id()
5957                    )
5958                })?
5959                .enabled;
5960            current.insert(
5961                module.module_id().to_string(),
5962                (module, spec, health, enabled),
5963            );
5964        }
5965        let configured = configured_modules
5966            .into_iter()
5967            .map(|module| (module.module_id.clone(), module))
5968            .collect::<BTreeMap<_, _>>();
5969
5970        let added = configured
5971            .keys()
5972            .filter(|module_id| !current.contains_key(*module_id))
5973            .cloned()
5974            .collect::<Vec<_>>();
5975        let removed = current
5976            .keys()
5977            .filter(|module_id| !configured.contains_key(*module_id))
5978            .cloned()
5979            .collect::<Vec<_>>();
5980        let mut changed_pending_reload = Vec::new();
5981        let mut configuration_changes = BTreeSet::new();
5982        let mut enabled_changes = BTreeSet::new();
5983        let mut unchanged = 0_u32;
5984
5985        for (module_id, configured_module) in &configured {
5986            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5987            else {
5988                continue;
5989            };
5990            // Compare the whole launch spec so a future launch field cannot
5991            // accidentally become a live-only policy change. Health is stored
5992            // separately and applies live without replacing the process.
5993            let launch_changed = *current_spec != configured_module.module_spec();
5994            let configuration_changed =
5995                launch_changed || *current_health != configured_module.health;
5996            let enabled_changed = *current_enabled != configured_module.enabled;
5997            if configuration_changed {
5998                configuration_changes.insert(module_id.clone());
5999            }
6000            if launch_changed {
6001                changed_pending_reload.push(module_id.clone());
6002            }
6003            if enabled_changed {
6004                enabled_changes.insert(module_id.clone());
6005            }
6006            if !configuration_changed && !enabled_changed {
6007                unchanged = unchanged.saturating_add(1);
6008            }
6009        }
6010        Ok(RescanPlan {
6011            current,
6012            configured,
6013            added,
6014            removed,
6015            changed_pending_reload,
6016            configuration_changes,
6017            enabled_changes,
6018            unchanged,
6019        })
6020    }
6021
6022    /// Reconcile the running module set against the configured one.
6023    ///
6024    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6025    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6026    /// deliberately shares this function with the executing path rather than
6027    /// computing the same diff somewhere else -- two implementations of one
6028    /// decision agree until they do not, and the whole value of a preview is that
6029    /// it describes the operation that will actually run.
6030    async fn reconcile_supervised_modules(
6031        &self,
6032        supervisor: &Supervisor,
6033        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6034        preview: bool,
6035        requester: Principal,
6036    ) -> Result<SupervisorRescanResult, String> {
6037        let plan = self.plan_rescan(configured_modules)?;
6038
6039        // Everything above this point is pure computation over two snapshots.
6040        // Everything below MUTATES. The preview returns here so the boundary is a
6041        // single early return rather than a condition repeated at each mutation
6042        // site, where one missed guard would apply part of a change the caller was
6043        // told would not happen.
6044        if preview {
6045            return Ok(plan.preview_result());
6046        }
6047        let RescanPlan {
6048            current,
6049            configured,
6050            added,
6051            removed,
6052            changed_pending_reload,
6053            configuration_changes,
6054            enabled_changes,
6055            unchanged,
6056        } = plan;
6057        // The staged generation moves before the first change rather than
6058        // after the last, so a rescan that fails partway still invalidates
6059        // every earlier `supervisor.rescan_checked` preview: part of its diff
6060        // may have run.
6061        if !(added.is_empty()
6062            && removed.is_empty()
6063            && configuration_changes.is_empty()
6064            && enabled_changes.is_empty())
6065        {
6066            if let Some(context) = &self.rescan {
6067                context
6068                    .generation
6069                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6070            }
6071        }
6072
6073        for module_id in &removed {
6074            let module = &current
6075                .get(module_id)
6076                .expect("removed module came from current supervisor state")
6077                .0;
6078            module
6079                .retire_requested(Some(requester.clone()))
6080                .await
6081                .map_err(|err| {
6082                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6083                })?;
6084            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6085            //
6086            // `handle_route_open` resolves an absent module in three steps:
6087            // registry, then supervisor status, then tombstone. Retiring first
6088            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6089            // went with the teardown above, the supervisor entry went with
6090            // `retire`, and the tombstone does not exist yet -- so a route.open
6091            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6092            // it") for a module that was deliberately removed and whose caller
6093            // should get `module_removed` (TERMINAL, carrying a removal age).
6094            //
6095            // Writing the tombstone first closes it: during the window the
6096            // supervisor entry still answers, so the caller gets
6097            // `target_unavailable` -- retryable, and TRUE, because the module
6098            // is mid-teardown. After both statements it is `module_removed`.
6099            // No instant remains where a removed module reads as one that
6100            // never existed.
6101            //
6102            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6103            // the absence of a test beside a fix invites deletion: these are
6104            // two sync statements with no await between them, so reaching the
6105            // window needs a second worker thread to land exactly between them
6106            // and there is no hook to force it. MEASURED: the 25 daemon_config
6107            // tests pass identically with the old order and the new one, so
6108            // the existing suite cannot see this and a green run is not
6109            // evidence either way. What the suite does hold is the
6110            // post-condition -- a removed module answers `module_removed` --
6111            // which this preserves.
6112            //
6113            // Found by an Athena panel reading the shipped tree against a
6114            // design note (2026-09-19), as the one concrete instance of that
6115            // note's class that survived contact with source. Direction is
6116            // benign: retryable where terminal was intended, never the reverse.
6117            self.supervisor.record_rescan_removal(module_id);
6118            self.supervisor.retire(module_id);
6119            self.route_outages.forget(module_id);
6120        }
6121
6122        for module_id in configured.keys() {
6123            let Some((module, _, _, _)) = current.get(module_id) else {
6124                continue;
6125            };
6126            let configured_module = configured
6127                .get(module_id)
6128                .expect("configured module id came from configured map");
6129            if configuration_changes.contains(module_id) {
6130                module
6131                    .update_configuration(
6132                        configured_module.module_spec(),
6133                        configured_module.health.clone(),
6134                        configured_module.drain_timeout_ms,
6135                    )
6136                    .await
6137                    .map_err(|err| {
6138                        format!(
6139                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6140                        )
6141                    })?;
6142            }
6143            if enabled_changes.contains(module_id) {
6144                // A rescan that starts or stops a module applies an operator's
6145                // edit to the config, so the resulting outage was asked for.
6146                self.route_outages.mark_operator_action(module_id);
6147                module
6148                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6149                    .await
6150                    .map_err(|err| {
6151                        self.route_outages.operator_action_ended_unrefused(module_id);
6152                        format!(
6153                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6154                            configured_module.enabled
6155                        )
6156                    })?;
6157            }
6158        }
6159
6160        for module_id in &added {
6161            let configured_module = configured
6162                .get(module_id)
6163                .expect("added module id came from configured map");
6164            supervisor
6165                .supervise_configured_requested(
6166                    configured_module.module_spec(),
6167                    configured_module.enabled,
6168                    configured_module.health.clone(),
6169                    configured_module.drain_timeout_ms,
6170                    configured_module.restart,
6171                    Some(requester.clone()),
6172                )
6173                .map_err(|err| {
6174                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6175                })?;
6176        }
6177
6178        Ok(SupervisorRescanResult {
6179            added,
6180            removed,
6181            changed_pending_reload,
6182            enabled_changes: enabled_changes.iter().cloned().collect(),
6183            unchanged,
6184            preview: false,
6185            // Filled by the caller, which is the only layer that can see the
6186            // previous config to diff against.
6187            restart_required: Vec::new(),
6188            capability_warnings: Vec::new(),
6189            module_actions: Vec::new(),
6190            earlier_pending: Vec::new(),
6191            expect: None,
6192        })
6193    }
6194
6195    async fn handle_supervisor_set_enabled(
6196        &self,
6197        frame: Frame,
6198        module_id: String,
6199        enabled: bool,
6200        requester: Principal,
6201    ) -> Result<Vec<Frame>, RouterError> {
6202        let operation_lock = self.supervisor.operation_lock();
6203        let _operation_guard = operation_lock.lock().await;
6204        let Some(module) = self.supervisor.get(&module_id) else {
6205            return Ok(vec![control_error_frame(
6206                &frame,
6207                "unknown_module",
6208                format!("module_id '{module_id}' is not supervised"),
6209            )?]);
6210        };
6211
6212        // Enabling counts as well as disabling: a module an operator starts
6213        // is refused until it registers, and that wait was asked for.
6214        self.route_outages.mark_operator_action(&module_id);
6215        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6216            Ok(applied) => applied,
6217            Err(err) => {
6218                self.route_outages
6219                    .operator_action_ended_unrefused(&module_id);
6220                return Ok(vec![control_error_frame(
6221                    &frame,
6222                    "target_unavailable",
6223                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6224                )?]);
6225            }
6226        };
6227        if !applied {
6228            // Already in the requested state: nothing was made unavailable,
6229            // so the mark must not outlive this request.
6230            self.route_outages
6231                .operator_action_ended_unrefused(&module_id);
6232        }
6233
6234        self.capability_evaluator.wake_deadline_loop();
6235        self.refresh_capability_requirements();
6236        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6237        Ok(vec![control_response_body_frame(
6238            &frame,
6239            &response,
6240            "ClientControlResponse::SupervisorAck",
6241        )?])
6242    }
6243
6244    async fn handle_supervisor_health_probe(
6245        &self,
6246        frame: Frame,
6247        module_id: String,
6248    ) -> Result<Vec<Frame>, RouterError> {
6249        self.refresh_capability_requirements();
6250        let Some(registration) = self
6251            .registry
6252            .get_module(&module_id)
6253            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6254        else {
6255            return Ok(vec![control_error_frame(
6256                &frame,
6257                "unknown_module",
6258                format!("module_id '{module_id}' is not registered"),
6259            )?]);
6260        };
6261
6262        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6263        // named for it. Making `module_registration_grants_op` return false
6264        // unconditionally reddens five tests, and every one is named for something
6265        // else -- capability relay, probe/bind demultiplexing, supervision-only
6266        // probing. They exercise a successful advertisement check on the way to their
6267        // own subject.
6268        //
6269        // Real protection, fragile in a specific way: narrowing any of those tests to
6270        // focus on its stated subject would silently remove coverage nobody knows
6271        // they are carrying. Recorded here rather than as a sixth test, because the
6272        // useful fact is WHICH tests hold the guard up -- a new test would add
6273        // coverage without telling the next person what the existing ones quietly do.
6274        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6275        {
6276            return Ok(vec![control_error_frame(
6277                &frame,
6278                "health_not_advertised",
6279                format!("module_id '{module_id}' did not advertise health.check"),
6280            )?]);
6281        }
6282
6283        let deadline = Instant::now() + self.health_probe_timeout;
6284        let pending = match self.forwarding.begin_module_control_rpc_for(
6285            &module_id,
6286            MODULE_CONTROL_OP_HEALTH_CHECK,
6287            deadline,
6288        ) {
6289            Ok(pending) => pending,
6290            Err(err) => {
6291                return Ok(vec![control_error_frame(
6292                    &frame,
6293                    forwarding_error_code(&err),
6294                    err.to_string(),
6295                )?])
6296            }
6297        };
6298
6299        let PendingModuleControlRpc {
6300            endpoint,
6301            module_sink,
6302            negotiated_ver,
6303            corr: probe_corr,
6304            receiver,
6305        } = pending;
6306        let mut guard =
6307            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6308        let probe_body =
6309            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6310                RouterError::backend(
6311                    0,
6312                    frame.header.corr,
6313                    format!("failed to encode health.check request: {err}"),
6314                )
6315            })?;
6316        let probe_frame = Frame::build_with_version(
6317            negotiated_ver,
6318            FrameType::Request,
6319            control_flags(),
6320            0,
6321            0,
6322            probe_corr,
6323            probe_body,
6324        )
6325        .map_err(RouterError::FrameBuild)?;
6326
6327        // Queue admission and the reply share one deadline, so a module that
6328        // stops reading cannot park the requesting connection's reader forever.
6329        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6330            Ok(Ok(())) => {}
6331            Ok(Err(err)) => {
6332                return Ok(vec![control_error_frame(
6333                    &frame,
6334                    "target_unavailable",
6335                    err.to_string(),
6336                )?]);
6337            }
6338            Err(_) => {
6339                return Ok(vec![control_error_frame(
6340                    &frame,
6341                    "module_timeout",
6342                    format!(
6343                        "module_id '{module_id}' did not answer health.check within {:?}",
6344                        self.health_probe_timeout
6345                    ),
6346                )?]);
6347            }
6348        }
6349
6350        match timeout_at(deadline, receiver).await {
6351            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6352                guard.disarm();
6353                let Some(report) = response.health_report() else {
6354                    return Ok(vec![control_error_frame(
6355                        &frame,
6356                        "invalid_control_body",
6357                        "health.check RPC returned a non-health response",
6358                    )?]);
6359                };
6360                // Metrics go out whole here. The supervisor's cached snapshot
6361                // caps this blob (see truncate_health_metrics), and this path
6362                // exists precisely to answer without that cap -- so applying it
6363                // here would leave no way to see what the cached view drops.
6364                let HealthReport {
6365                    status,
6366                    detail,
6367                    metrics,
6368                } = report;
6369                let capability_detail = self
6370                    .capability_evaluator
6371                    .required_problem_detail(&module_id);
6372                let response = ClientControlResponse::SupervisorHealthProbe {
6373                    module_id,
6374                    status,
6375                    detail: append_capability_problem_detail(detail, capability_detail),
6376                    metrics,
6377                };
6378                Ok(vec![control_response_body_frame(
6379                    &frame,
6380                    &response,
6381                    "ClientControlResponse::SupervisorHealthProbe",
6382                )?])
6383            }
6384            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6385                guard.disarm();
6386                Ok(vec![control_error_body_frame(&frame, body)?])
6387            }
6388            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6389                guard.disarm();
6390                Ok(vec![control_error_frame(
6391                    &frame,
6392                    "target_unavailable",
6393                    message,
6394                )?])
6395            }
6396            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6397                guard.disarm();
6398                Ok(vec![control_error_frame(
6399                    &frame,
6400                    "invalid_control_body",
6401                    message,
6402                )?])
6403            }
6404            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6405                guard.disarm();
6406                Ok(vec![control_error_frame(
6407                    &frame,
6408                    "invalid_control_body",
6409                    format!("expected module-control op '{expected}', got '{actual}'"),
6410                )?])
6411            }
6412            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6413                guard.disarm();
6414                Ok(vec![control_error_frame(
6415                    &frame,
6416                    "module_timeout",
6417                    format!(
6418                        "module_id '{module_id}' answered health.check after {:?}",
6419                        self.health_probe_timeout
6420                    ),
6421                )?])
6422            }
6423            Ok(Err(_)) => Ok(vec![control_error_frame(
6424                &frame,
6425                "target_unavailable",
6426                "health.check waiter was canceled before the module responded",
6427            )?]),
6428            Err(_) => Ok(vec![control_error_frame(
6429                &frame,
6430                "module_timeout",
6431                format!(
6432                    "module_id '{module_id}' did not answer health.check within {:?}",
6433                    self.health_probe_timeout
6434                ),
6435            )?]),
6436        }
6437    }
6438
6439    fn supervisor_status(
6440        &self,
6441        module_id: &str,
6442        corr: u64,
6443    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6444        self.supervisor
6445            .get(module_id)
6446            .map(|module| {
6447                let warming = module.is_warming_for_control("status").map_err(|err| {
6448                    RouterError::backend(
6449                        0,
6450                        corr,
6451                        format!(
6452                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6453                        ),
6454                    )
6455                })?;
6456                module.status_for_control("status").map_err(|err| {
6457                    RouterError::backend(
6458                        0,
6459                        corr,
6460                        format!(
6461                            "failed to read supervisor status for module_id '{module_id}': {err}"
6462                        ),
6463                    )
6464                }).map(|status| (status, warming))
6465            })
6466            .transpose()
6467    }
6468
6469    fn guard_module_control_op(
6470        &self,
6471        frame: &Frame,
6472        module_id: &str,
6473        op: &str,
6474    ) -> Result<Option<Frame>, RouterError> {
6475        if self.module_grants_op(module_id, op, frame.header.corr)? {
6476            return Ok(None);
6477        }
6478
6479        Ok(Some(control_error_frame(
6480            frame,
6481            "op_not_allowed",
6482            format!("module_id '{module_id}' did not grant control op '{op}'"),
6483        )?))
6484    }
6485
6486    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6487        let Some(registration) = self
6488            .registry
6489            .get_module(module_id)
6490            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6491        else {
6492            return Ok(false);
6493        };
6494        Ok(module_registration_grants_op(&registration.control_ops, op))
6495    }
6496
6497    fn handle_status_update(
6498        &self,
6499        endpoint: ModuleEndpointId,
6500        frame: Frame,
6501    ) -> Result<Vec<Frame>, RouterError> {
6502        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6503            Ok(update) => update,
6504            Err(err) => {
6505                // Forward-compat: a newer module may push a channel-0 op this subc
6506                // version doesn't know. The control contract says unknown push ops
6507                // are IGNORED, never answered with an error. Only a malformed body
6508                // for an op we DO know is a real error worth surfacing.
6509                if is_known_module_push_op(&frame.body) {
6510                    return Ok(vec![control_error_frame(
6511                        &frame,
6512                        "invalid_control_body",
6513                        format!("malformed module control push body: {err}"),
6514                    )?]);
6515                }
6516                return Ok(Vec::new());
6517            }
6518        };
6519
6520        match update {
6521            ModuleControlPush::RouteStatus {
6522                route_channel,
6523                route_epoch,
6524                status,
6525            } => {
6526                self.forwarding
6527                    .cache_status(endpoint, route_channel, route_epoch, status)
6528                    .map_err(RouterError::Forwarding)?;
6529            }
6530        }
6531        Ok(Vec::new())
6532    }
6533
6534    fn handle_route_poll(
6535        &self,
6536        ctx: &RouteCtx,
6537        frame: Frame,
6538        route_channel: u16,
6539        route_epoch: u32,
6540        kind: PollKind,
6541    ) -> Result<Vec<Frame>, RouterError> {
6542        let snapshot = self
6543            .forwarding
6544            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6545            .map_err(RouterError::Forwarding)?;
6546        let response = match (kind, snapshot) {
6547            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6548                ClientControlResponse::RoutePoll {
6549                    route_channel,
6550                    route_epoch,
6551                    status,
6552                    live: None,
6553                }
6554            }
6555            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6556                route_channel,
6557                route_epoch,
6558                status: None,
6559                live: None,
6560            },
6561            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6562                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6563                // is what makes reporting `true` correct rather than a
6564                // confident guess. `process_live` returns None only when the
6565                // module id has no supervisor snapshot at all -- an
6566                // externally-started module the daemon did not spawn -- and
6567                // for those the supervisor has no opinion to offer, ever. It
6568                // is never None for a supervised module in an unknown state:
6569                // a supervised module always has a snapshot, and the answer
6570                // comes from `state == Running && process_alive`.
6571                //
6572                // The route is Bound, so the module completed a HELLO on a
6573                // live connection; "the process this route points at is
6574                // running" is therefore attested by the binding rather than
6575                // assumed. Reporting `false` for an unsupervised module would
6576                // be the actual lie -- it would tell a client its healthy
6577                // route is dead because the daemon does not manage the
6578                // process.
6579                //
6580                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6581                // module whose liveness is genuinely unknown, e.g. a snapshot
6582                // that has not been populated yet -- THIS DEFAULT BECOMES
6583                // WRONG and must split: unsupervised stays true, unknown
6584                // becomes null so the client can tell the two apart. The
6585                // response field is already `Option<bool>`, so the wire can
6586                // carry that distinction today.
6587                let live = self
6588                    .process_liveness
6589                    .as_ref()
6590                    .and_then(|source| source.process_live(&module_id))
6591                    .unwrap_or(true);
6592                ClientControlResponse::RoutePoll {
6593                    route_channel,
6594                    route_epoch,
6595                    status: None,
6596                    live: Some(live),
6597                }
6598            }
6599            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6600                route_channel,
6601                route_epoch,
6602                status: None,
6603                live: Some(false),
6604            },
6605        };
6606
6607        Ok(vec![control_response_body_frame(
6608            &frame,
6609            &response,
6610            "ClientControlResponse::RoutePoll",
6611        )?])
6612    }
6613
6614    pub(crate) fn observe_module_control_completion(
6615        &self,
6616        completion: ModuleControlRpcCompletion,
6617    ) -> bool {
6618        match completion {
6619            ModuleControlRpcCompletion::Unknown => false,
6620            ModuleControlRpcCompletion::Settled => true,
6621            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6622                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6623                info!(
6624                    module_id = %module_id,
6625                    latency_ms,
6626                    "late health.check answer proves the module is alive"
6627                );
6628                match self
6629                    .supervisor
6630                    .record_late_health_answer(&module_id, latency_ms)
6631                {
6632                    Ok(true) => {}
6633                    Ok(false) => debug!(
6634                        module_id = %module_id,
6635                        latency_ms,
6636                        "late health.check answer has no active supervisor snapshot"
6637                    ),
6638                    Err(err) => warn!(
6639                        module_id = %module_id,
6640                        latency_ms,
6641                        error = %err,
6642                        "failed to record late health.check answer"
6643                    ),
6644                }
6645                true
6646            }
6647        }
6648    }
6649
6650    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6651    /// the module connection whose frame is being handled, or to the client that
6652    /// relay was opened for.
6653    ///
6654    /// This runs on the MODULE connection's frame handler, where returning `Err`
6655    /// ends that connection -- and a module connection carries every client's
6656    /// routes to that module, so ending it costs the whole fleet its tools.
6657    /// `ConnectionClosing` carries the id of the connection that is closing, and
6658    /// when that id is a CLIENT's, the condition is entirely about that one
6659    /// client's route.open. A client-scoped condition has no authority over a
6660    /// shared module connection, so it is logged and the single relay is dropped:
6661    /// the client is going away, and `complete_pending_relay` already removed the
6662    /// relay before failing, so there is nothing left to settle. Anything that
6663    /// relay still reserved is released by that client's own connection teardown,
6664    /// which is already under way -- that is what "closing" means.
6665    ///
6666    /// Every other failure is a statement about THIS connection and stays fatal:
6667    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6668    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6669    /// frames correctly.
6670    fn refuse_to_end_module_connection_for_a_client(
6671        &self,
6672        module_connection_id: ConnectionId,
6673        corr: u64,
6674        err: ForwardingError,
6675    ) -> Result<(), RouterError> {
6676        if let ForwardingError::ConnectionClosing { connection_id } = err {
6677            if connection_id != module_connection_id {
6678                warn!(
6679                    module_connection_id = module_connection_id.get(),
6680                    client_connection_id = connection_id.get(),
6681                    corr,
6682                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6683                );
6684                return Ok(());
6685            }
6686        }
6687        Err(RouterError::Forwarding(err))
6688    }
6689
6690    fn handle_module_relay_response(
6691        &self,
6692        connection_id: ConnectionId,
6693        frame: Frame,
6694    ) -> Result<Vec<Frame>, RouterError> {
6695        let mut secondary_error = None;
6696        let outcome = match frame.header.ty {
6697            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6698                Ok(probe) if probe.op == "route.bind" => {
6699                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6700                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6701                            RouteBindRelayOutcome::Accepted
6702                        }
6703                        Ok(other) => {
6704                            let message =
6705                                format!("route.bind response carried unexpected body: {other:?}");
6706                            secondary_error = Some(control_error_frame(
6707                                &frame,
6708                                "invalid_control_body",
6709                                message.clone(),
6710                            )?);
6711                            RouteBindRelayOutcome::ModuleGone(message)
6712                        }
6713                        Err(err) => {
6714                            let message = format!("malformed route.bind response body: {err}");
6715                            secondary_error = Some(control_error_frame(
6716                                &frame,
6717                                "invalid_control_body",
6718                                message.clone(),
6719                            )?);
6720                            RouteBindRelayOutcome::ModuleGone(message)
6721                        }
6722                    }
6723                }
6724                Ok(probe) => {
6725                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6726                    {
6727                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6728                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6729                            "malformed {} response body: {err}",
6730                            probe.op
6731                        )),
6732                    };
6733                    let completion = self
6734                        .forwarding
6735                        .complete_module_control_rpc(
6736                            connection_id,
6737                            frame.header.corr,
6738                            Some(&probe.op),
6739                            outcome,
6740                        )
6741                        .map_err(RouterError::Forwarding)?;
6742                    if !self.observe_module_control_completion(completion) {
6743                        debug!(
6744                            connection_id = connection_id.get(),
6745                            corr = frame.header.corr,
6746                            op = %probe.op,
6747                            "dropping late or unknown module-control RPC response"
6748                        );
6749                    }
6750                    return Ok(Vec::new());
6751                }
6752                Err(err) => {
6753                    if let Some(expected_op) = self
6754                        .forwarding
6755                        .pending_module_control_op(connection_id, frame.header.corr)
6756                        .map_err(RouterError::Forwarding)?
6757                    {
6758                        let completion = self
6759                            .forwarding
6760                            .complete_module_control_rpc(
6761                                connection_id,
6762                                frame.header.corr,
6763                                None,
6764                                ModuleControlRpcOutcome::MalformedResponse(format!(
6765                                    "malformed {expected_op} response body: {err}"
6766                                )),
6767                            )
6768                            .map_err(RouterError::Forwarding)?;
6769                        if !self.observe_module_control_completion(completion) {
6770                            debug!(
6771                                connection_id = connection_id.get(),
6772                                corr = frame.header.corr,
6773                                "dropping late malformed module-control RPC response"
6774                            );
6775                        }
6776                        return Ok(Vec::new());
6777                    }
6778                    let message = format!("malformed route.bind response body: {err}");
6779                    secondary_error = Some(control_error_frame(
6780                        &frame,
6781                        "invalid_control_body",
6782                        message.clone(),
6783                    )?);
6784                    RouteBindRelayOutcome::ModuleGone(message)
6785                }
6786            },
6787            FrameType::Error => {
6788                if self
6789                    .forwarding
6790                    .pending_module_control_op(connection_id, frame.header.corr)
6791                    .map_err(RouterError::Forwarding)?
6792                    .is_some()
6793                {
6794                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6795                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6796                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6797                            "malformed module-control ERROR body: {err}"
6798                        )),
6799                    };
6800                    let completion = self
6801                        .forwarding
6802                        .complete_module_control_rpc(
6803                            connection_id,
6804                            frame.header.corr,
6805                            None,
6806                            outcome,
6807                        )
6808                        .map_err(RouterError::Forwarding)?;
6809                    if !self.observe_module_control_completion(completion) {
6810                        debug!(
6811                            connection_id = connection_id.get(),
6812                            corr = frame.header.corr,
6813                            "dropping late or unknown module-control RPC error"
6814                        );
6815                    }
6816                    return Ok(Vec::new());
6817                }
6818                match serde_json::from_slice::<ErrorBody>(&frame.body) {
6819                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
6820                    Err(err) => {
6821                        let message = format!("malformed route.bind ERROR body: {err}");
6822                        secondary_error = Some(control_error_frame(
6823                            &frame,
6824                            "invalid_control_body",
6825                            message.clone(),
6826                        )?);
6827                        RouteBindRelayOutcome::ModuleGone(message)
6828                    }
6829                }
6830            }
6831            ty => {
6832                return Ok(vec![control_error_frame(
6833                    &frame,
6834                    "unsupported_control_frame",
6835                    format!("unsupported module channel-0 frame {ty:?}"),
6836                )?])
6837            }
6838        };
6839
6840        let settled =
6841            self.forwarding
6842                .complete_pending_relay(connection_id, frame.header.corr, outcome);
6843        let completion = match settled {
6844            Ok(completion) => completion,
6845            Err(err) => {
6846                self.refuse_to_end_module_connection_for_a_client(
6847                    connection_id,
6848                    frame.header.corr,
6849                    err,
6850                )?;
6851                return Ok(secondary_error.into_iter().collect());
6852            }
6853        };
6854        if let Some(target) = completion.abandoned.as_ref() {
6855            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6856        }
6857        if !completion.settled {
6858            debug!(
6859                connection_id = connection_id.get(),
6860                corr = frame.header.corr,
6861                frame_type = ?frame.header.ty,
6862                "dropping late or unknown route.bind relay response"
6863            );
6864        }
6865        Ok(secondary_error.into_iter().collect())
6866    }
6867
6868    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6869        debug!(connection_id = connection_id.get(), "handling GOODBYE");
6870        // GOODBYE ends the connection's logical session even when its socket
6871        // stays open. Use disconnect teardown so verdicts, client notices and
6872        // scope authority are released at the same lifecycle boundary.
6873        self.cleanup_connection_with_end_reason(
6874            connection_id,
6875            RegistrationEndReason::ExplicitGoodbye,
6876        )
6877        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6878        Ok(Vec::new())
6879    }
6880}
6881
6882impl Default for ControlHandler {
6883    fn default() -> Self {
6884        Self::new(Arc::new(Registry::default()))
6885    }
6886}
6887
6888impl crate::supervise::SwapPromotionObserver for ControlHandler {
6889    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6890        self.apply_registration_capabilities(registration);
6891    }
6892}
6893
6894fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6895    CapabilityRequirementStatus {
6896        consumer: status.consumer,
6897        capability: status.capability,
6898        need: match status.need {
6899            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6900            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6901        },
6902        verdict: status.verdict.as_str().to_string(),
6903        episode_seq: status.episode_seq,
6904        config_satisfiable: status.config_satisfiable,
6905        runtime_available: status.runtime_available,
6906        detail: status.detail,
6907    }
6908}
6909
6910fn append_capability_problem_detail(
6911    detail: Option<String>,
6912    capability_detail: Option<String>,
6913) -> Option<String> {
6914    match (detail, capability_detail) {
6915        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6916        (Some(detail), None) => Some(detail),
6917        (None, Some(capability_detail)) => Some(capability_detail),
6918        (None, None) => None,
6919    }
6920}
6921
6922fn subc_ops() -> Vec<String> {
6923    SUBC_CONTROL_OPS
6924        .iter()
6925        .map(|op| (*op).to_string())
6926        .collect()
6927}
6928
6929fn module_subc_ops() -> Vec<String> {
6930    SUBC_CONTROL_OPS
6931        .iter()
6932        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6933        .map(|op| (*op).to_string())
6934        .collect()
6935}
6936
6937#[cfg(test)]
6938fn module_baseline_control_ops() -> Vec<String> {
6939    MODULE_BASELINE_CONTROL_OPS
6940        .iter()
6941        .map(|op| (*op).to_string())
6942        .collect()
6943}
6944
6945fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6946    let mut seen = HashSet::new();
6947    let mut effective = Vec::new();
6948    for op in MODULE_BASELINE_CONTROL_OPS {
6949        if seen.insert((*op).to_string()) {
6950            effective.push((*op).to_string());
6951        }
6952    }
6953    for op in declared.unwrap_or_default() {
6954        if seen.insert(op.clone()) {
6955            effective.push(op);
6956        }
6957    }
6958    effective
6959}
6960
6961fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6962    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6963}
6964
6965fn target_module_id(target: &RouteTarget) -> &str {
6966    match target {
6967        RouteTarget::ToolProvider { module_id }
6968        | RouteTarget::ManagementSurface { module_id }
6969        | RouteTarget::InternalService { module_id, .. } => module_id,
6970    }
6971}
6972
6973fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6974    roles.iter().any(|role| match (target, role) {
6975        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6976        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6977        (
6978            RouteTarget::InternalService { service_id, .. },
6979            ProviderRole::InternalService {
6980                service_id: provided,
6981                ..
6982            },
6983        ) => service_id == provided,
6984        _ => false,
6985    })
6986}
6987
6988fn is_routable_role(role: &ProviderRole) -> bool {
6989    matches!(
6990        role,
6991        ProviderRole::ToolProvider { .. }
6992            | ProviderRole::ManagementSurface { .. }
6993            | ProviderRole::InternalService { .. }
6994    )
6995}
6996
6997#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6998enum ControlRequestBodyError {
6999    UnknownOp,
7000    InvalidBody,
7001}
7002
7003#[derive(Debug, Deserialize)]
7004struct ControlOpProbe {
7005    op: String,
7006}
7007
7008/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7009/// this set is treated as a forward-compat unknown and ignored rather than errored.
7010const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7011
7012fn is_known_module_push_op(body: &[u8]) -> bool {
7013    serde_json::from_slice::<ControlOpProbe>(body)
7014        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7015        .unwrap_or(false)
7016}
7017
7018fn is_known_module_request_op(body: &[u8]) -> bool {
7019    serde_json::from_slice::<ControlOpProbe>(body)
7020        .map(|probe| is_module_to_subc_op(&probe.op))
7021        .unwrap_or(false)
7022}
7023
7024fn is_module_to_subc_op(op: &str) -> bool {
7025    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7026}
7027
7028fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7029    debug!(
7030        op = %op,
7031        connection_id = connection_id.get(),
7032        corr,
7033        "control dispatch"
7034    );
7035}
7036
7037fn log_slow_control_dispatch(
7038    dispatch_started_at: Option<StdInstant>,
7039    op: &'static str,
7040    connection_id: ConnectionId,
7041    corr: u64,
7042    route_open_timings: Option<&RouteOpenTimings>,
7043) {
7044    let Some(dispatch_started_at) = dispatch_started_at else {
7045        return;
7046    };
7047    // Use the same monotonic clock as the phase timers and relay deadline.
7048    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7049    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7050        if let Some(timings) = route_open_timings {
7051            warn!(
7052                op = %op,
7053                connection_id = connection_id.get(),
7054                corr,
7055                elapsed_ms = elapsed.as_millis() as u64,
7056                module_id = ?timings.module_id,
7057                principal = %timings.principal,
7058                admission_ms = timings.admission.as_millis() as u64,
7059                project_root_ms = timings.project_root.as_millis() as u64,
7060                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7061                bind_send_ms = timings.bind_send.as_millis() as u64,
7062                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7063                "slow control dispatch"
7064            );
7065            return;
7066        }
7067        warn!(
7068            op = %op,
7069            connection_id = connection_id.get(),
7070            corr,
7071            elapsed_ms = elapsed.as_millis() as u64,
7072            "slow control dispatch"
7073        );
7074    }
7075}
7076
7077/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7078#[cfg(test)]
7079type TestCallerLookup =
7080    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7081
7082/// Every new client operation needs an explicit privilege decision here.
7083fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7084    match request {
7085        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7086            Some(PrivilegedOperation::Restart { module_id })
7087        }
7088        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7089            Some(PrivilegedOperation::Swap { module_id })
7090        }
7091        ClientControlRequest::SupervisorReload { module_id } => {
7092            Some(PrivilegedOperation::Reload { module_id })
7093        }
7094        ClientControlRequest::SupervisorRescan { preview } => {
7095            if *preview {
7096                None
7097            } else {
7098                Some(PrivilegedOperation::Rescan)
7099            }
7100        }
7101        // Only the well-formed apply asks for approval. A preview changes
7102        // nothing, and any other combination is refused before it touches
7103        // anything, so prompting the operator for it would ask about a request
7104        // that cannot run.
7105        ClientControlRequest::SupervisorRescanChecked {
7106            preview,
7107            candidate,
7108            expect,
7109        } => (!*preview && candidate.is_none() && expect.is_some())
7110            .then_some(PrivilegedOperation::Rescan),
7111        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7112            Some(PrivilegedOperation::SetEnabled {
7113                module_id,
7114                enabled: *enabled,
7115            })
7116        }
7117        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7118            Some(PrivilegedOperation::ReleaseReserved { module_id })
7119        }
7120        ClientControlRequest::ServerDescribe {}
7121        | ClientControlRequest::CatalogList { .. }
7122        | ClientControlRequest::RouteOpen { .. }
7123        | ClientControlRequest::RoutePoll { .. }
7124        | ClientControlRequest::SupervisorList {}
7125        | ClientControlRequest::SupervisorHealth {}
7126        | ClientControlRequest::SupervisorHealthProbe { .. }
7127        | ClientControlRequest::SupervisorRoutes { .. }
7128        | ClientControlRequest::SupervisorProvenance { .. }
7129        | ClientControlRequest::SupervisorStderrTail { .. }
7130        | ClientControlRequest::SupervisorTerminals { .. }
7131        | ClientControlRequest::SupervisorSpawnSnapshot {}
7132        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7133    }
7134}
7135
7136fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7137    match request {
7138        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7139        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7140        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7141        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7142        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7143        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7144        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7145        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7146        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7147        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7148        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7149        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7150        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7151        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7152        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7153        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7154        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7155        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7156        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7157        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7158    }
7159}
7160
7161/// The `scope=` field of the `route.open accepted` line, from the stamp that
7162/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7163/// so a ref holding control characters cannot forge a log line, and cut to 64
7164/// characters with a trailing `…` to bound the line. The bind proof and the
7165/// scope's attributes never appear.
7166fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7167    use subc_protocol::scope::ScopeKind;
7168    const MAX_REF_CHARS: usize = 64;
7169    let kind = match stamp.kind {
7170        ScopeKind::Head => "head",
7171        ScopeKind::Worker => "worker",
7172        ScopeKind::Ephemeral => "ephemeral",
7173    };
7174    let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7175    if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7176        scope_ref.push('…');
7177    }
7178    format!(
7179        "{kind}/{}/{scope_ref:?}@{}",
7180        principal_label(&stamp.owner),
7181        stamp.scope_epoch
7182    )
7183}
7184
7185fn principal_label(principal: &Principal) -> String {
7186    match principal {
7187        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7188        Principal::Direct => "direct".to_string(),
7189        other => format!("{other:?}"),
7190    }
7191}
7192
7193fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7194    match request {
7195        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7196        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7197        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7198        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7199        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7200    }
7201}
7202
7203fn parse_client_control_request(
7204    body: &[u8],
7205) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7206    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7207        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7208            Ok(probe)
7209                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7210                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7211            {
7212                ControlRequestBodyError::InvalidBody
7213            }
7214            Ok(_) => ControlRequestBodyError::UnknownOp,
7215            Err(_) => ControlRequestBodyError::InvalidBody,
7216        };
7217        (err, classification)
7218    })
7219}
7220
7221fn parse_module_control_request_from_module(
7222    body: &[u8],
7223) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7224    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7225        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7226            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7227            Ok(_) => ControlRequestBodyError::UnknownOp,
7228            Err(_) => ControlRequestBodyError::InvalidBody,
7229        };
7230        (err, classification)
7231    })
7232}
7233
7234#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7235enum ProviderRoleKind {
7236    ToolProvider,
7237    PipelineStage,
7238    ManagementSurface,
7239    InternalService,
7240}
7241
7242fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7243    match role {
7244        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7245        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7246        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7247        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7248    }
7249}
7250
7251fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7252    roles.iter().map(provider_role_kind).collect()
7253}
7254
7255/// Most refused scope records named individually in the log per sync; the
7256/// `refused` count on the accepted line is always complete.
7257const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7258
7259/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7260#[derive(Debug, Default, PartialEq, Eq)]
7261struct ScopeOutcomeCounts {
7262    created: usize,
7263    replaced: usize,
7264    updated: usize,
7265    unchanged: usize,
7266    refused: usize,
7267}
7268
7269impl ScopeOutcomeCounts {
7270    fn of(results: &[ScopeRecordResult]) -> Self {
7271        let mut counts = Self::default();
7272        for result in results {
7273            let slot = match result.outcome {
7274                ScopeRecordOutcome::Created => &mut counts.created,
7275                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7276                ScopeRecordOutcome::Updated => &mut counts.updated,
7277                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7278                ScopeRecordOutcome::Refused => &mut counts.refused,
7279            };
7280            *slot += 1;
7281        }
7282        counts
7283    }
7284}
7285
7286#[cfg(test)]
7287mod scope_outcome_count_tests {
7288    use super::*;
7289
7290    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7291        ScopeRecordResult {
7292            scope_ref: "r".to_string(),
7293            scope_epoch: 1,
7294            outcome,
7295            code: None,
7296            message: None,
7297            version: None,
7298            parent_state: None,
7299        }
7300    }
7301
7302    /// Each outcome lands in its own count, so a refused record can never be
7303    /// hidden inside the total the log already printed.
7304    #[test]
7305    fn every_outcome_is_counted_in_its_own_field() {
7306        let results = [
7307            result(ScopeRecordOutcome::Created),
7308            result(ScopeRecordOutcome::Created),
7309            result(ScopeRecordOutcome::Replaced),
7310            result(ScopeRecordOutcome::Updated),
7311            result(ScopeRecordOutcome::Unchanged),
7312            result(ScopeRecordOutcome::Refused),
7313            result(ScopeRecordOutcome::Refused),
7314            result(ScopeRecordOutcome::Refused),
7315        ];
7316        assert_eq!(
7317            ScopeOutcomeCounts::of(&results),
7318            ScopeOutcomeCounts {
7319                created: 2,
7320                replaced: 1,
7321                updated: 1,
7322                unchanged: 1,
7323                refused: 3,
7324            }
7325        );
7326    }
7327}
7328
7329/// Return whether a catalog change can create a newly violating live route.
7330/// Removing an attested claim is intentionally excluded: it makes fewer routes
7331/// forbidden and therefore must leave the existing route census untouched.
7332fn capability_census_trigger(
7333    old: Option<&CapabilityDeclarations>,
7334    new: Option<&CapabilityDeclarations>,
7335) -> bool {
7336    let old_provides = old
7337        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7338        .unwrap_or_default();
7339    let old_denies = old
7340        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7341        .unwrap_or_default();
7342    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7343        provides: Vec::new(),
7344        requires: Vec::new(),
7345        must_never_reach: Vec::new(),
7346    });
7347
7348    new.provides
7349        .iter()
7350        .any(|capability| !old_provides.contains(capability))
7351        || new
7352            .must_never_reach
7353            .iter()
7354            .any(|capability| !old_denies.contains(capability))
7355}
7356
7357/// Find the first capability an attested opener denies that an attested target
7358/// claims. Both manifests are live registry records, never cached or client data.
7359fn denied_capability<'a>(
7360    opening_manifest: &'a ModuleManifest,
7361    target_manifest: &ModuleManifest,
7362) -> Option<&'a str> {
7363    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7364    let target_capabilities = target_manifest.capabilities.as_ref()?;
7365    opening_capabilities
7366        .must_never_reach
7367        .iter()
7368        .find(|denied| {
7369            target_capabilities
7370                .provides
7371                .iter()
7372                .any(|provided| provided == *denied)
7373        })
7374        .map(String::as_str)
7375}
7376
7377fn catalog_update_frozen_field_message(
7378    registered: &ModuleManifest,
7379    provides: &[ProviderRole],
7380) -> Option<String> {
7381    let old_has_provides = !registered.provides.is_empty();
7382    let new_has_provides = !provides.is_empty();
7383    if old_has_provides != new_has_provides {
7384        return Some(format!(
7385            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7386            registered.module_id
7387        ));
7388    }
7389
7390    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7391        return Some(format!(
7392            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7393            registered.module_id
7394        ));
7395    }
7396
7397    let registered_concurrency = manifest_concurrency(registered);
7398    let mut candidate = registered.clone();
7399    candidate.provides = provides.to_vec();
7400    let candidate_concurrency = manifest_concurrency(&candidate);
7401    if candidate_concurrency != registered_concurrency {
7402        return Some(format!(
7403            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7404            registered.module_id, registered_concurrency, candidate_concurrency
7405        ));
7406    }
7407
7408    // control_ops live beside the manifest in the HELLO body, not inside
7409    // ModuleManifest, so a provides-only catalog.update cannot change them.
7410    None
7411}
7412
7413fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7414    manifest.provides.iter().any(is_routable_role)
7415}
7416
7417/// Returns the routable-provider concurrency subc should enforce for this manifest.
7418///
7419/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7420/// InternalService has no role-specific concurrency field, so it retains the
7421/// existing ModuleManaged default for backward compatibility.
7422fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7423    manifest
7424        .provides
7425        .iter()
7426        .find_map(|provider| match provider {
7427            ProviderRole::ToolProvider { concurrency, .. }
7428            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7429            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7430        })
7431        .unwrap_or(Concurrency::ModuleManaged)
7432}
7433
7434/// True when the manifest carries a ManagementSurface role whose concurrency
7435/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7436/// bytes because the typed manifest deliberately erases that distinction: the
7437/// default exists for wire compatibility, and this probe exists so the default
7438/// stays observable. Any parse irregularity returns false -- the caller only
7439/// logs, and a malformed body already failed registration upstream.
7440fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7441    let has_management_surface = manifest
7442        .provides
7443        .iter()
7444        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7445    if !has_management_surface {
7446        return false;
7447    }
7448    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7449        return false;
7450    };
7451    let Some(provides) = raw
7452        .get("manifest")
7453        .and_then(|manifest| manifest.get("provides"))
7454        .and_then(serde_json::Value::as_array)
7455    else {
7456        return false;
7457    };
7458    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7459    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7460    // against the management_surface_manifest_without_concurrency golden, not
7461    // recalled (the externally-tagged guess was this function's first bug).
7462    provides.iter().any(|role| {
7463        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7464            && role.get("concurrency").is_none()
7465    })
7466}
7467
7468fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7469    if peer_version != PROTOCOL_VERSION {
7470        return Err(format!(
7471            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7472        ));
7473    }
7474    Ok(PROTOCOL_VERSION)
7475}
7476
7477fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7478    Frame::build_with_version(
7479        response_version(frame),
7480        FrameType::Pong,
7481        frame.header.flags,
7482        0,
7483        0,
7484        frame.header.corr,
7485        Vec::new(),
7486    )
7487    .map_err(RouterError::FrameBuild)
7488}
7489
7490pub(crate) fn control_error_frame(
7491    frame: &Frame,
7492    code: &'static str,
7493    message: impl Into<String>,
7494) -> Result<Frame, RouterError> {
7495    control_error_body_frame(
7496        frame,
7497        ErrorBody {
7498            code: code.to_string(),
7499            message: message.into(),
7500            detail: None,
7501        },
7502    )
7503}
7504
7505fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7506    let body = serde_json::to_vec(&error).map_err(|err| {
7507        RouterError::backend(
7508            0,
7509            frame.header.corr,
7510            format!("failed to encode control ERROR: {err}"),
7511        )
7512    })?;
7513
7514    Frame::build_with_version(
7515        response_version(frame),
7516        FrameType::Error,
7517        control_flags(),
7518        0,
7519        0,
7520        frame.header.corr,
7521        body,
7522    )
7523    .map_err(RouterError::FrameBuild)
7524}
7525
7526fn control_response_body_frame<T: Serialize>(
7527    frame: &Frame,
7528    reply: &T,
7529    label: &'static str,
7530) -> Result<Frame, RouterError> {
7531    let body = serde_json::to_vec(reply).map_err(|err| {
7532        RouterError::backend(
7533            0,
7534            frame.header.corr,
7535            format!("failed to encode {label}: {err}"),
7536        )
7537    })?;
7538
7539    Frame::build_with_version(
7540        response_version(frame),
7541        FrameType::Response,
7542        control_flags(),
7543        0,
7544        0,
7545        frame.header.corr,
7546        body,
7547    )
7548    .map_err(RouterError::FrameBuild)
7549}
7550
7551/// Map a forwarding failure to the wire code a client sees.
7552///
7553/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7554/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7555/// chosen here decides whether a caller retries or gives up.
7556///
7557/// That makes attribution the load-bearing property, not merely having a code. A
7558/// permanent fault published as a retryable one produces a fleet-wide retry storm
7559/// against something that can never recover; a transient fault published as
7560/// permanent gives up on work that would have succeeded. Both look correct in a
7561/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7562/// the mapping per variant rather than merely asserting that some code exists.
7563///
7564/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7565/// two codes on the same side of the boundary passes it. Measured rather than
7566/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7567/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7568/// a test named for something else that happens to assert the string.
7569///
7570/// That accidental coverage is deliberately left alone rather than promoted to a
7571/// named test, because it guards a property this function does not promise.
7572/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7573/// specific code within a class, so identity is free to change and only the
7574/// partition is a contract. Splitting it out would assert a guarantee nothing
7575/// depends on — and a suite that promises more than the code does is the harder
7576/// thing to correct later, because the next reader cannot tell which assertions
7577/// are load-bearing.
7578///
7579/// Pin identity here the moment a consumer branches on a specific code.
7580fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7581    match err {
7582        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7583        ForwardingError::NoModuleConnection => "target_unavailable",
7584        ForwardingError::ModuleReloading { .. } => "module_reloading",
7585        ForwardingError::ClientRouteChannelExhausted { .. }
7586        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7587        ForwardingError::StaleModuleEndpoint
7588        | ForwardingError::UnknownReservation { .. }
7589        | ForwardingError::ConnectionClosing { .. }
7590        | ForwardingError::ClientEgressClosed { .. }
7591        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7592        // Only a swap candidate's registration can produce this, and it means
7593        // exactly what a second active HELLO for a live id means.
7594        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7595        ForwardingError::RelayCorrelationExhausted
7596        | ForwardingError::RouteOpenBuild(_)
7597        | ForwardingError::Poisoned => "forwarding_error",
7598    }
7599}
7600
7601fn response_version(frame: &Frame) -> u8 {
7602    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7603        frame.header.ver
7604    } else {
7605        PROTOCOL_VERSION
7606    }
7607}
7608
7609fn control_flags() -> Flags {
7610    Flags::new(false, Priority::Passive, false)
7611}
7612
7613/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7614/// channel. The target is the module (a client never saw the route), so this
7615/// takes the module path: delivered late rather than dropped when the module's
7616/// queue is momentarily full, and never closing its connection.
7617fn send_goodbye_target_best_effort(
7618    counters: &DaemonCounters,
7619    target: &GoodbyeTarget,
7620    context: &'static str,
7621) {
7622    let Ok(frame) = Frame::build_with_version(
7623        target.negotiated_ver,
7624        FrameType::Goodbye,
7625        control_flags(),
7626        target.channel,
7627        target.epoch,
7628        0,
7629        Vec::new(),
7630    ) else {
7631        return;
7632    };
7633    crate::forwarding::send_module_route_goodbye(
7634        counters,
7635        &target.sink,
7636        frame,
7637        target.module_id.as_deref(),
7638        context,
7639    );
7640}
7641
7642pub(crate) fn send_route_control_pushes(
7643    forwarding: &ForwardingTable,
7644    routes: Vec<EndpointRoute>,
7645    push: ClientControlPush,
7646) {
7647    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7648    for route in routes {
7649        let target = route.goodbye_target;
7650        if let Some((existing, channels)) = targets
7651            .iter_mut()
7652            .find(|(existing, _)| existing.connection_id == target.connection_id)
7653        {
7654            debug_assert_eq!(
7655                existing.negotiated_ver, target.negotiated_ver,
7656                "one connection cannot negotiate multiple frame versions"
7657            );
7658            if !channels.contains(&target.channel) {
7659                channels.push(target.channel);
7660            }
7661            continue;
7662        }
7663        let channel = target.channel;
7664        targets.push((target, vec![channel]));
7665    }
7666    for (target, mut channels) in targets {
7667        channels.sort_unstable();
7668        let mut push = push.clone();
7669        match &mut push {
7670            ClientControlPush::RouteClosing {
7671                channels: covered, ..
7672            }
7673            | ClientControlPush::RouteClosed {
7674                channels: covered, ..
7675            } => *covered = channels,
7676        }
7677        let body = match serde_json::to_vec(&push) {
7678            Ok(body) => body,
7679            Err(err) => {
7680                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7681                continue;
7682            }
7683        };
7684        let frame = match Frame::build_with_version(
7685            target.negotiated_ver,
7686            FrameType::Push,
7687            control_flags(),
7688            0,
7689            0,
7690            0,
7691            body.clone(),
7692        ) {
7693            Ok(frame) => frame,
7694            Err(err) => {
7695                warn!(
7696                    route_channel = target.channel,
7697                    error = %err,
7698                    "failed to build route lifecycle control PUSH frame"
7699                );
7700                continue;
7701            }
7702        };
7703        if let Err(err) = target.sink.try_send(frame) {
7704            if target.close_on_delivery_failure() {
7705                warn!(
7706                    target_connection_id = target.connection_id.get(),
7707                    route_channel = target.channel,
7708                    error = %err,
7709                    "route lifecycle control PUSH was not delivered to client; closing target connection"
7710                );
7711                let _ = forwarding.escalate_client_delivery_failure(
7712                    target.connection_id,
7713                    target.channel,
7714                    target.epoch,
7715                    CloseReason::new(
7716                        "route_lifecycle_push_delivery_failed",
7717                        format!(
7718                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7719                            target.channel
7720                        ),
7721                    ),
7722                    crate::forwarding::UndeliveredFrame {
7723                        module_id: target.module_id.as_deref(),
7724                        sink: &target.sink,
7725                    },
7726                );
7727            }
7728        }
7729    }
7730}
7731
7732#[cfg(test)]
7733mod tests {
7734    mod activity_read {
7735        include!("activity_read_tests.rs");
7736    }
7737
7738    mod lifecycle_activity {
7739        include!("control_lifecycle_tests.rs");
7740    }
7741    #[cfg(unix)]
7742    #[tokio::test]
7743    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7744        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7745        let path = dir.join("subc.jsonc");
7746        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7747            "program":"/bin/sleep","args":["60"],"protocol":"none",
7748            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7749        }}}).to_string()).unwrap();
7750        let mut configured = crate::daemon_config::load(&path)
7751            .unwrap()
7752            .unwrap()
7753            .modules
7754            .pop()
7755            .unwrap();
7756        let registry = std::sync::Arc::new(crate::Registry::default());
7757        let handle = crate::SupervisorHandle::new();
7758        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7759            .with_handle(handle.clone());
7760        let module = supervisor
7761            .supervise_configured_with_health(
7762                configured.module_spec(),
7763                true,
7764                configured.health.clone(),
7765                None,
7766                configured.restart,
7767            )
7768            .unwrap();
7769        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7770        let before = module.status().unwrap().pid;
7771        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7772        configured.health.cadence = std::time::Duration::from_secs(3600);
7773        let health_only = handler
7774            .reconcile_supervised_modules(
7775                &supervisor,
7776                vec![configured.clone()],
7777                false,
7778                Principal::Direct,
7779            )
7780            .await
7781            .unwrap();
7782        assert!(
7783            health_only.changed_pending_reload.is_empty(),
7784            "health policy is already applied live"
7785        );
7786        assert_eq!(module.status().unwrap().pid, before);
7787        assert_eq!(
7788            module.configuration().unwrap().1.http,
7789            configured.health.http
7790        );
7791        configured.args = vec!["61".into()];
7792        let launch = handler
7793            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7794            .await
7795            .unwrap();
7796        assert_eq!(launch.changed_pending_reload, ["stock"]);
7797        assert_eq!(
7798            module.status().unwrap().pid,
7799            before,
7800            "a launch edit is stored until reload"
7801        );
7802        module.drain().await.unwrap();
7803    }
7804    use cortexkit_test_support::ScratchDir;
7805    use std::{
7806        collections::BTreeMap,
7807        fmt,
7808        path::PathBuf,
7809        sync::{Arc, Mutex},
7810        time::Duration,
7811    };
7812
7813    use serde_json::{json, Value};
7814    use subc_protocol::{
7815        manifest::{
7816            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7817            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7818        },
7819        session::HealthStatus,
7820        FrameType,
7821    };
7822
7823    use super::*;
7824    use crate::{
7825        forwarding::{DataRoute, DataRouteState},
7826        registry::ChannelState,
7827        router::FrameSink,
7828        stderr_tail::DEFAULT_MAX_LINE_BYTES,
7829        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7830        RouteCtx, Router,
7831    };
7832    use tokio::{
7833        sync::mpsc,
7834        time::{sleep, Instant},
7835    };
7836    use tracing::{
7837        field::{Field, Visit},
7838        Event, Subscriber,
7839    };
7840    use tracing_subscriber::{layer::Context, prelude::*, Layer};
7841
7842    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
7843    ///
7844    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
7845    /// is only populated for `tests/*.rs` integration test binaries -- this file
7846    /// compiles as part of the library target, which gets neither. This test's
7847    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
7848    /// and the sibling binary lives two directories up at
7849    /// `<target-dir>/<profile>/fake-aft-stub`.
7850    ///
7851    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
7852    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
7853    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
7854    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
7855    /// `NotFound`, which reads as a broken test rather than an unbuilt
7856    /// dependency -- so state the cause and the remedy instead. Deliberately a
7857    /// panic and not a silent skip: a test that quietly passes when it could not
7858    /// run is worse than one that fails, because it reports health it never
7859    /// verified.
7860    fn fake_aft_stub_path() -> PathBuf {
7861        let mut path = std::env::current_exe().expect("current_exe available in tests");
7862        path.pop(); // .../deps/
7863        path.pop(); // .../<profile>/
7864        path.push(if cfg!(windows) {
7865            "fake-aft-stub.exe"
7866        } else {
7867            "fake-aft-stub"
7868        });
7869        assert!(
7870            path.exists(),
7871            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7872             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7873            path.display()
7874        );
7875        path
7876    }
7877
7878    /// Whether clients retry `code` in place: the predicate itself, never a copy
7879    /// of its set. A copied list breaks silently when a code is added to or
7880    /// removed from the real one, and a stale copy here would let exactly the
7881    /// failure this test exists to catch pass.
7882    fn client_retries(code: &str) -> bool {
7883        subc_protocol::error_codes::is_retryable_route_open(code)
7884    }
7885
7886    /// A code is not a label — clients branch on it, so publishing the wrong KIND
7887    /// of failure is worse than publishing none. A permanent fault dressed as
7888    /// retryable makes every client in the fleet retry forever against something
7889    /// that cannot recover; a transient fault dressed as permanent abandons work
7890    /// that would have succeeded.
7891    ///
7892    /// Asserting "a code exists" cannot catch either, because the string is free
7893    /// to say anything. This enumerates every variant and pins which side of the
7894    /// retry boundary it lands on, so a new variant must be classified here
7895    /// deliberately rather than inheriting whichever arm it was appended to.
7896    #[test]
7897    fn retryability_of_forwarding_codes_matches_the_failure() {
7898        // Transient by nature: the target is booting, reloading, or its endpoint
7899        // was swapped mid-flight. Retrying is how these resolve.
7900        let transient = [
7901            ForwardingError::NoModuleConnection,
7902            ForwardingError::ModuleReloading {
7903                module_id: "m".into(),
7904            },
7905            ForwardingError::StaleModuleEndpoint,
7906            ForwardingError::UnknownReservation {
7907                client_channel: 1,
7908                module_channel: 1,
7909            },
7910            ForwardingError::ConnectionClosing {
7911                connection_id: ConnectionId::new(1),
7912            },
7913            ForwardingError::ClientEgressClosed {
7914                connection_id: ConnectionId::new(1),
7915            },
7916            ForwardingError::ModuleEgressUnavailable {
7917                connection_id: ConnectionId::new(1),
7918            },
7919        ];
7920        for err in transient {
7921            let code = forwarding_error_code(&err);
7922            assert!(
7923                client_retries(code),
7924                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7925            );
7926        }
7927
7928        // Not fixed by retrying. Channel and correlation exhaustion need the
7929        // caller to close routes, and a poisoned lock is a daemon that cannot
7930        // recover at all — the worst thing to advertise as retryable, since every
7931        // client would storm a daemon that will never answer.
7932        let permanent = [
7933            ForwardingError::ConnectionRoleConflict {
7934                connection_id: ConnectionId::new(1),
7935            },
7936            ForwardingError::ClientRouteChannelExhausted {
7937                connection_id: ConnectionId::new(1),
7938            },
7939            ForwardingError::ModuleRouteChannelExhausted {
7940                endpoint: ModuleEndpointId {
7941                    connection_id: ConnectionId::new(1),
7942                    generation: 1,
7943                },
7944            },
7945            ForwardingError::RelayCorrelationExhausted,
7946            ForwardingError::RouteOpenBuild("x".into()),
7947            ForwardingError::Poisoned,
7948        ];
7949        for err in permanent {
7950            let code = forwarding_error_code(&err);
7951            assert!(
7952                !client_retries(code),
7953                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7954            );
7955        }
7956    }
7957
7958    /// The principal is the daemon's answer to "who is calling", and modules
7959    /// branch on it: aft gates bash on it, cerebellum gates browser control,
7960    /// plexus gates connector invocation. So a stamp is an authorization input in
7961    /// another process, not a label — and both possible answers SUCCEED, which is
7962    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
7963    /// first-party capability to something that never proved it; a supervised one
7964    /// stamped `Direct` silently strips a module of capability it is entitled to.
7965    ///
7966    /// Neither shows up in a test that only checks the bind succeeded. Before this
7967    /// test the only coverage was accidental —
7968    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
7969    /// stamped principal on its way past, so narrowing that wire-shape test to its
7970    /// stated subject would have deleted the last assertion on this value. It
7971    /// still asserts the stamp, which is now redundancy rather than the only
7972    /// guard: both fail under the same mutation, and this one names the reason.
7973    /// SCOPE: this handler's supervisor has spawned nothing, so
7974    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
7975    /// is unreachable here. Both assertions below are refusals, and a mutant that
7976    /// refuses everything would satisfy them.
7977    ///
7978    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
7979    /// spawns a supervised consumer, reads its live nonce, and asserts the module
7980    /// observed `principal.kind == "reserved"` carrying that module_id — verified
7981    /// at source rather than assumed, since a citation is a claim about another
7982    /// file and ages like one. Recorded because a harness that structurally
7983    /// cannot reach an arm reports "none" for that arm identically to one that
7984    /// covers it and found nothing.
7985    #[tokio::test]
7986    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7987        let handler = ControlHandler::default();
7988        let frame =
7989            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7990
7991        // Absent consumer_identity is the ordinary case: a human at a terminal, or
7992        // any process holding the connection file. Nothing was proved, so nothing
7993        // may be granted beyond the unattested floor.
7994        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7995        assert_eq!(
7996            stamped,
7997            Principal::Direct,
7998            "a caller that proved nothing must not be stamped as a supervised module"
7999        );
8000
8001        // A claimed module_id with a nonce no supervised child was given is a
8002        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8003        // quietly demoted to Direct, or an impersonation attempt looks identical
8004        // to an ordinary unattested connection.
8005        let forged = handler
8006            .route_open_principal(
8007                &frame,
8008                Some(ConsumerIdentity {
8009                    module_id: "aft".to_string(),
8010                    launch_nonce: "not-a-real-nonce".to_string(),
8011                }),
8012            )
8013            .unwrap();
8014        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8015        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8016    }
8017
8018    /// The test above hands `route_open_principal` an identity it built itself,
8019    /// which proves the stamping rule and nothing about where the identity comes
8020    /// from. The real producer is a wire body, and the two are joined by a serde
8021    /// field name that nothing else asserts.
8022    ///
8023    /// That join fails quietly in one specific way: an unrecognised key is simply
8024    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8025    /// yields `None` and every supervised module silently drops to `Direct`.
8026    /// Capability-wise that is the safe direction, but it surfaces far from its
8027    /// cause — as a module mysteriously refused bash — and it would pass every
8028    /// test that builds its own input.
8029    ///
8030    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8031    /// would break every client the moment the daemon gains a field, trading a
8032    /// quiet demotion for a hard refusal on additive change. Asserting the join
8033    /// instead means a rename breaks a test here rather than the fleet.
8034    #[test]
8035    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8036        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8037        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8038        let ClientControlRequest::RouteOpen {
8039            consumer_identity, ..
8040        } = parsed
8041        else {
8042            panic!("route.open body must parse as RouteOpen");
8043        };
8044        assert_eq!(
8045            consumer_identity,
8046            Some(ConsumerIdentity {
8047                module_id: "aft".to_string(),
8048                launch_nonce: "n".to_string(),
8049            }),
8050            "the wire field name must reach the value route_open_principal reads"
8051        );
8052    }
8053
8054    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8055        ModuleManifest::builder(module_id, "0.1.0")
8056            .protocol_ver(protocol_ver)
8057            .provides(vec![ProviderRole::ToolProvider {
8058                tools: vec![Tool {
8059                    name: "read".to_string(),
8060                    description: None,
8061                    execution_mode: ExecutionMode::Pure,
8062                    schema: json!({"type": "object"}),
8063                }],
8064                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8065                concurrency: Concurrency::ModuleManaged,
8066                emits_push: true,
8067                sub_supervises: true,
8068            }])
8069            .build()
8070    }
8071
8072    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8073        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8074    }
8075
8076    fn hello_frame_with_control_ops(
8077        module_id: &str,
8078        protocol_ver: u8,
8079        corr: u64,
8080        control_ops: Option<Vec<String>>,
8081    ) -> Frame {
8082        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8083    }
8084
8085    fn hello_frame_with_nonce(
8086        module_id: &str,
8087        protocol_ver: u8,
8088        corr: u64,
8089        launch_nonce: Option<&str>,
8090    ) -> Frame {
8091        hello_frame_full(
8092            module_id,
8093            protocol_ver,
8094            corr,
8095            None,
8096            launch_nonce.map(ToOwned::to_owned),
8097        )
8098    }
8099
8100    fn hello_frame_full(
8101        module_id: &str,
8102        protocol_ver: u8,
8103        corr: u64,
8104        control_ops: Option<Vec<String>>,
8105        launch_nonce: Option<String>,
8106    ) -> Frame {
8107        let body = serde_json::to_vec(&ModuleHelloBody {
8108            manifest: manifest(module_id, protocol_ver),
8109            protocol_ver,
8110            control_ops,
8111            launch_nonce,
8112        })
8113        .unwrap();
8114        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8115    }
8116
8117    fn non_routable_hello_frame_with_control_ops(
8118        module_id: &str,
8119        corr: u64,
8120        control_ops: Option<Vec<String>>,
8121    ) -> Frame {
8122        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8123        manifest.provides.clear();
8124        let body = serde_json::to_vec(&ModuleHelloBody {
8125            manifest,
8126            protocol_ver: PROTOCOL_VERSION,
8127            control_ops,
8128            launch_nonce: None,
8129        })
8130        .unwrap();
8131        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8132    }
8133
8134    fn capability_grammar_hello_frame(
8135        capabilities: Value,
8136        runtime_computed: Option<Value>,
8137        corr: u64,
8138    ) -> Frame {
8139        let mut body = serde_json::to_value(ModuleHelloBody {
8140            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8141            protocol_ver: PROTOCOL_VERSION,
8142            control_ops: None,
8143            launch_nonce: None,
8144        })
8145        .expect("HELLO body serializes");
8146        body["manifest"]["capabilities"] = capabilities;
8147        if let Some(runtime_computed) = runtime_computed {
8148            body["runtime_computed"] = runtime_computed;
8149        }
8150        Frame::build(
8151            FrameType::Hello,
8152            control_flags(),
8153            0,
8154            0,
8155            corr,
8156            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8157        )
8158        .expect("HELLO frame builds")
8159    }
8160
8161    fn channel_request(channel: u16, corr: u64) -> Frame {
8162        Frame::build(
8163            FrameType::Request,
8164            Flags::new(true, Priority::Interactive, false),
8165            channel,
8166            0,
8167            corr,
8168            b"opaque".to_vec(),
8169        )
8170        .unwrap()
8171    }
8172
8173    fn route_ctx(
8174        connection_id: ConnectionId,
8175    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8176        let (tx, rx) = mpsc::channel(8);
8177        (
8178            RouteCtx {
8179                connection_id,
8180                egress: FrameSink::new(tx),
8181            },
8182            rx,
8183        )
8184    }
8185
8186    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8187        serde_json::from_slice(&frame.body).unwrap()
8188    }
8189
8190    /// Register a module over a connection that has a sink and return the
8191    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8192    /// module's own sink rather than returning it as a reply, so the ack is
8193    /// taken off `rx` here and whatever the test reads next is what followed it.
8194    async fn hello_via_sink(
8195        handler: &ControlHandler,
8196        ctx: &RouteCtx,
8197        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8198        hello: Frame,
8199    ) -> Frame {
8200        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8201        assert!(
8202            replies.is_empty(),
8203            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8204        );
8205        let ack = rx
8206            .try_recv()
8207            .expect("HELLO_ACK is queued on the module sink")
8208            .frame;
8209        assert_eq!(ack.header.ty, FrameType::HelloAck);
8210        ack
8211    }
8212
8213    fn parse_error(frame: &Frame) -> Value {
8214        serde_json::from_slice(&frame.body).unwrap()
8215    }
8216
8217    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8218        serde_json::from_slice(&frame.body).unwrap()
8219    }
8220
8221    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8222        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8223            route_channel,
8224            route_epoch: 0,
8225            kind,
8226        })
8227        .unwrap();
8228        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8229    }
8230
8231    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8232        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8233            module_id: module_id.to_string(),
8234        })
8235        .unwrap();
8236        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8237    }
8238
8239    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8240        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8241    }
8242
8243    fn route_open_frame_with_consumer_capabilities(
8244        corr: u64,
8245        module_id: &str,
8246        project_root: ScratchDir,
8247        consumer_capabilities: Option<Vec<String>>,
8248    ) -> Frame {
8249        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8250            target: RouteTarget::ToolProvider {
8251                module_id: module_id.to_string(),
8252            },
8253            identity: BindIdentity::new(
8254                project_root.path().to_path_buf(),
8255                "unit".to_string(),
8256                "session".to_string(),
8257            ),
8258            consumer_identity: None,
8259            consumer_capabilities,
8260            role_versions: None,
8261            admission_facts: None,
8262            scope: None,
8263        })
8264        .unwrap();
8265        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8266    }
8267
8268    fn route_open_frame_with_role_versions(
8269        corr: u64,
8270        module_id: &str,
8271        project_root: ScratchDir,
8272        role_versions: Option<BTreeMap<String, String>>,
8273    ) -> Frame {
8274        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8275            target: RouteTarget::ToolProvider {
8276                module_id: module_id.to_string(),
8277            },
8278            identity: BindIdentity::new(
8279                project_root.path().to_path_buf(),
8280                "unit".to_string(),
8281                format!("session-{corr}"),
8282            ),
8283            consumer_identity: None,
8284            consumer_capabilities: None,
8285            role_versions,
8286            admission_facts: None,
8287            scope: None,
8288        })
8289        .unwrap();
8290        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8291    }
8292
8293    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8294        entries
8295            .iter()
8296            .map(|(role, version)| (role.to_string(), version.to_string()))
8297            .collect()
8298    }
8299
8300    fn route_open_frame_with_admission_facts(
8301        corr: u64,
8302        module_id: &str,
8303        project_root: ScratchDir,
8304        consumer_identity: Option<subc_control::ConsumerIdentity>,
8305        facts: Option<Value>,
8306    ) -> Frame {
8307        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8308            target: RouteTarget::ToolProvider {
8309                module_id: module_id.to_string(),
8310            },
8311            identity: BindIdentity::new(
8312                project_root.path().to_path_buf(),
8313                "unit".to_string(),
8314                format!("session-{corr}"),
8315            ),
8316            consumer_identity,
8317            consumer_capabilities: None,
8318            role_versions: None,
8319            admission_facts: facts,
8320            scope: None,
8321        })
8322        .unwrap();
8323        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8324    }
8325
8326    #[derive(Clone, Default)]
8327    struct EventCapture {
8328        events: Arc<Mutex<Vec<CapturedEvent>>>,
8329    }
8330
8331    #[derive(Clone, Debug)]
8332    struct CapturedEvent {
8333        target: String,
8334        level: tracing::Level,
8335        fields: BTreeMap<String, String>,
8336    }
8337
8338    impl EventCapture {
8339        fn events(&self) -> Vec<CapturedEvent> {
8340            self.events.lock().unwrap().clone()
8341        }
8342    }
8343
8344    impl<S> Layer<S> for EventCapture
8345    where
8346        S: Subscriber,
8347    {
8348        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8349            let mut visitor = EventFieldVisitor::default();
8350            event.record(&mut visitor);
8351            self.events.lock().unwrap().push(CapturedEvent {
8352                target: event.metadata().target().to_string(),
8353                level: *event.metadata().level(),
8354                fields: visitor.fields,
8355            });
8356        }
8357    }
8358
8359    #[derive(Default)]
8360    struct EventFieldVisitor {
8361        fields: BTreeMap<String, String>,
8362    }
8363
8364    impl Visit for EventFieldVisitor {
8365        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8366            self.fields
8367                .insert(field.name().to_string(), format!("{value:?}"));
8368        }
8369    }
8370
8371    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8372        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8373            status,
8374            detail: Some("warming".to_string()),
8375            metrics: Some(json!({"queue_depth": 3})),
8376        })
8377        .unwrap();
8378        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8379    }
8380
8381    fn route_bind_ack(corr: u64) -> Frame {
8382        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8383        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8384    }
8385
8386    fn unique_project_root(label: &str) -> ScratchDir {
8387        ScratchDir::new(label)
8388    }
8389
8390    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8391        match parse_route_poll(frame) {
8392            ClientControlResponse::RoutePoll {
8393                status: None,
8394                live: Some(live),
8395                ..
8396            } => assert_eq!(live, expected_live),
8397            other => panic!("unexpected route.poll response: {other:?}"),
8398        }
8399    }
8400
8401    fn bind_liveness_route(
8402        registry: &Registry,
8403        forwarding: &ForwardingTable,
8404        module_id: &str,
8405    ) -> (RouteCtx, u16, u32) {
8406        let module_connection = ConnectionId::new(101);
8407        let client_connection = ConnectionId::new(202);
8408        let registration = registry
8409            .register_with_control_ops(
8410                manifest(module_id, PROTOCOL_VERSION),
8411                PROTOCOL_VERSION,
8412                module_connection,
8413                module_baseline_control_ops(),
8414            )
8415            .unwrap();
8416        let (module_tx, _module_rx) = mpsc::channel(8);
8417        let endpoint = forwarding
8418            .register_module_connection(
8419                module_connection,
8420                module_id.to_string(),
8421                PROTOCOL_VERSION,
8422                manifest_concurrency(&registration.manifest),
8423                FrameSink::new(module_tx),
8424            )
8425            .unwrap();
8426        let (client_ctx, _client_rx) = route_ctx(client_connection);
8427        let pending = forwarding
8428            .begin_route_bind_relay_for_test(
8429                client_connection,
8430                client_ctx.egress.clone(),
8431                1,
8432                module_id,
8433            )
8434            .unwrap();
8435        assert_eq!(pending.endpoint, endpoint);
8436        let route_channel = pending.client_channel;
8437        let route_epoch = pending.client_epoch;
8438        forwarding
8439            .complete_pending_relay(
8440                module_connection,
8441                pending.corr,
8442                RouteBindRelayOutcome::Accepted,
8443            )
8444            .unwrap();
8445        (client_ctx, route_channel, route_epoch)
8446    }
8447
8448    struct FakeProcessLiveness {
8449        live: Option<bool>,
8450    }
8451
8452    impl ModuleProcessLiveness for FakeProcessLiveness {
8453        fn process_live(&self, _module_id: &str) -> Option<bool> {
8454            self.live
8455        }
8456    }
8457
8458    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8459    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8460    {
8461        let registry = Arc::new(Registry::default());
8462        let supervisor_handle = SupervisorHandle::new();
8463        let supervisor = Supervisor::new_for_test(
8464            Arc::clone(&registry),
8465            RestartPolicy::new(1, Duration::from_millis(10)),
8466        )
8467        .with_handle(supervisor_handle.clone());
8468        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8469        let module = supervisor
8470            .spawn(ModuleSpec {
8471                module_id: "stderr-tail-wire".to_string(),
8472                program: fake_aft_stub_path(),
8473                args: Vec::new(),
8474                env: vec![
8475                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8476                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8477                ],
8478                reserved: false,
8479                reserved_prefixes: Vec::new(),
8480                protocol: ModuleProtocol::Subc,
8481                overlap: Default::default(),
8482            })
8483            .unwrap();
8484
8485        let deadline = Instant::now() + Duration::from_secs(5);
8486        loop {
8487            let tail = module.stderr_tail(None, None);
8488            if tail
8489                .entries
8490                .iter()
8491                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8492                && tail.entries.iter().any(|entry| {
8493                    matches!(
8494                        entry,
8495                        TailEntry::Line {
8496                            truncated: true,
8497                            ..
8498                        }
8499                    )
8500                })
8501            {
8502                break;
8503            }
8504            assert!(
8505                Instant::now() < deadline,
8506                "module did not produce a truncated line and restart boundary: {tail:?}"
8507            );
8508            sleep(Duration::from_millis(10)).await;
8509        }
8510
8511        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8512        let request = ClientControlRequest::SupervisorStderrTail {
8513            module_id: "stderr-tail-wire".to_string(),
8514            max_lines: None,
8515            max_bytes: None,
8516        };
8517        let frame = Frame::build(
8518            FrameType::Request,
8519            control_flags(),
8520            0,
8521            0,
8522            1,
8523            serde_json::to_vec(&request).unwrap(),
8524        )
8525        .unwrap();
8526        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8527        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8528        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8529            serde_json::from_slice(&responses[0].body).unwrap()
8530        else {
8531            panic!("expected supervisor.stderr_tail response");
8532        };
8533
8534        assert!(
8535            tail.entries
8536                .iter()
8537                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8538            "the control response lost the restart boundary"
8539        );
8540        let Some(StderrTailEntry::Line {
8541            text,
8542            truncated,
8543            at_ms,
8544        }) = tail.entries.iter().find(|entry| {
8545            matches!(
8546                entry,
8547                StderrTailEntry::Line {
8548                    truncated: true,
8549                    ..
8550                }
8551            )
8552        })
8553        else {
8554            panic!("the control response lost the truncated line");
8555        };
8556        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8557        assert!(*truncated);
8558        assert!(
8559            at_ms.is_some(),
8560            "the control response lost the line's capture time"
8561        );
8562    }
8563
8564    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8565    /// read done on the worker thread would stall every other task until it
8566    /// finished; the read must run off the worker so this test's own task keeps
8567    /// running while the read is paused.
8568    #[tokio::test(flavor = "current_thread")]
8569    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8570        let dir = ScratchDir::new("terminals-off-worker");
8571        let journal_path = dir.join("terminals.jsonl");
8572        let registry = Arc::new(Registry::default());
8573        let supervisor_handle = SupervisorHandle::new();
8574        let supervisor =
8575            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8576                .with_handle(supervisor_handle.clone())
8577                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8578        let module = supervisor
8579            .spawn(ModuleSpec {
8580                module_id: "terminal-off-worker".to_string(),
8581                program: fake_aft_stub_path(),
8582                args: Vec::new(),
8583                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8584                reserved: false,
8585                reserved_prefixes: Vec::new(),
8586                protocol: ModuleProtocol::Subc,
8587                overlap: Default::default(),
8588            })
8589            .unwrap();
8590        let deadline = Instant::now() + Duration::from_secs(5);
8591        while module.terminal_history().entries.len() != 2 {
8592            assert!(Instant::now() < deadline, "module did not record two exits");
8593            sleep(Duration::from_millis(10)).await;
8594        }
8595
8596        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8597        let handler =
8598            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8599        let frame = Frame::build(
8600            FrameType::Request,
8601            control_flags(),
8602            0,
8603            0,
8604            1,
8605            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8606                module_id: "terminal-off-worker".to_string(),
8607            })
8608            .unwrap(),
8609        )
8610        .unwrap();
8611        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8612        let spawned_at = std::time::Instant::now();
8613        let read = tokio::spawn({
8614            let handler = Arc::clone(&handler);
8615            async move { handler.handle_control_frame(&ctx, frame).await }
8616        });
8617        // Waiting for the pause from a blocking thread keeps this task pending,
8618        // so the runtime's single worker is free to run the read task.
8619        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8620            .await
8621            .unwrap()
8622            .expect("the history read reached its pause");
8623        let elapsed = spawned_at.elapsed();
8624        assert!(
8625            elapsed < Duration::from_secs(2) && !read.is_finished(),
8626            "this task could not run while the history read was paused \
8627             (resumed after {elapsed:?}, read finished: {})",
8628            read.is_finished()
8629        );
8630
8631        drop(release);
8632        let responses = read.await.unwrap().unwrap();
8633        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8634        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8635            panic!("expected supervisor.terminals response");
8636        };
8637        assert_eq!(terminals.entries.len(), 2);
8638        assert_eq!(terminals.journal_skipped_lines, 0);
8639        assert_eq!(terminals.journal_read_errors, 0);
8640    }
8641
8642    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8643    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8644        let registry = Arc::new(Registry::default());
8645        let supervisor_handle = SupervisorHandle::new();
8646        let supervisor =
8647            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8648                .with_handle(supervisor_handle.clone());
8649        let module = supervisor
8650            .spawn(ModuleSpec {
8651                module_id: "terminal-golden".to_string(),
8652                program: fake_aft_stub_path(),
8653                args: Vec::new(),
8654                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8655                reserved: false,
8656                reserved_prefixes: Vec::new(),
8657                protocol: ModuleProtocol::Subc,
8658                overlap: Default::default(),
8659            })
8660            .unwrap();
8661
8662        let deadline = Instant::now() + Duration::from_secs(5);
8663        while module.terminal_history().entries.len() != 2 {
8664            assert!(
8665                Instant::now() < deadline,
8666                "module did not retain two terminal exits: {:?}",
8667                module.terminal_history()
8668            );
8669            sleep(Duration::from_millis(10)).await;
8670        }
8671
8672        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8673        let request = ClientControlRequest::SupervisorTerminals {
8674            module_id: "terminal-golden".to_string(),
8675        };
8676        let frame = Frame::build(
8677            FrameType::Request,
8678            control_flags(),
8679            0,
8680            0,
8681            1,
8682            serde_json::to_vec(&request).unwrap(),
8683        )
8684        .unwrap();
8685        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8686        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8687        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8688        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8689            panic!("expected supervisor.terminals response");
8690        };
8691        assert_eq!(terminals.entries.len(), 2);
8692        assert_eq!(terminals.dropped, 0);
8693
8694        let mut rendered = serde_json::to_value(response).unwrap();
8695        // Wall-clock fields are the observation contract, but not stable fixture
8696        // bytes; normalize only them after the real handler has shaped the response.
8697        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8698        for (index, entry) in rendered["entries"]
8699            .as_array_mut()
8700            .expect("terminal response entries array")
8701            .iter_mut()
8702            .enumerate()
8703        {
8704            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8705        }
8706
8707        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8708            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8709        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8710        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8711            std::fs::write(&golden_path, &serialized).unwrap();
8712        }
8713        let expected: Value =
8714            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8715        assert_eq!(rendered, expected);
8716    }
8717
8718    #[test]
8719    fn hello_registers_manifest_and_returns_ack() {
8720        let registry = Arc::new(Registry::default());
8721        let handler = ControlHandler::new(Arc::clone(&registry));
8722        let conn = ConnectionId::new(1);
8723
8724        let responses = handler
8725            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8726            .unwrap();
8727
8728        assert_eq!(responses.len(), 1);
8729        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8730        assert_eq!(responses[0].header.channel, 0);
8731        assert_eq!(responses[0].header.corr, 7);
8732        let ack = parse_ack(&responses[0]);
8733        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8734        assert!(ack
8735            .subc_capabilities
8736            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8737        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8738        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8739        assert!(ack
8740            .subc_ops
8741            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8742        assert!(ack
8743            .subc_ops
8744            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8745
8746        let registration = registry.get_module("aft").unwrap().unwrap();
8747        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8748        assert_eq!(registration.state, ChannelState::Active);
8749        assert_eq!(registration.connection_id, conn);
8750        assert_eq!(registration.control_ops, module_baseline_control_ops());
8751    }
8752
8753    #[test]
8754    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8755        let invalid_identifiers = [
8756            ("case_change", "credentials-Provider/v1"),
8757            ("leading_zero", "credentials-provider/v01"),
8758            ("trailing_hyphen", "credentials-provider-/v1"),
8759            ("consecutive_hyphens", "credentials--provider/v1"),
8760            ("uppercase", "Credentials-provider/v1"),
8761            ("missing_v", "credentials-provider/1"),
8762            ("whitespace", "credentials provider/v1"),
8763            ("zero_version", "credentials-provider/v0"),
8764            ("out_of_range_version", "credentials-provider/v4294967296"),
8765            (
8766                "overlength_name",
8767                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8768            ),
8769        ];
8770        let mut cases = invalid_identifiers
8771            .into_iter()
8772            .map(|(name, identifier)| {
8773                (
8774                    format!("identifier_{name}"),
8775                    "capabilities.provides[0]".to_string(),
8776                    identifier.to_string(),
8777                    json!({ "provides": [identifier] }),
8778                    None,
8779                )
8780            })
8781            .collect::<Vec<_>>();
8782        cases.extend([
8783            (
8784                "unknown_need".to_string(),
8785                "capabilities.requires[0].need".to_string(),
8786                "deferred".to_string(),
8787                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8788                None,
8789            ),
8790            (
8791                "duplicate_provides".to_string(),
8792                "capabilities.provides[1]".to_string(),
8793                "credentials-provider/v1".to_string(),
8794                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8795                None,
8796            ),
8797            (
8798                "duplicate_must_never_reach".to_string(),
8799                "capabilities.must_never_reach[1]".to_string(),
8800                "credentials-provider/v1".to_string(),
8801                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8802                None,
8803            ),
8804            (
8805                "duplicate_requires_same_need".to_string(),
8806                "capabilities.requires[1]".to_string(),
8807                "credentials-provider/v1".to_string(),
8808                json!({ "requires": [
8809                    { "capability": "credentials-provider/v1", "need": "required" },
8810                    { "capability": "credentials-provider/v1", "need": "required" }
8811                ] }),
8812                None,
8813            ),
8814            (
8815                "duplicate_requires_conflicting_need".to_string(),
8816                "capabilities.requires[1]".to_string(),
8817                "credentials-provider/v1".to_string(),
8818                json!({ "requires": [
8819                    { "capability": "credentials-provider/v1", "need": "required" },
8820                    { "capability": "credentials-provider/v1", "need": "optional" }
8821                ] }),
8822                None,
8823            ),
8824            (
8825                "capabilities_root_pointer".to_string(),
8826                "runtime_computed[0]".to_string(),
8827                "/capabilities".to_string(),
8828                json!({}),
8829                Some(json!(["/capabilities"])),
8830            ),
8831            (
8832                "capabilities_descendant_pointer".to_string(),
8833                "runtime_computed[0]".to_string(),
8834                "/capabilities/provides".to_string(),
8835                json!({}),
8836                Some(json!(["/capabilities/provides"])),
8837            ),
8838            (
8839                "malformed_pointer_without_leading_slash".to_string(),
8840                "runtime_computed[0]".to_string(),
8841                "capabilities".to_string(),
8842                json!({}),
8843                Some(json!(["capabilities"])),
8844            ),
8845            (
8846                "malformed_pointer_escape".to_string(),
8847                "runtime_computed[0]".to_string(),
8848                "/roles/~2/tools".to_string(),
8849                json!({}),
8850                Some(json!(["/roles/~2/tools"])),
8851            ),
8852            (
8853                "unknown_capabilities_field".to_string(),
8854                "capabilities.future".to_string(),
8855                "<array>".to_string(),
8856                json!({ "future": [] }),
8857                None,
8858            ),
8859        ]);
8860
8861        for (index, (name, field, value, capabilities, runtime_computed)) in
8862            cases.into_iter().enumerate()
8863        {
8864            let registry = Arc::new(Registry::default());
8865            let handler = ControlHandler::new(Arc::clone(&registry));
8866            let response = handler
8867                .handle_control(
8868                    ConnectionId::new((index + 1) as u64),
8869                    capability_grammar_hello_frame(
8870                        capabilities,
8871                        runtime_computed,
8872                        index as u64 + 1,
8873                    ),
8874                )
8875                .expect("invalid HELLO returns a refusal");
8876
8877            assert_eq!(response.len(), 1, "{name} must emit one refusal");
8878            let error = parse_error(&response[0]);
8879            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8880            let message = error["message"]
8881                .as_str()
8882                .expect("error message is a string");
8883            assert!(
8884                message.contains(&field),
8885                "{name}: field missing from {message}"
8886            );
8887            assert!(
8888                message.contains(&value),
8889                "{name}: value missing from {message}"
8890            );
8891            assert_eq!(
8892                registry
8893                    .active_registration_count()
8894                    .expect("registry reads"),
8895                0,
8896                "{name}: refused HELLO must not create a catalog entry"
8897            );
8898        }
8899    }
8900
8901    #[test]
8902    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8903        let registry = Arc::new(Registry::default());
8904        let handler = ControlHandler::new(Arc::clone(&registry));
8905        let capabilities = json!({
8906            "provides": ["credentials-provider/v1"],
8907            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8908            "must_never_reach": ["federation-transport/v1"]
8909        });
8910        let response = handler
8911            .handle_control(
8912                ConnectionId::new(99),
8913                capability_grammar_hello_frame(
8914                    capabilities.clone(),
8915                    Some(json!(["/roles/0/tools"])),
8916                    99,
8917                ),
8918            )
8919            .expect("valid HELLO registers");
8920        assert_eq!(response[0].header.ty, FrameType::HelloAck);
8921
8922        let request = Frame::build(
8923            FrameType::Request,
8924            control_flags(),
8925            0,
8926            0,
8927            100,
8928            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8929                .expect("catalog request serializes"),
8930        )
8931        .expect("catalog request frame builds");
8932        let response = handler
8933            .handle_catalog_list(request, None)
8934            .expect("catalog list succeeds");
8935        let ClientControlResponse::CatalogList { modules, .. } =
8936            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8937        else {
8938            panic!("catalog request must return catalog.list");
8939        };
8940        assert_eq!(modules.len(), 1);
8941        assert_eq!(
8942            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8943            capabilities
8944        );
8945    }
8946
8947    #[test]
8948    fn catalog_list_mirrors_management_operation_description() {
8949        let registry = Arc::new(Registry::default());
8950        let handler = ControlHandler::new(Arc::clone(&registry));
8951        let description = "List managed records and return their identifiers and metadata.";
8952        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8953        manifest.provides = vec![ProviderRole::ManagementSurface {
8954            operations: vec![ManagementOperation {
8955                name: "records.list".to_string(),
8956                kind: ManagementOperationKind::Query,
8957                description: Some(description.to_string()),
8958            }],
8959            config_schema: json!({"type": "object"}),
8960            observability: vec![ObservabilitySurface {
8961                name: "records.stats".to_string(),
8962                kind: ObservabilityKind::Snapshot,
8963            }],
8964            identity_scope: vec![IdentityScope::Project],
8965            concurrency: Concurrency::ModuleManaged,
8966        }];
8967        registry
8968            .register_with_control_ops(
8969                manifest,
8970                PROTOCOL_VERSION,
8971                ConnectionId::new(99),
8972                Vec::new(),
8973            )
8974            .expect("described management manifest registers");
8975
8976        let request = Frame::build(
8977            FrameType::Request,
8978            control_flags(),
8979            0,
8980            0,
8981            100,
8982            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8983                .expect("catalog request serializes"),
8984        )
8985        .expect("catalog request frame builds");
8986        let response = handler
8987            .handle_catalog_list(request, None)
8988            .expect("catalog list succeeds");
8989        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8990        assert_eq!(
8991            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8992            "catalog.list must preserve the declared operation description verbatim"
8993        );
8994    }
8995
8996    #[test]
8997    fn catalog_list_mirrors_registered_operation_delivery() {
8998        let registry = Arc::new(Registry::default());
8999        let handler = ControlHandler::new(Arc::clone(&registry));
9000        let delivery =
9001            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9002        let mut body = serde_json::to_value(ModuleHelloBody {
9003            manifest: manifest("watch-management", PROTOCOL_VERSION),
9004            protocol_ver: PROTOCOL_VERSION,
9005            control_ops: None,
9006            launch_nonce: None,
9007        })
9008        .unwrap();
9009        body["manifest"]["provides"] = json!([{
9010            "role": "management_surface",
9011            "operations": [
9012                {"name": "status", "kind": "query"},
9013                {"name": "future", "kind": "query"},
9014                {"name": "reset", "kind": "mutate"}
9015            ],
9016            "config_schema": {}, "observability": [], "identity_scope": []
9017        }]);
9018        body["manifest"]["operation_delivery"] = delivery.clone();
9019        let hello = Frame::build(
9020            FrameType::Hello,
9021            control_flags(),
9022            0,
9023            0,
9024            99,
9025            serde_json::to_vec(&body).unwrap(),
9026        )
9027        .unwrap();
9028        let response = handler
9029            .handle_control(ConnectionId::new(99), hello)
9030            .unwrap();
9031        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9032
9033        let (_, registrations) = registry.list_modules().unwrap();
9034        let registered = &registrations[0].manifest;
9035        assert!(registered.is_watch_operation("status"));
9036        assert!(!registered.is_watch_operation("future"));
9037        assert!(!registered.is_watch_operation("reset"));
9038        assert!(!registered.is_watch_operation("missing"));
9039
9040        let request = Frame::build(
9041            FrameType::Request,
9042            control_flags(),
9043            0,
9044            0,
9045            100,
9046            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9047        )
9048        .unwrap();
9049        let response = handler.handle_catalog_list(request, None).unwrap();
9050        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9051        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9052        let ClientControlResponse::CatalogList { modules, .. } =
9053            serde_json::from_slice(&response[0].body).unwrap()
9054        else {
9055            panic!("expected catalog.list");
9056        };
9057        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9058    }
9059
9060    #[test]
9061    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9062        let registry = Arc::new(Registry::default());
9063        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9064            [("vault".to_string(), true), ("squatter".to_string(), true)],
9065            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9066        );
9067        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9068        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9069            provides: vec!["credentials-provider/v1".to_string()],
9070            requires: Vec::new(),
9071            must_never_reach: Vec::new(),
9072        });
9073        let frame = Frame::build(
9074            FrameType::Hello,
9075            control_flags(),
9076            0,
9077            0,
9078            77,
9079            serde_json::to_vec(&ModuleHelloBody {
9080                manifest: squatter,
9081                protocol_ver: PROTOCOL_VERSION,
9082                control_ops: None,
9083                launch_nonce: None,
9084            })
9085            .expect("HELLO serializes"),
9086        )
9087        .expect("HELLO frame builds");
9088        let response = handler
9089            .handle_control(ConnectionId::new(77), frame)
9090            .expect("reserved claim receives a typed refusal");
9091        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9092        assert_eq!(
9093            registry
9094                .active_registration_count()
9095                .expect("registry reads"),
9096            0,
9097            "a reserved capability refusal must not leave a catalog entry"
9098        );
9099    }
9100
9101    #[test]
9102    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9103        for settlement in ["timeout", "inconclusive", "drop"] {
9104            let breakers = RouteBindBreakers::default();
9105            let RouteBindAdmission::Admitted {
9106                guard: mut old,
9107                probe: false,
9108            } = breakers.admit("prov")
9109            else {
9110                panic!("ordinary relay admitted")
9111            };
9112            let RouteBindAdmission::Admitted {
9113                guard: mut opener, ..
9114            } = breakers.admit("prov")
9115            else {
9116                panic!("second relay admitted")
9117            };
9118            assert!(
9119                !opener
9120                    .record_timeout(1, Duration::ZERO)
9121                    .unwrap()
9122                    .reopened_after_probe
9123            );
9124            let RouteBindAdmission::Admitted {
9125                guard: mut probe,
9126                probe: true,
9127            } = breakers.admit("prov")
9128            else {
9129                panic!("one cooldown probe admitted")
9130            };
9131            match settlement {
9132                "timeout" => assert!(
9133                    !old.record_timeout(1, Duration::ZERO)
9134                        .unwrap()
9135                        .reopened_after_probe
9136                ),
9137                "inconclusive" => old.record_inconclusive(),
9138                "drop" => drop(old),
9139                _ => unreachable!(),
9140            }
9141            assert!(
9142                matches!(
9143                    breakers.admit("prov"),
9144                    RouteBindAdmission::Refused {
9145                        probe_in_flight: true,
9146                        ..
9147                    }
9148                ),
9149                "{settlement} of a pre-open relay cannot release the real probe"
9150            );
9151            assert!(
9152                probe
9153                    .record_timeout(1, Duration::ZERO)
9154                    .unwrap()
9155                    .reopened_after_probe
9156            );
9157            assert!(matches!(
9158                breakers.admit("prov"),
9159                RouteBindAdmission::Admitted { probe: true, .. }
9160            ));
9161        }
9162        let breakers = RouteBindBreakers::default();
9163        let admit = || match breakers.admit("prov") {
9164            RouteBindAdmission::Admitted { guard, .. } => guard,
9165            _ => panic!("relay admitted"),
9166        };
9167        admit().record_timeout(1, Duration::ZERO);
9168        let mut old_probe = admit();
9169        breakers.reset_for_new_module_connection("prov");
9170        admit().record_timeout(1, Duration::ZERO);
9171        let _new_probe = admit();
9172        old_probe.record_inconclusive();
9173        assert!(matches!(
9174            breakers.admit("prov"),
9175            RouteBindAdmission::Refused {
9176                probe_in_flight: true,
9177                ..
9178            }
9179        ));
9180    }
9181
9182    #[tokio::test]
9183    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9184        for candidate in [false, true] {
9185            let registry = Arc::new(Registry::default());
9186            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9187                [("vault".to_string(), true), ("squatter".to_string(), true)],
9188                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9189            );
9190            let conn = ConnectionId::new(77);
9191            let (ctx, mut rx) = route_ctx(conn);
9192            let initial = capability_manifest("squatter", &[], &[]);
9193            if candidate {
9194                registry
9195                    .register_candidate_with_control_ops(
9196                        initial.clone(),
9197                        PROTOCOL_VERSION,
9198                        conn,
9199                        module_baseline_control_ops(),
9200                    )
9201                    .unwrap();
9202                handler
9203                    .forwarding
9204                    .register_candidate_module_connection(
9205                        conn,
9206                        "squatter".to_string(),
9207                        PROTOCOL_VERSION,
9208                        manifest_concurrency(&initial),
9209                        ctx.egress.clone(),
9210                    )
9211                    .unwrap();
9212            } else {
9213                hello_via_sink(
9214                    &handler,
9215                    &ctx,
9216                    &mut rx,
9217                    hello_frame_with_manifest(initial.clone(), 1),
9218                )
9219                .await;
9220            }
9221            let response = handler
9222                .handle_control_frame(
9223                    &ctx,
9224                    catalog_update_with_capabilities_frame(
9225                        2,
9226                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
9227                            .capabilities
9228                            .unwrap(),
9229                    ),
9230                )
9231                .await
9232                .unwrap();
9233            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9234            assert_eq!(
9235                registry
9236                    .get_module_by_connection(conn)
9237                    .unwrap()
9238                    .unwrap()
9239                    .manifest,
9240                initial
9241            );
9242        }
9243    }
9244
9245    #[test]
9246    fn server_describe_surfaces_required_capability_verdict_fields() {
9247        let registry = Arc::new(Registry::default());
9248        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9249            [
9250                ("consumer".to_string(), true),
9251                ("provider".to_string(), false),
9252            ],
9253            BTreeMap::new(),
9254        );
9255        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9256        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9257            provides: Vec::new(),
9258            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9259                capability: "credentials-provider/v1".to_string(),
9260                need: subc_protocol::manifest::CapabilityNeed::Required,
9261            }],
9262            must_never_reach: Vec::new(),
9263        });
9264        let hello = Frame::build(
9265            FrameType::Hello,
9266            control_flags(),
9267            0,
9268            0,
9269            78,
9270            serde_json::to_vec(&ModuleHelloBody {
9271                manifest: consumer,
9272                protocol_ver: PROTOCOL_VERSION,
9273                control_ops: None,
9274                launch_nonce: None,
9275            })
9276            .expect("HELLO serializes"),
9277        )
9278        .expect("HELLO frame builds");
9279        handler
9280            .handle_control(ConnectionId::new(78), hello)
9281            .expect("consumer registers");
9282        let describe = Frame::build(
9283            FrameType::Request,
9284            control_flags(),
9285            0,
9286            0,
9287            79,
9288            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9289                .expect("request serializes"),
9290        )
9291        .expect("describe frame builds");
9292        let response = handler
9293            .handle_server_describe(describe)
9294            .expect("server.describe succeeds");
9295        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9296        let requirement = &rendered["capability_requirements"][0];
9297        assert_eq!(requirement["consumer"], "consumer");
9298        assert_eq!(requirement["verdict"], "never_provided");
9299        assert_eq!(requirement["episode_seq"], 1);
9300        assert_eq!(requirement["config_satisfiable"], false);
9301        assert_eq!(requirement["runtime_available"], false);
9302        assert!(requirement["detail"]
9303            .as_str()
9304            .expect("detail string")
9305            .contains("credentials-provider/v1"));
9306    }
9307
9308    /// Register a consumer that requires `project-identity/v1` and the provider
9309    /// of it, optionally mark daemon shutdown as begun, then deregister the
9310    /// provider as its connection closes. Returns the `capability.requirement`
9311    /// events that deregistration logged and the verdict the evaluator then
9312    /// holds for the requirement (the one `route.open` refusals read).
9313    fn deregister_required_provider(
9314        daemon_shutdown_begun: bool,
9315    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9316        const CAPABILITY: &str = "project-identity/v1";
9317        let registry = Arc::new(Registry::default());
9318        let daemon_shutdown = DaemonShutdownFlag::default();
9319        let handler = ControlHandler::new(Arc::clone(&registry))
9320            .with_capability_config(
9321                [
9322                    ("consumer".to_string(), true),
9323                    ("provider".to_string(), true),
9324                ],
9325                BTreeMap::new(),
9326            )
9327            .with_daemon_shutdown_flag(daemon_shutdown.clone());
9328        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9329        consumer.capabilities = Some(CapabilityDeclarations {
9330            provides: Vec::new(),
9331            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9332                capability: CAPABILITY.to_string(),
9333                need: subc_protocol::manifest::CapabilityNeed::Required,
9334            }],
9335            must_never_reach: Vec::new(),
9336        });
9337        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9338        let provider_conn = ConnectionId::new(182);
9339        for (conn, manifest, corr) in [
9340            (ConnectionId::new(181), consumer, 181),
9341            (provider_conn, provider, 182),
9342        ] {
9343            let hello = handler
9344                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9345                .expect("module registers");
9346            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9347        }
9348        assert_eq!(
9349            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9350            Some(CapabilityVerdict::Provided),
9351            "the registered provider must satisfy the requirement first"
9352        );
9353        if daemon_shutdown_begun {
9354            daemon_shutdown.set_for_test();
9355        }
9356
9357        let capture = EventCapture::default();
9358        let _subscriber =
9359            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9360        handler
9361            .cleanup_connection(provider_conn)
9362            .expect("provider deregisters");
9363        let events = capture
9364            .events()
9365            .into_iter()
9366            .filter(|event| {
9367                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9368            })
9369            .collect();
9370        (
9371            events,
9372            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9373        )
9374    }
9375
9376    #[test]
9377    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9378        let (events, verdict) = deregister_required_provider(true);
9379        assert_eq!(
9380            verdict,
9381            Some(CapabilityVerdict::NeverProvided),
9382            "shutdown must not change the verdict route.open refusals read"
9383        );
9384        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9385        let event = &events[0];
9386        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9387        assert_eq!(
9388            event.fields.get("cause").map(String::as_str),
9389            Some("\"daemon_shutdown\""),
9390            "{event:?}"
9391        );
9392        assert_eq!(
9393            event.fields.get("verdict").map(String::as_str),
9394            Some("\"never_provided\""),
9395            "{event:?}"
9396        );
9397    }
9398
9399    #[test]
9400    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9401        let (events, verdict) = deregister_required_provider(false);
9402        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9403        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9404        let event = &events[0];
9405        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9406        assert_eq!(event.fields.get("cause"), None, "{event:?}");
9407        assert_eq!(
9408            event.fields.get("verdict").map(String::as_str),
9409            Some("\"never_provided\""),
9410            "{event:?}"
9411        );
9412    }
9413
9414    #[test]
9415    fn catalog_list_omits_capabilities_for_legacy_manifest() {
9416        let registry = Arc::new(Registry::default());
9417        let handler = ControlHandler::new(Arc::clone(&registry));
9418        let hello = handler
9419            .handle_control(
9420                ConnectionId::new(101),
9421                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9422            )
9423            .expect("legacy HELLO registers");
9424        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9425
9426        let request = Frame::build(
9427            FrameType::Request,
9428            control_flags(),
9429            0,
9430            0,
9431            102,
9432            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9433                .expect("catalog request serializes"),
9434        )
9435        .expect("catalog request frame builds");
9436        let response = handler
9437            .handle_catalog_list(request, None)
9438            .expect("catalog list succeeds");
9439        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9440        assert!(
9441            body["modules"][0].get("capabilities").is_none(),
9442            "legacy manifest must retain an absent capabilities field on catalog.list"
9443        );
9444    }
9445
9446    #[test]
9447    fn hello_ack_omits_storage_when_no_storage_config() {
9448        let registry = Arc::new(Registry::default());
9449        let handler = ControlHandler::new(Arc::clone(&registry));
9450        let responses = handler
9451            .handle_control(
9452                ConnectionId::new(1),
9453                hello_frame("aft", PROTOCOL_VERSION, 7),
9454            )
9455            .unwrap();
9456        let ack = parse_ack(&responses[0]);
9457        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9458        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9459    }
9460
9461    #[tokio::test]
9462    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9463        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9464        let registry = Arc::new(Registry::default());
9465        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
9466        let responses = handler
9467            .handle_control(
9468                ConnectionId::new(1),
9469                hello_frame("aft", PROTOCOL_VERSION, 7),
9470            )
9471            .unwrap();
9472        let ack = parse_ack(&responses[0]);
9473        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9474
9475        let described = handler
9476            .handle_control_frame(
9477                &route_ctx(ConnectionId::new(2)).0,
9478                Frame::build(
9479                    FrameType::Request,
9480                    control_flags(),
9481                    0,
9482                    0,
9483                    9,
9484                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9485                )
9486                .unwrap(),
9487            )
9488            .await
9489            .unwrap();
9490        let ClientControlResponse::ServerDescribe { machine_id, .. } =
9491            serde_json::from_slice(&described[0].body).unwrap()
9492        else {
9493            panic!("server.describe answered with another shape");
9494        };
9495        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9496    }
9497
9498    #[test]
9499    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9500        // With a central sqlite storage policy, each registering module gets its
9501        // own resolved descriptor in HELLO_ACK, keyed by its module id.
9502        let registry = Arc::new(Registry::default());
9503        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
9504            crate::daemon_config::StorageConfig::Sqlite {
9505                data_home: std::path::PathBuf::from("/data"),
9506            },
9507        ));
9508
9509        let responses = handler
9510            .handle_control(
9511                ConnectionId::new(1),
9512                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9513            )
9514            .unwrap();
9515        let ack = parse_ack(&responses[0]);
9516        assert_eq!(
9517            ack.storage,
9518            Some(serde_json::json!({
9519                "module_id": "alfonso-routing",
9520                "storage_namespace": "default",
9521                "isolation": { "kind": "module" },
9522                "backend": {
9523                    "backend": "sqlite",
9524                    "path": "/data/cortexkit/alfonso-routing/store.db"
9525                }
9526            })),
9527            "the delivered descriptor is the module's own sqlite store path"
9528        );
9529    }
9530
9531    #[test]
9532    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9533        let registry = Arc::new(Registry::default());
9534        let handler = ControlHandler::new(Arc::clone(&registry));
9535        let conn = ConnectionId::new(1);
9536        let responses = handler
9537            .handle_control(
9538                conn,
9539                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9540            )
9541            .unwrap();
9542        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9543        let registration = registry.get_module("aft").unwrap().unwrap();
9544        assert_eq!(registration.control_ops, module_baseline_control_ops());
9545
9546        let frame =
9547            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9548        assert!(handler
9549            .guard_module_control_op(&frame, "aft", "route.bind")
9550            .unwrap()
9551            .is_none());
9552        let error = handler
9553            .guard_module_control_op(&frame, "aft", "test.synthetic")
9554            .unwrap()
9555            .expect("synthetic ungranted op should be rejected");
9556        assert_eq!(error.header.ty, FrameType::Error);
9557        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9558    }
9559
9560    #[test]
9561    fn hello_control_ops_some_adds_optional_grants() {
9562        let registry = Arc::new(Registry::default());
9563        let handler = ControlHandler::new(Arc::clone(&registry));
9564        handler
9565            .handle_control(
9566                ConnectionId::new(1),
9567                hello_frame_with_control_ops(
9568                    "aft",
9569                    PROTOCOL_VERSION,
9570                    7,
9571                    Some(vec![
9572                        "future.synthetic".to_string(),
9573                        "route.bind".to_string(),
9574                    ]),
9575                ),
9576            )
9577            .unwrap();
9578        let registration = registry.get_module("aft").unwrap().unwrap();
9579        assert_eq!(
9580            registration.control_ops,
9581            vec![
9582                "route.bind".to_string(),
9583                "route.status".to_string(),
9584                "future.synthetic".to_string(),
9585            ]
9586        );
9587        let frame =
9588            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9589        assert!(handler
9590            .guard_module_control_op(&frame, "aft", "future.synthetic")
9591            .unwrap()
9592            .is_none());
9593    }
9594
9595    #[tokio::test]
9596    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9597        let registry = Arc::new(Registry::default());
9598        let forwarding = Arc::new(ForwardingTable::default());
9599        let handler =
9600            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9601        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9602        hello_via_sink(
9603            &handler,
9604            &module_ctx,
9605            &mut module_rx,
9606            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9607        )
9608        .await;
9609
9610        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9611        let responses = handler
9612            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9613            .await
9614            .unwrap();
9615        assert_eq!(responses.len(), 1);
9616        assert_eq!(responses[0].header.ty, FrameType::Error);
9617        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9618        assert!(module_rx.try_recv().is_err());
9619    }
9620
9621    #[tokio::test]
9622    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9623        let registry = Arc::new(Registry::default());
9624        let forwarding = Arc::new(ForwardingTable::default());
9625        let handler =
9626            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9627        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9628        hello_via_sink(
9629            &handler,
9630            &module_ctx,
9631            &mut module_rx,
9632            hello_frame_with_control_ops(
9633                "aft",
9634                PROTOCOL_VERSION,
9635                7,
9636                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9637            ),
9638        )
9639        .await;
9640
9641        let project_root = unique_project_root("demux");
9642        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9643        let route_handler = handler.clone();
9644        let route_task = tokio::spawn(async move {
9645            route_handler
9646                .handle_control_frame(
9647                    &route_client_ctx,
9648                    route_open_frame(100, "aft", project_root),
9649                )
9650                .await
9651                .unwrap()
9652        });
9653        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9654            .await
9655            .unwrap()
9656            .unwrap();
9657        assert!(matches!(
9658            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9659            ModuleControlRequest::RouteBind { .. }
9660        ));
9661
9662        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9663        let health_handler = handler.clone();
9664        let health_task = tokio::spawn(async move {
9665            health_handler
9666                .handle_control_frame(
9667                    &health_client_ctx,
9668                    supervisor_health_probe_frame(101, "aft"),
9669                )
9670                .await
9671                .unwrap()
9672        });
9673        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9674            .await
9675            .unwrap()
9676            .unwrap();
9677        assert_eq!(
9678            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9679            ModuleControlRequest::HealthCheck {}
9680        );
9681
9682        handler
9683            .handle_control_frame(
9684                &module_ctx,
9685                health_response(health_frame.header.corr, HealthStatus::Degraded),
9686            )
9687            .await
9688            .unwrap();
9689        let health_response = health_task.await.unwrap();
9690        assert_eq!(health_response.len(), 1);
9691        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9692            ClientControlResponse::SupervisorHealthProbe {
9693                module_id,
9694                status,
9695                detail,
9696                metrics,
9697            } => {
9698                assert_eq!(module_id, "aft");
9699                assert_eq!(status, HealthStatus::Degraded);
9700                assert_eq!(detail.as_deref(), Some("warming"));
9701                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9702            }
9703            other => panic!("unexpected health response: {other:?}"),
9704        }
9705
9706        handler
9707            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9708            .await
9709            .unwrap();
9710        let route_response = route_task.await.unwrap();
9711        assert!(route_response.is_empty());
9712        let published = route_client_rx.recv().await.unwrap();
9713        assert!(matches!(
9714            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9715            ClientControlResponse::RouteOpen { .. }
9716        ));
9717    }
9718
9719    /// Start one `route.open` on `client_connection` and return its still-running
9720    /// handler task together with the `route.bind` the module received for it.
9721    /// The handler blocks until the module answers, so it has to run as a task
9722    /// while the test drives the module side.
9723    async fn relay_route_open(
9724        handler: &ControlHandler,
9725        client_connection: ConnectionId,
9726        client_egress: &FrameSink,
9727        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9728        corr: u64,
9729        module_id: &str,
9730        project_root_label: &str,
9731    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9732        let ctx = RouteCtx {
9733            connection_id: client_connection,
9734            egress: client_egress.clone(),
9735        };
9736        let handler = handler.clone();
9737        let project_root = unique_project_root(project_root_label);
9738        let module_id = module_id.to_string();
9739        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9740        let task = tokio::spawn(async move {
9741            let _guard = tracing::dispatcher::set_default(&dispatch);
9742            handler
9743                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9744                .await
9745                .unwrap()
9746        });
9747        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9748            .await
9749            .expect("module receives the relayed route.bind")
9750            .expect("module egress is open");
9751        (task, bind.frame)
9752    }
9753
9754    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9755        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9756            ModuleControlRequest::RouteBind {
9757                route_channel,
9758                epoch,
9759                ..
9760            } => (route_channel, epoch),
9761            other => panic!("expected a route.bind request, got {other:?}"),
9762        }
9763    }
9764
9765    fn published_route(frame: &Frame) -> (u16, u32) {
9766        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9767            ClientControlResponse::RouteOpen {
9768                route_channel,
9769                route_epoch,
9770            } => (route_channel, route_epoch),
9771            other => panic!("expected a route.open response, got {other:?}"),
9772        }
9773    }
9774
9775    /// Reproduction of a production outage. A client had `route.open`s in
9776    /// flight to a module and was already marked closing -- its egress had refused a
9777    /// module frame, so the daemon asked its connection to end -- while its sink
9778    /// was still open. When the module acked those binds, the daemon refused to
9779    /// commit a route for a closing client, and that refusal was returned from
9780    /// the MODULE connection's frame handler, where a router error that has no
9781    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
9782    /// the supervisor correctly did not respawn a clean exit, and every seat lost
9783    /// its tools for hours -- one client's teardown took down a connection
9784    /// carrying ~170 other routes.
9785    ///
9786    /// The window is opened here by calling the production path that opens it
9787    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
9788    /// state that matters is "in `closing_connections`, sink still open, relay
9789    /// still pending", and it lasts only from the close request until the
9790    /// connection loop reacts to it; a socket-level test can flood a client into
9791    /// that escalation but cannot pin the module's ack inside the window. Closing
9792    /// the socket instead takes the other path entirely -- connection teardown
9793    /// removes the pending relay under the same lock, so the ack finds nothing.
9794    #[tokio::test]
9795    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9796        let registry = Arc::new(Registry::default());
9797        let forwarding = Arc::new(ForwardingTable::default());
9798        let handler =
9799            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9800
9801        let module_connection = ConnectionId::new(30);
9802        let (module_ctx, mut module_rx) = route_ctx(module_connection);
9803        hello_via_sink(
9804            &handler,
9805            &module_ctx,
9806            &mut module_rx,
9807            hello_frame("aft", PROTOCOL_VERSION, 7),
9808        )
9809        .await;
9810
9811        let dying_client = ConnectionId::new(31);
9812        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9813
9814        // A published route on the dying client. The escalation below only marks
9815        // a connection closing for a route it has already published.
9816        let (first_task, first_bind) = relay_route_open(
9817            &handler,
9818            dying_client,
9819            &dying_ctx.egress,
9820            &mut module_rx,
9821            100,
9822            "aft",
9823            "closing-first",
9824        )
9825        .await;
9826        handler
9827            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9828            .await
9829            .unwrap();
9830        assert!(first_task.await.unwrap().is_empty());
9831        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9832
9833        // A second route.open from the same client, relayed and awaiting its ack.
9834        let (second_task, second_bind) = relay_route_open(
9835            &handler,
9836            dying_client,
9837            &dying_ctx.egress,
9838            &mut module_rx,
9839            101,
9840            "aft",
9841            "closing-second",
9842        )
9843        .await;
9844        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9845
9846        // The window: the client is closing, its sink is still open, and its
9847        // second bind is still pending.
9848        assert!(forwarding
9849            .escalate_client_delivery_failure(
9850                dying_client,
9851                first_channel,
9852                first_epoch,
9853                CloseReason::new(
9854                    "module_to_client_delivery_failed",
9855                    "client egress refused a module frame",
9856                ),
9857                crate::forwarding::UndeliveredFrame {
9858                    module_id: None,
9859                    sink: &dying_ctx.egress,
9860                },
9861            )
9862            .unwrap());
9863        assert!(!dying_ctx.egress.is_closed());
9864
9865        // The frame that used to end the module connection.
9866        let ack = handler
9867            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9868            .await;
9869        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9870        if module_loop_error.is_some() {
9871            // What the server's connection loop does with a router error that has
9872            // no ERROR-frame translation: end the connection, which releases the
9873            // module's registration and every route on it.
9874            handler.cleanup_connection(module_connection).unwrap();
9875        }
9876        // Read the module's next frame before opening the co-tenant's route, so
9877        // the GOODBYE assertion below is about THIS ack and not about later
9878        // traffic. `None` means the module was told nothing.
9879        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9880            .await
9881            .ok()
9882            .flatten();
9883
9884        // 1. The module connection is still registered.
9885        assert!(
9886            registry
9887                .get_module_by_connection(module_connection)
9888                .unwrap()
9889                .is_some(),
9890            "one client's closing connection ended the shared module connection: \
9891             {module_loop_error:?}"
9892        );
9893        // ...and still serving: another client can open and use a route on it.
9894        let cotenant = ConnectionId::new(32);
9895        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9896        let (cotenant_task, cotenant_bind) = relay_route_open(
9897            &handler,
9898            cotenant,
9899            &cotenant_ctx.egress,
9900            &mut module_rx,
9901            102,
9902            "aft",
9903            "closing-cotenant",
9904        )
9905        .await;
9906        handler
9907            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9908            .await
9909            .unwrap();
9910        assert!(cotenant_task.await.unwrap().is_empty());
9911        let (cotenant_channel, cotenant_epoch) =
9912            published_route(&cotenant_rx.recv().await.unwrap());
9913        assert!(matches!(
9914            forwarding
9915                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9916                .unwrap(),
9917            DataRoute::Client(DataRouteState::Bound(_))
9918        ));
9919
9920        // 2. The module was told to drop the binding it created for the route
9921        //    that will never be published.
9922        let goodbye = post_ack_module_frame
9923            .expect("module receives a GOODBYE for the abandoned route channel");
9924        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9925        assert_eq!(goodbye.header.channel, abandoned_channel);
9926        assert_eq!(goodbye.header.epoch, abandoned_epoch);
9927
9928        // 3. The dying client received nothing: no route was ever published to
9929        //    it. Its route.open is answered as unavailable, which the connection
9930        //    loop would write to a socket that is already going away.
9931        assert!(dying_rx.try_recv().is_err());
9932        let second_response = second_task.await.unwrap();
9933        assert_eq!(second_response.len(), 1);
9934        assert_eq!(
9935            parse_error(&second_response[0])["code"],
9936            "target_unavailable"
9937        );
9938    }
9939
9940    /// One `catalog.list` as a client sees it: the generation and the listed
9941    /// module ids with their readiness.
9942    async fn catalog_snapshot(
9943        handler: &ControlHandler,
9944        client_ctx: &RouteCtx,
9945        corr: u64,
9946    ) -> (u64, Vec<(String, bool)>) {
9947        let body =
9948            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9949        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9950        let reply = handler
9951            .handle_control_frame(client_ctx, request)
9952            .await
9953            .unwrap()
9954            .pop()
9955            .unwrap();
9956        let ClientControlResponse::CatalogList {
9957            generation,
9958            modules,
9959            ..
9960        } = serde_json::from_slice(&reply.body).unwrap()
9961        else {
9962            panic!("not a catalog.list reply");
9963        };
9964        (
9965            generation,
9966            modules
9967                .into_iter()
9968                .map(|entry| (entry.module_id, entry.ready))
9969                .collect(),
9970        )
9971    }
9972
9973    /// A handler whose HELLO for module "aft" on connection 40 is parked
9974    /// between reserving its registry entry and installing its forwarding
9975    /// endpoint. Returns the handler, the two-party barrier (wait once to know
9976    /// the HELLO is parked, once more to release it), the HELLO's thread, and
9977    /// the module's context and receiver.
9978    fn hello_held_before_forwarding() -> (
9979        ControlHandler,
9980        Arc<std::sync::Barrier>,
9981        std::thread::JoinHandle<Vec<Frame>>,
9982        RouteCtx,
9983        mpsc::Receiver<crate::router::OutboundFrame>,
9984    ) {
9985        let hold = Arc::new(std::sync::Barrier::new(2));
9986        let handler = ControlHandler::with_forwarding(
9987            Arc::new(Registry::default()),
9988            Arc::new(ForwardingTable::default()),
9989        )
9990        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9991        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9992        let hello = {
9993            let handler = handler.clone();
9994            let module_ctx = module_ctx.clone();
9995            std::thread::spawn(move || {
9996                handler
9997                    .handle_hello(
9998                        module_ctx.connection_id,
9999                        Some(module_ctx.egress.clone()),
10000                        hello_frame("aft", PROTOCOL_VERSION, 7),
10001                    )
10002                    .unwrap()
10003            })
10004        };
10005        (handler, hold, hello, module_ctx, module_rx)
10006    }
10007
10008    /// While a HELLO has reserved its registry entry but its forwarding
10009    /// endpoint is not installed yet, `route.open` to the module can only be
10010    /// refused. `catalog.list` must not list the module in that window: a
10011    /// client that opens a route as soon as it sees a module listed and ready
10012    /// would be refused for a module the daemon had just advertised. The
10013    /// refusal in the window stays the retryable `target_unavailable`, never
10014    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10015    /// Once the HELLO completes, the module is listed and a route to it opens.
10016    #[tokio::test]
10017    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10018        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10019        let client = ConnectionId::new(41);
10020        let (client_ctx, mut client_rx) = route_ctx(client);
10021
10022        hold.wait();
10023        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10024        assert!(handler.registry.get_module("aft").unwrap().is_some());
10025        assert!(!handler
10026            .forwarding
10027            .has_live_module_connection("aft")
10028            .unwrap());
10029        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10030        assert!(
10031            listed.is_empty(),
10032            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10033        );
10034        let refused = handler
10035            .handle_control_frame(
10036                &client_ctx,
10037                route_open_frame(301, "aft", unique_project_root("listed-window")),
10038            )
10039            .await
10040            .unwrap();
10041        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10042        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10043        let duplicate = handler
10044            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10045            .await
10046            .unwrap();
10047        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10048
10049        hold.wait();
10050        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10051        let ack = module_rx.try_recv().unwrap().frame;
10052        assert_eq!(ack.header.ty, FrameType::HelloAck);
10053        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10054        assert_eq!(listed, vec![("aft".to_string(), true)]);
10055
10056        let (open_task, bind) = relay_route_open(
10057            &handler,
10058            client,
10059            &client_ctx.egress,
10060            &mut module_rx,
10061            303,
10062            "aft",
10063            "listed-after",
10064        )
10065        .await;
10066        assert!(handler
10067            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10068            .await
10069            .unwrap()
10070            .is_empty());
10071        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10072        published_route(&client_rx.try_recv().unwrap().frame);
10073    }
10074
10075    /// A catalog watcher keys on the generation: when it changes, it re-reads
10076    /// the listing. So the generation must advance exactly when the module
10077    /// appears. If it advanced while the module was still hidden, a watcher
10078    /// would read the new generation without the module, and the module would
10079    /// later appear under that same generation with nothing telling the
10080    /// watcher to look again.
10081    #[tokio::test]
10082    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10083        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10084        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10085        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10086
10087        hold.wait();
10088        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10089        assert!(listed.is_empty());
10090        assert_eq!(
10091            during, before,
10092            "the generation moved while the module was not listed"
10093        );
10094
10095        hold.wait();
10096        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10097        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10098        assert_eq!(listed, vec![("aft".to_string(), true)]);
10099        assert_eq!(
10100            after,
10101            before + 1,
10102            "the module appeared without exactly one generation bump"
10103        );
10104    }
10105
10106    /// A HELLO whose forwarding endpoint cannot be installed (here the
10107    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
10108    /// refused and leaves nothing behind: no registry entry, no catalog entry,
10109    /// and the catalog generation where it was.
10110    #[tokio::test]
10111    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10112        let registry = Arc::new(Registry::default());
10113        let handler = ControlHandler::with_forwarding(
10114            Arc::clone(&registry),
10115            Arc::new(ForwardingTable::default()),
10116        );
10117        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10118        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10119        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10120        drop(module_rx);
10121
10122        let replies = handler
10123            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10124            .await
10125            .unwrap();
10126        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10127        assert!(registry.get_module("aft").unwrap().is_none());
10128        assert!(registry
10129            .get_module_by_connection(module_ctx.connection_id)
10130            .unwrap()
10131            .is_none());
10132        assert_eq!(
10133            catalog_snapshot(&handler, &client_ctx, 301).await,
10134            (before, Vec::new())
10135        );
10136    }
10137
10138    /// A handler whose `route.open` parks after queueing the route.bind and
10139    /// before marking it enqueued, with module "aft" registered on connection
10140    /// 40. Returns the two-party barrier that parks and releases it.
10141    async fn route_open_held_after_bind_send() -> (
10142        ControlHandler,
10143        Arc<tokio::sync::Barrier>,
10144        RouteCtx,
10145        mpsc::Receiver<crate::router::OutboundFrame>,
10146    ) {
10147        let hold = Arc::new(tokio::sync::Barrier::new(2));
10148        let handler = ControlHandler::with_forwarding(
10149            Arc::new(Registry::default()),
10150            Arc::new(ForwardingTable::default()),
10151        )
10152        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10153        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10154        hello_via_sink(
10155            &handler,
10156            &module_ctx,
10157            &mut module_rx,
10158            hello_frame("aft", PROTOCOL_VERSION, 7),
10159        )
10160        .await;
10161        (handler, hold, module_ctx, module_rx)
10162    }
10163
10164    /// A fast module acks the route.bind, and the reader commits the route,
10165    /// before `route.open` has marked the relay enqueued. The daemon used to
10166    /// read the missing relay as an abandoned bind and send the module a
10167    /// GOODBYE for the route it had just committed: the client was told its
10168    /// route was open, the module unbound the channel, and the client's first
10169    /// request was dropped on the module side as unbound.
10170    #[tokio::test]
10171    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10172        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10173        let client = ConnectionId::new(41);
10174        let (client_ctx, mut client_rx) = route_ctx(client);
10175
10176        let (open_task, bind) = relay_route_open(
10177            &handler,
10178            client,
10179            &client_ctx.egress,
10180            &mut module_rx,
10181            200,
10182            "aft",
10183            "acked-before-mark",
10184        )
10185        .await;
10186        let (module_channel, module_epoch) = route_bind_channel(&bind);
10187        // route.open is now parked between queueing the bind and marking it.
10188        hold.wait().await;
10189
10190        // The module answers inside that window, and the route commits.
10191        assert!(handler
10192            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10193            .await
10194            .unwrap()
10195            .is_empty());
10196        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10197
10198        hold.wait().await;
10199        assert!(
10200            open_task.await.unwrap().is_empty(),
10201            "route.open answers accepted"
10202        );
10203
10204        // Every frame route.open could have sent is already queued, because it
10205        // has returned; the module must have been sent nothing.
10206        match module_rx.try_recv() {
10207            Err(mpsc::error::TryRecvError::Empty) => {}
10208            Ok(outbound) => panic!(
10209                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10210                outbound.frame.header.ty,
10211                outbound.frame.header.channel,
10212                outbound.frame.header.epoch
10213            ),
10214            Err(err) => panic!("module egress closed: {err}"),
10215        }
10216
10217        // The client's first request on the route is the next thing the module
10218        // reads, on the binding it just created.
10219        let router = Router::with_control_handler(Arc::new(handler.clone()));
10220        let request = Frame::build(
10221            FrameType::Request,
10222            Flags::new(true, Priority::Interactive, false),
10223            client_channel,
10224            client_epoch,
10225            201,
10226            b"first request".to_vec(),
10227        )
10228        .unwrap();
10229        router
10230            .route_for_connection(&client_ctx, request)
10231            .await
10232            .unwrap();
10233        let delivered = module_rx.try_recv().unwrap().frame;
10234        assert_eq!(delivered.header.ty, FrameType::Request);
10235        assert_eq!(
10236            (delivered.header.channel, delivered.header.epoch),
10237            (module_channel, module_epoch)
10238        );
10239        assert_eq!(delivered.body, b"first request");
10240    }
10241
10242    /// The client goes away while `route.open` is still queueing the bind, so
10243    /// the relay is dropped with no module reply. The teardown cannot send the
10244    /// GOODBYE itself (it could reach the module ahead of the bind), so
10245    /// `route.open` must send it once the bind is queued; otherwise the module
10246    /// keeps a binding the daemon will never commit.
10247    #[tokio::test]
10248    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10249        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10250        let client = ConnectionId::new(42);
10251        let (client_ctx, _client_rx) = route_ctx(client);
10252
10253        let (open_task, bind) = relay_route_open(
10254            &handler,
10255            client,
10256            &client_ctx.egress,
10257            &mut module_rx,
10258            300,
10259            "aft",
10260            "released-before-mark",
10261        )
10262        .await;
10263        let (module_channel, module_epoch) = route_bind_channel(&bind);
10264        hold.wait().await;
10265
10266        handler.cleanup_connection(client).unwrap();
10267        assert!(
10268            module_rx.try_recv().is_err(),
10269            "client teardown must not GOODBYE a bind route.open is still queueing"
10270        );
10271
10272        hold.wait().await;
10273        let response = open_task.await.unwrap();
10274        assert_eq!(response.len(), 1);
10275        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10276
10277        let goodbye = module_rx
10278            .try_recv()
10279            .expect("module is told to drop the binding the daemon abandoned")
10280            .frame;
10281        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10282        assert_eq!(
10283            (goodbye.header.channel, goodbye.header.epoch),
10284            (module_channel, module_epoch)
10285        );
10286        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10287    }
10288
10289    /// The module acks inside the same window, but the client's egress closed
10290    /// first, so the reader refuses to commit. The ack proves the module holds
10291    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
10292    /// send a second one.
10293    #[tokio::test]
10294    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10295        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10296        let client = ConnectionId::new(43);
10297        let (client_ctx, client_rx) = route_ctx(client);
10298
10299        let (open_task, bind) = relay_route_open(
10300            &handler,
10301            client,
10302            &client_ctx.egress,
10303            &mut module_rx,
10304            400,
10305            "aft",
10306            "gone-client-before-mark",
10307        )
10308        .await;
10309        let (module_channel, module_epoch) = route_bind_channel(&bind);
10310        hold.wait().await;
10311
10312        drop(client_rx);
10313        assert!(client_ctx.egress.is_closed());
10314        handler
10315            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10316            .await
10317            .unwrap();
10318
10319        hold.wait().await;
10320        let response = open_task.await.unwrap();
10321        assert_eq!(response.len(), 1);
10322        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10323
10324        let goodbye = module_rx
10325            .try_recv()
10326            .expect("module is told to drop the binding it acked")
10327            .frame;
10328        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10329        assert_eq!(
10330            (goodbye.header.channel, goodbye.header.epoch),
10331            (module_channel, module_epoch)
10332        );
10333        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10334        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10335    }
10336
10337    /// The fence at the module-loop boundary, stated as its own contract: which
10338    /// forwarding failures are allowed to end the module connection that is being
10339    /// served. A `ConnectionClosing` naming some client is about that client, and
10340    /// a module connection is shared; the same error naming the module's own
10341    /// connection is about this connection and must stay fatal, as must failures
10342    /// that are about the forwarding table itself.
10343    #[test]
10344    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10345        let handler = ControlHandler::default();
10346        let module_connection = ConnectionId::new(30);
10347        let client_connection = ConnectionId::new(31);
10348
10349        handler
10350            .refuse_to_end_module_connection_for_a_client(
10351                module_connection,
10352                77,
10353                ForwardingError::ConnectionClosing {
10354                    connection_id: client_connection,
10355                },
10356            )
10357            .expect("a closing client must never end the module connection");
10358
10359        assert!(matches!(
10360            handler.refuse_to_end_module_connection_for_a_client(
10361                module_connection,
10362                78,
10363                ForwardingError::ConnectionClosing {
10364                    connection_id: module_connection,
10365                },
10366            ),
10367            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10368                connection_id
10369            })) if connection_id == module_connection
10370        ));
10371        assert!(matches!(
10372            handler.refuse_to_end_module_connection_for_a_client(
10373                module_connection,
10374                79,
10375                ForwardingError::Poisoned,
10376            ),
10377            Err(RouterError::Forwarding(ForwardingError::Poisoned))
10378        ));
10379        assert!(matches!(
10380            handler.refuse_to_end_module_connection_for_a_client(
10381                module_connection,
10382                80,
10383                ForwardingError::StaleModuleEndpoint,
10384            ),
10385            Err(RouterError::Forwarding(
10386                ForwardingError::StaleModuleEndpoint
10387            ))
10388        ));
10389    }
10390
10391    /// The spawn-attestation guard is what stops a connected module from claiming
10392    /// another module's identity and being stamped `Reserved` for it. Every other
10393    /// test that supplies a consumer_identity supplies a CORRECT one, because a
10394    /// correct one is what the rest of the flow needs -- so the guard's rejection
10395    /// branch was never the subject of an assertion, only its acceptance branch.
10396    ///
10397    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
10398    /// whole subc-core library suite green; only the forwarding integration tests
10399    /// notice, and they notice for unrelated reasons. This test exists so the
10400    /// refusal itself is asserted where the guard lives: it fails if the identity
10401    /// check stops refusing, which is the direction that matters, since a guard
10402    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
10403    #[tokio::test]
10404    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10405        let registry = Arc::new(Registry::default());
10406        let forwarding = Arc::new(ForwardingTable::default());
10407        let supervisor = SupervisorHandle::new();
10408        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10409        let handler =
10410            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10411                .with_supervisor(supervisor);
10412
10413        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10414        hello_via_sink(
10415            &handler,
10416            &target_ctx,
10417            &mut target_rx,
10418            hello_frame("target", PROTOCOL_VERSION, 1),
10419        )
10420        .await;
10421
10422        // A real supervised module id presenting the wrong nonce. This is the
10423        // impersonation case: the attacker knows a privileged module_id, which is
10424        // public, and guesses at the nonce, which is not.
10425        let wrong_nonce = handler
10426            .handle_control_frame(
10427                &route_ctx(ConnectionId::new(91)).0,
10428                route_open_frame_with_admission_facts(
10429                    20,
10430                    "target",
10431                    unique_project_root("admission-facts"),
10432                    Some(subc_control::ConsumerIdentity {
10433                        module_id: "fed".to_string(),
10434                        launch_nonce: "not-the-real-nonce".to_string(),
10435                    }),
10436                    None,
10437                ),
10438            )
10439            .await
10440            .unwrap();
10441        assert_eq!(
10442            parse_error(&wrong_nonce[0])["code"],
10443            "bad_consumer_identity",
10444            "a mismatched launch nonce must be refused, not stamped Reserved"
10445        );
10446
10447        // A module id the supervisor never spawned at all, so no nonce exists to
10448        // compare against. An implementation that treats "no record" as "nothing
10449        // to check" fails open here while passing the case above.
10450        let never_spawned = handler
10451            .handle_control_frame(
10452                &route_ctx(ConnectionId::new(92)).0,
10453                route_open_frame_with_admission_facts(
10454                    21,
10455                    "target",
10456                    unique_project_root("admission-facts"),
10457                    Some(subc_control::ConsumerIdentity {
10458                        module_id: "never-spawned".to_string(),
10459                        launch_nonce: "any-nonce".to_string(),
10460                    }),
10461                    None,
10462                ),
10463            )
10464            .await
10465            .unwrap();
10466        assert_eq!(
10467            parse_error(&never_spawned[0])["code"],
10468            "bad_consumer_identity",
10469            "an unspawned module_id must be refused rather than accepted for lack of a record"
10470        );
10471    }
10472
10473    /// The refusal test above proves the guard says NO. Nothing proved it can say
10474    /// YES, and the difference is not academic: replacing the whole authorization
10475    /// with `false` -- admitting no consumer identity at all, revoking Reserved
10476    /// standing for every supervised module in the fleet -- leaves 110 of the 111
10477    /// library tests GREEN. The one that notices does so by HANGING, because it
10478    /// waits for a bind that can no longer happen.
10479    ///
10480    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
10481    /// or flaky test, invites a RETRY rather than an investigation, and the retry
10482    /// hangs too and gets blamed on the runner. So a total revocation of the
10483    /// daemon's trust grant would have shipped behind a symptom nobody attributes
10484    /// to code.
10485    ///
10486    /// The bias is structural rather than accidental. A REFUSAL looks like a
10487    /// failure someone writes a test for; a GRANT looks like the happy path. Every
10488    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
10489    /// suite for that reason, and this one is the purest case in the daemon.
10490    ///
10491    /// This test asserts the EFFECT rather than the absence of an error: the module
10492    /// receives a RouteBind and it carries `Reserved` naming the attested module.
10493    /// A guard that admitted nobody would produce no bind at all; one that admitted
10494    /// everybody would stamp the wrong principal, which the refusal test catches.
10495    #[tokio::test]
10496    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10497        let registry = Arc::new(Registry::default());
10498        let forwarding = Arc::new(ForwardingTable::default());
10499        let supervisor = SupervisorHandle::new();
10500        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10501        let handler =
10502            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10503                .with_supervisor(supervisor);
10504
10505        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10506        hello_via_sink(
10507            &handler,
10508            &target_ctx,
10509            &mut target_rx,
10510            hello_frame("target", PROTOCOL_VERSION, 1),
10511        )
10512        .await;
10513
10514        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10515        let route_handler = handler.clone();
10516        let route_task = tokio::spawn(async move {
10517            route_handler
10518                .handle_control_frame(
10519                    &client_ctx,
10520                    route_open_frame_with_admission_facts(
10521                        30,
10522                        "target",
10523                        unique_project_root("admission-facts"),
10524                        Some(subc_control::ConsumerIdentity {
10525                            module_id: "fed".to_string(),
10526                            launch_nonce: "fed-nonce".to_string(),
10527                        }),
10528                        None,
10529                    ),
10530                )
10531                .await
10532                .unwrap()
10533        });
10534
10535        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
10536        // the very mutation it exists to catch -- a guard that admits nobody -- no
10537        // bind is ever sent, so it HUNG rather than failing. That reproduces the
10538        // exact defect being fixed: a total revocation detected only as a stalled
10539        // suite, which reads as flakiness and invites a retry. An acceptance test
10540        // that waits for an effect must bound the wait, or a red becomes a hang.
10541        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10542            .await
10543            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10544            .expect("module control channel closed before route.bind");
10545        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10546        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10547            panic!("expected route.bind")
10548        };
10549        assert_eq!(
10550            principal,
10551            Some(Principal::Reserved {
10552                module_id: "fed".to_string()
10553            }),
10554            "a correctly attested consumer must be stamped Reserved for its own id"
10555        );
10556
10557        handler
10558            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10559            .await
10560            .unwrap();
10561        assert!(route_task.await.unwrap().is_empty());
10562        assert!(
10563            matches!(
10564                serde_json::from_slice::<ClientControlResponse>(
10565                    &client_rx.recv().await.unwrap().body
10566                )
10567                .unwrap(),
10568                ClientControlResponse::RouteOpen { .. }
10569            ),
10570            "the route must actually open, not merely avoid an error"
10571        );
10572    }
10573
10574    #[tokio::test(start_paused = true)]
10575    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10576        let registry = Arc::new(Registry::default());
10577        let forwarding = Arc::new(ForwardingTable::default());
10578        let supervisor = SupervisorHandle::new();
10579        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10580        let handler =
10581            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10582                .with_supervisor(supervisor);
10583
10584        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10585        hello_via_sink(
10586            &handler,
10587            &target_ctx,
10588            &mut target_rx,
10589            hello_frame("target", PROTOCOL_VERSION, 1),
10590        )
10591        .await;
10592
10593        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10594        let direct_handler = handler.clone();
10595        let direct_open = tokio::spawn(async move {
10596            direct_handler
10597                .handle_control_frame(
10598                    &direct_ctx,
10599                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
10600                )
10601                .await
10602                .unwrap()
10603        });
10604        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10605            .await
10606            .expect("no direct route.bind within 5s")
10607            .expect("target control channel closed before direct route.bind");
10608        handler
10609            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10610            .await
10611            .unwrap();
10612        assert!(direct_open.await.unwrap().is_empty());
10613        let _ = direct_rx.recv().await.unwrap();
10614
10615        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10616        let reserved_handler = handler.clone();
10617        let reserved_open = tokio::spawn(async move {
10618            reserved_handler
10619                .handle_control_frame(
10620                    &reserved_ctx,
10621                    route_open_frame_with_admission_facts(
10622                        3,
10623                        "target",
10624                        unique_project_root("admission-facts"),
10625                        Some(ConsumerIdentity {
10626                            module_id: "fed".to_string(),
10627                            launch_nonce: "fed-nonce".to_string(),
10628                        }),
10629                        None,
10630                    ),
10631                )
10632                .await
10633                .unwrap()
10634        });
10635        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10636            .await
10637            .expect("no reserved route.bind within 5s")
10638            .expect("target control channel closed before reserved route.bind");
10639        handler
10640            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10641            .await
10642            .unwrap();
10643        assert!(reserved_open.await.unwrap().is_empty());
10644        let _ = reserved_rx.recv().await.unwrap();
10645
10646        forwarding
10647            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10648            .unwrap();
10649        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10650        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10651            module_id: Some("target".to_string()),
10652        })
10653        .unwrap();
10654        let census_frame =
10655            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10656        let response = handler
10657            .handle_control_frame(&census_ctx, census_frame)
10658            .await
10659            .unwrap()
10660            .pop()
10661            .unwrap();
10662        let actual: Value = serde_json::from_slice(&response.body).unwrap();
10663        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10664        assert!(matches!(
10665            decoded,
10666            ClientControlResponse::SupervisorRoutes { .. }
10667        ));
10668        let routes = actual["modules"][0]["routes"].as_array().unwrap();
10669        assert_eq!(routes.len(), 2);
10670        assert!(routes.iter().all(|route| route["draining"] == true));
10671        // The census carries WHY: the reason the drain was begun with, in the
10672        // route.closing vocabulary, on every draining route this drain marked.
10673        assert!(
10674            routes.iter().all(|route| route["drain_reason"] == "reload"),
10675            "draining routes must name the drain's reason: {routes:?}"
10676        );
10677        assert!(routes.iter().any(|route| {
10678            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10679        }));
10680        assert!(routes.iter().any(|route| {
10681            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10682        }));
10683
10684        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10685            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10686        if std::env::var_os("UPDATE_GOLDEN").is_some() {
10687            std::fs::write(
10688                &golden_path,
10689                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10690            )
10691            .unwrap();
10692        }
10693        let expected: Value =
10694            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10695        assert_eq!(actual, expected);
10696    }
10697
10698    async fn query_live_roots(
10699        handler: &ControlHandler,
10700        module_ctx: &RouteCtx,
10701    ) -> ModuleControlResponseToModule {
10702        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10703        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10704        let response = handler
10705            .handle_control_frame(module_ctx, frame)
10706            .await
10707            .unwrap()
10708            .pop()
10709            .unwrap();
10710        serde_json::from_slice(&response.body).unwrap()
10711    }
10712
10713    #[tokio::test(start_paused = true)]
10714    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10715        let registry = Arc::new(Registry::default());
10716        let forwarding = Arc::new(ForwardingTable::default());
10717        let handler = ControlHandler::with_forwarding(registry, forwarding);
10718        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10719        hello_via_sink(
10720            &handler,
10721            &target_ctx,
10722            &mut target_rx,
10723            hello_frame("target", PROTOCOL_VERSION, 1),
10724        )
10725        .await;
10726        let root = unique_project_root("live-roots-known");
10727        let path = ProjectRootId::from_path_allowing_missing(root.path())
10728            .unwrap()
10729            .as_path()
10730            .to_path_buf();
10731        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10732        let open_handler = handler.clone();
10733        let opened = tokio::spawn(async move {
10734            open_handler
10735                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10736                .await
10737                .unwrap()
10738        });
10739        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10740            .await
10741            .unwrap()
10742            .unwrap();
10743        handler
10744            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10745            .await
10746            .unwrap();
10747        assert!(opened.await.unwrap().is_empty());
10748        let _ = client_rx.recv().await.unwrap();
10749
10750        let root = unique_project_root("live-roots-pending");
10751        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10752            .unwrap()
10753            .as_path()
10754            .to_path_buf();
10755        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10756        let open_handler = handler.clone();
10757        let pending = tokio::spawn(async move {
10758            open_handler
10759                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10760                .await
10761                .unwrap()
10762        });
10763        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10764            .await
10765            .unwrap()
10766            .unwrap();
10767        let actual = query_live_roots(&handler, &target_ctx).await;
10768        let ModuleControlResponseToModule::LiveRoots {
10769            roots,
10770            unknown_root_bindings,
10771            total_bindings,
10772        } = actual
10773        else {
10774            panic!("expected live roots")
10775        };
10776        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10777        assert_eq!(unknown_root_bindings, 0);
10778        assert_eq!(
10779            roots.len(),
10780            2,
10781            "root-known arm must retain each canonical root"
10782        );
10783        assert_eq!(
10784            total_bindings,
10785            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10786        );
10787        let counts = roots
10788            .iter()
10789            .map(|root| (root.project_root.clone(), root.bound, root.pending))
10790            .collect::<Vec<_>>();
10791        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10792        expected.sort_by(|a, b| a.0.cmp(&b.0));
10793        assert_eq!(
10794            counts, expected,
10795            "roots must sort by path and count pending separately"
10796        );
10797        handler
10798            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10799            .await
10800            .unwrap();
10801        assert!(pending.await.unwrap().is_empty());
10802    }
10803
10804    #[tokio::test(start_paused = true)]
10805    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10806        let forwarding = Arc::new(ForwardingTable::default());
10807        let handler =
10808            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10809        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10810        hello_via_sink(
10811            &handler,
10812            &target_ctx,
10813            &mut target_rx,
10814            hello_frame("target", PROTOCOL_VERSION, 1),
10815        )
10816        .await;
10817        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10818        let pending = forwarding
10819            .begin_route_bind_relay_for_test(
10820                client_ctx.connection_id,
10821                client_ctx.egress.clone(),
10822                2,
10823                "target",
10824            )
10825            .unwrap();
10826        forwarding
10827            .complete_pending_relay(
10828                target_ctx.connection_id,
10829                pending.corr,
10830                RouteBindRelayOutcome::Accepted,
10831            )
10832            .unwrap();
10833        let actual = query_live_roots(&handler, &target_ctx).await;
10834        let ModuleControlResponseToModule::LiveRoots {
10835            roots,
10836            unknown_root_bindings,
10837            total_bindings,
10838        } = actual
10839        else {
10840            panic!("expected live roots")
10841        };
10842        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10843        assert_eq!(
10844            unknown_root_bindings, 1,
10845            "unknown-root arm must not read as no bindings"
10846        );
10847        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10848        assert_eq!(
10849            total_bindings,
10850            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10851        );
10852    }
10853
10854    /// A module reads HELLO_ACK as its first frame and exits on anything else,
10855    /// so the ack has to be on its outbound queue before the module is
10856    /// routable. The connection loop writes a handler's replies only after the
10857    /// handler returns; this test stops in exactly that gap, runs a real
10858    /// route.open from another connection, and only then writes whatever the
10859    /// HELLO handler returned, the way the loop would. If the ack were still a
10860    /// reply, the route.bind request would reach the module first.
10861    #[tokio::test(start_paused = true)]
10862    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10863        let forwarding = Arc::new(ForwardingTable::default());
10864        let handler =
10865            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10866        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10867        let replies = handler
10868            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10869            .await
10870            .unwrap();
10871        let queued_by_hello = module_rx.len();
10872
10873        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10874        let open_handler = handler.clone();
10875        let open = tokio::spawn(async move {
10876            open_handler
10877                .handle_control_frame(
10878                    &client_ctx,
10879                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10880                )
10881                .await
10882                .unwrap()
10883        });
10884        // Let the route.open run until its route.bind is on the module's queue.
10885        let mut spins = 0;
10886        while module_rx.len() == queued_by_hello {
10887            spins += 1;
10888            assert!(spins < 10_000, "route.open never queued a route.bind");
10889            tokio::task::yield_now().await;
10890        }
10891
10892        // Now the connection loop's half: write the HELLO handler's replies.
10893        for reply in replies {
10894            module_ctx.egress.send(reply).await.unwrap();
10895        }
10896
10897        let first = module_rx.recv().await.unwrap().frame;
10898        assert_eq!(
10899            first.header.ty,
10900            FrameType::HelloAck,
10901            "the first frame a registering module reads must be its HELLO_ACK"
10902        );
10903        assert_eq!(first.header.corr, 7);
10904        let second = module_rx.recv().await.unwrap().frame;
10905        assert_eq!(second.header.ty, FrameType::Request);
10906        assert!(
10907            matches!(
10908                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10909                ModuleControlRequest::RouteBind { .. }
10910            ),
10911            "the route.bind follows the ack"
10912        );
10913        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10914
10915        handler
10916            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10917            .await
10918            .unwrap();
10919        assert!(open.await.unwrap().is_empty());
10920        let _ = client_rx.recv().await.unwrap();
10921    }
10922
10923    #[tokio::test(start_paused = true)]
10924    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10925        let handler = ControlHandler::with_forwarding(
10926            Arc::new(Registry::default()),
10927            Arc::new(ForwardingTable::default()),
10928        );
10929        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10930        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10931        hello_via_sink(
10932            &handler,
10933            &first_ctx,
10934            &mut first_rx,
10935            hello_frame("first", PROTOCOL_VERSION, 1),
10936        )
10937        .await;
10938        hello_via_sink(
10939            &handler,
10940            &second_ctx,
10941            &mut second_rx,
10942            hello_frame("second", PROTOCOL_VERSION, 2),
10943        )
10944        .await;
10945        let root = unique_project_root("second-only");
10946        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10947        let cloned = handler.clone();
10948        let open = tokio::spawn(async move {
10949            cloned
10950                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10951                .await
10952                .unwrap()
10953        });
10954        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10955            .await
10956            .unwrap()
10957            .unwrap();
10958        let first = query_live_roots(&handler, &first_ctx).await;
10959        let second = query_live_roots(&handler, &second_ctx).await;
10960        assert!(
10961            matches!(
10962                first,
10963                ModuleControlResponseToModule::LiveRoots {
10964                    total_bindings: 0,
10965                    ..
10966                }
10967            ),
10968            "cross-module scope must not expose another module's roots"
10969        );
10970        assert!(
10971            matches!(
10972                second,
10973                ModuleControlResponseToModule::LiveRoots {
10974                    total_bindings: 1,
10975                    ..
10976                }
10977            ),
10978            "second module must see its pending route"
10979        );
10980        handler
10981            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10982            .await
10983            .unwrap();
10984        assert!(open.await.unwrap().is_empty());
10985    }
10986
10987    #[tokio::test(start_paused = true)]
10988    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10989        let handler = ControlHandler::with_forwarding(
10990            Arc::new(Registry::default()),
10991            Arc::new(ForwardingTable::default()),
10992        );
10993        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10994        hello_via_sink(
10995            &handler,
10996            &target_ctx,
10997            &mut target_rx,
10998            hello_frame("target", PROTOCOL_VERSION, 1),
10999        )
11000        .await;
11001        let actual = query_live_roots(&handler, &target_ctx).await;
11002        let ModuleControlResponseToModule::LiveRoots {
11003            roots,
11004            unknown_root_bindings,
11005            total_bindings,
11006        } = actual
11007        else {
11008            panic!("expected live roots")
11009        };
11010        assert!(roots.is_empty());
11011        assert_eq!(unknown_root_bindings, 0);
11012        assert_eq!(total_bindings, 0);
11013        assert_eq!(
11014            total_bindings,
11015            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11016        );
11017    }
11018
11019    /// Read the vendored fed corpus rather than hand-building a package.
11020    ///
11021    /// A hand-built object encodes what the test author believed the carrier
11022    /// emits. These vectors are what it actually emits, and one of them exists
11023    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11024    /// ignores additive unknown fields at the traversal emit terminus."
11025    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11026        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11027            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11028        let text = std::fs::read_to_string(&path)
11029            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11030        let vectors: Vec<(String, Value)> = text
11031            .lines()
11032            .filter(|line| !line.trim().is_empty())
11033            .map(|line| {
11034                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11035                let id = entry["corpus_id"]
11036                    .as_str()
11037                    .expect("every vector carries a corpus_id")
11038                    .to_string();
11039                (id, entry["package"].clone())
11040            })
11041            .collect();
11042        // Pin the count: a corpus that silently shrinks would take its coverage
11043        // with it, and a suite reading N-1 vectors reports the same clean pass
11044        // as one reading N.
11045        assert_eq!(
11046            vectors.len(),
11047            3,
11048            "vendored fed corpus changed size; re-sync from subc-federation"
11049        );
11050
11051        // Pin what makes the corpus DISCRIMINATING, not just present.
11052        //
11053        // The relay test below takes its expected value from the corpus, so the
11054        // corpus supplies the test's power to detect a lossy relay rather than
11055        // its correctness. A relay that dropped unrecognised fields would still
11056        // be caught -- but only by a package carrying fields it does not know.
11057        // Shrink every package to the handful of keys any implementation would
11058        // recognise and the test keeps passing over an input that can no longer
11059        // fail, which is the same clean green as a corpus that shrank away.
11060        //
11061        // So assert the precondition rather than duplicating the packages here:
11062        // at least one vector must carry a field beyond the small common set.
11063        // That is one claim to maintain instead of nine, and it fails loudly if
11064        // a re-sync ever flattens the corpus.
11065        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11066        let richest = vectors
11067            .iter()
11068            .filter_map(|(_, package)| package.as_object())
11069            .map(|object| {
11070                object
11071                    .keys()
11072                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11073                    .count()
11074            })
11075            .max()
11076            .unwrap_or(0);
11077        assert!(
11078            richest >= 2,
11079            "vendored corpus no longer carries a package with unmodelled fields, \
11080             so the relay test can no longer distinguish a verbatim relay from a lossy one"
11081        );
11082
11083        vectors
11084    }
11085
11086    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
11087    ///
11088    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
11089    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
11090    /// three-key object, so a relay that quietly dropped fields it did not
11091    /// recognise would satisfy it. These vectors carry nine keys including ones
11092    /// this crate has no type for, so a typed relay fails here and only here.
11093    #[tokio::test]
11094    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11095        for (corpus_id, package) in fed_admission_facts_vectors() {
11096            let registry = Arc::new(Registry::default());
11097            let forwarding = Arc::new(ForwardingTable::default());
11098            let supervisor = SupervisorHandle::new();
11099            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11100            let handler =
11101                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11102                    .with_supervisor(supervisor)
11103                    .with_admission_facts_config(
11104                        Some("fed".to_string()),
11105                        Some(vec!["target".to_string()]),
11106                    );
11107
11108            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11109            hello_via_sink(
11110                &handler,
11111                &target_ctx,
11112                &mut target_rx,
11113                hello_frame("target", PROTOCOL_VERSION, 1),
11114            )
11115            .await;
11116
11117            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11118            let route_handler = handler.clone();
11119            let expected = package.clone();
11120            let route_task = tokio::spawn(async move {
11121                route_handler
11122                    .handle_control_frame(
11123                        &client_ctx,
11124                        route_open_frame_with_admission_facts(
11125                            20,
11126                            "target",
11127                            unique_project_root("admission-facts"),
11128                            Some(subc_control::ConsumerIdentity {
11129                                module_id: "fed".to_string(),
11130                                launch_nonce: "fed-nonce".to_string(),
11131                            }),
11132                            Some(package),
11133                        ),
11134                    )
11135                    .await
11136                    .unwrap()
11137            });
11138
11139            let bind_frame = target_rx.recv().await.unwrap();
11140            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11141            let ModuleControlRequest::RouteBind {
11142                admission_facts, ..
11143            } = bind
11144            else {
11145                panic!("{corpus_id}: expected route.bind")
11146            };
11147            assert_eq!(
11148                admission_facts,
11149                Some(expected),
11150                "{corpus_id}: relay must not add, drop or reshape any field"
11151            );
11152
11153            handler
11154                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11155                .await
11156                .unwrap();
11157            route_task.await.unwrap();
11158        }
11159    }
11160
11161    #[tokio::test]
11162    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11163        let registry = Arc::new(Registry::default());
11164        let forwarding = Arc::new(ForwardingTable::default());
11165        let supervisor = SupervisorHandle::new();
11166        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11167        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11168        let handler =
11169            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11170                .with_supervisor(supervisor)
11171                .with_admission_facts_config(
11172                    Some("fed".to_string()),
11173                    Some(vec!["target".to_string()]),
11174                );
11175
11176        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11177        hello_via_sink(
11178            &handler,
11179            &target_ctx,
11180            &mut target_rx,
11181            hello_frame("target", PROTOCOL_VERSION, 1),
11182        )
11183        .await;
11184        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11185        hello_via_sink(
11186            &handler,
11187            &other_ctx,
11188            &mut other_rx,
11189            hello_frame("other", PROTOCOL_VERSION, 2),
11190        )
11191        .await;
11192
11193        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11194        let expected_facts = facts.clone();
11195        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11196        let route_handler = handler.clone();
11197        let route_task = tokio::spawn(async move {
11198            route_handler
11199                .handle_control_frame(
11200                    &client_ctx,
11201                    route_open_frame_with_admission_facts(
11202                        10,
11203                        "target",
11204                        unique_project_root("admission-facts"),
11205                        Some(subc_control::ConsumerIdentity {
11206                            module_id: "fed".to_string(),
11207                            launch_nonce: "fed-nonce".to_string(),
11208                        }),
11209                        Some(facts.clone()),
11210                    ),
11211                )
11212                .await
11213                .unwrap()
11214        });
11215        let bind_frame = target_rx.recv().await.unwrap();
11216        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11217        let ModuleControlRequest::RouteBind {
11218            admission_facts, ..
11219        } = bind
11220        else {
11221            panic!("expected route.bind")
11222        };
11223        assert_eq!(admission_facts, Some(expected_facts));
11224        handler
11225            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11226            .await
11227            .unwrap();
11228        assert!(route_task.await.unwrap().is_empty());
11229        assert!(matches!(
11230            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11231                .unwrap(),
11232            ClientControlResponse::RouteOpen { .. }
11233        ));
11234
11235        let direct = handler
11236            .handle_control_frame(
11237                &route_ctx(ConnectionId::new(73)).0,
11238                route_open_frame_with_admission_facts(
11239                    11,
11240                    "target",
11241                    unique_project_root("admission-facts"),
11242                    None,
11243                    Some(json!({"x": 1})),
11244                ),
11245            )
11246            .await
11247            .unwrap();
11248        assert_eq!(
11249            parse_error(&direct[0])["code"],
11250            "admission_facts_not_permitted"
11251        );
11252
11253        let different_reserved = handler
11254            .handle_control_frame(
11255                &route_ctx(ConnectionId::new(77)).0,
11256                route_open_frame_with_admission_facts(
11257                    15,
11258                    "target",
11259                    unique_project_root("admission-facts"),
11260                    Some(subc_control::ConsumerIdentity {
11261                        module_id: "other".to_string(),
11262                        launch_nonce: "other-nonce".to_string(),
11263                    }),
11264                    Some(json!({"x": 1})),
11265                ),
11266            )
11267            .await
11268            .unwrap();
11269        assert_eq!(
11270            parse_error(&different_reserved[0])["code"],
11271            "admission_facts_not_permitted"
11272        );
11273
11274        let other_target = handler
11275            .handle_control_frame(
11276                &route_ctx(ConnectionId::new(74)).0,
11277                route_open_frame_with_admission_facts(
11278                    12,
11279                    "other",
11280                    unique_project_root("admission-facts"),
11281                    Some(subc_control::ConsumerIdentity {
11282                        module_id: "fed".to_string(),
11283                        launch_nonce: "fed-nonce".to_string(),
11284                    }),
11285                    Some(json!({"x": 1})),
11286                ),
11287            )
11288            .await
11289            .unwrap();
11290        assert_eq!(
11291            parse_error(&other_target[0])["code"],
11292            "admission_facts_target_not_allowed"
11293        );
11294
11295        let nonexistent = handler
11296            .handle_control_frame(
11297                &route_ctx(ConnectionId::new(75)).0,
11298                route_open_frame_with_admission_facts(
11299                    13,
11300                    "missing",
11301                    unique_project_root("admission-facts"),
11302                    None,
11303                    Some(json!({"x": 1})),
11304                ),
11305            )
11306            .await
11307            .unwrap();
11308        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11309
11310        let described = handler
11311            .handle_control_frame(
11312                &route_ctx(ConnectionId::new(76)).0,
11313                Frame::build(
11314                    FrameType::Request,
11315                    control_flags(),
11316                    0,
11317                    0,
11318                    14,
11319                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11320                )
11321                .unwrap(),
11322            )
11323            .await
11324            .unwrap();
11325        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11326            serde_json::from_slice(&described[0].body).unwrap()
11327        else {
11328            panic!("expected server.describe response")
11329        };
11330        assert!(capabilities
11331            .iter()
11332            .any(|cap| cap == "admission_facts_relay_v1"));
11333    }
11334
11335    #[tokio::test]
11336    async fn admission_facts_without_configured_carrier_are_rejected() {
11337        let registry = Arc::new(Registry::default());
11338        let forwarding = Arc::new(ForwardingTable::default());
11339        let handler = ControlHandler::with_forwarding(registry, forwarding);
11340        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11341        hello_via_sink(
11342            &handler,
11343            &target_ctx,
11344            &mut target_rx,
11345            hello_frame("target", PROTOCOL_VERSION, 1),
11346        )
11347        .await;
11348
11349        let responses = handler
11350            .handle_control_frame(
11351                &route_ctx(ConnectionId::new(79)).0,
11352                route_open_frame_with_admission_facts(
11353                    16,
11354                    "target",
11355                    unique_project_root("admission-facts"),
11356                    None,
11357                    Some(json!({"x": 1})),
11358                ),
11359            )
11360            .await
11361            .unwrap();
11362        assert_eq!(
11363            parse_error(&responses[0])["code"],
11364            "admission_facts_not_permitted"
11365        );
11366    }
11367
11368    #[tokio::test]
11369    async fn route_open_relays_consumer_capabilities_verbatim() {
11370        let registry = Arc::new(Registry::default());
11371        let forwarding = Arc::new(ForwardingTable::default());
11372        let handler =
11373            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11374        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11375        hello_via_sink(
11376            &handler,
11377            &module_ctx,
11378            &mut module_rx,
11379            hello_frame("aft", PROTOCOL_VERSION, 7),
11380        )
11381        .await;
11382
11383        let expected = vec!["elicitation".to_string(), "roots".to_string()];
11384        let expected_for_request = expected.clone();
11385        let project_root = unique_project_root("consumer-capabilities-present");
11386        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11387        let route_handler = handler.clone();
11388        let route_task = tokio::spawn(async move {
11389            route_handler
11390                .handle_control_frame(
11391                    &client_ctx,
11392                    route_open_frame_with_consumer_capabilities(
11393                        401,
11394                        "aft",
11395                        project_root,
11396                        Some(expected_for_request),
11397                    ),
11398                )
11399                .await
11400                .unwrap()
11401        });
11402        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11403            .await
11404            .unwrap()
11405            .unwrap();
11406        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11407        let ModuleControlRequest::RouteBind {
11408            consumer_capabilities,
11409            ..
11410        } = bind
11411        else {
11412            panic!("expected route.bind request, got {bind:?}");
11413        };
11414        assert_eq!(consumer_capabilities, Some(expected.clone()));
11415
11416        handler
11417            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11418            .await
11419            .unwrap();
11420        let route_response = route_task.await.unwrap();
11421        assert!(route_response.is_empty());
11422        let published = client_rx.recv().await.unwrap();
11423        assert!(matches!(
11424            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11425            ClientControlResponse::RouteOpen { .. }
11426        ));
11427    }
11428
11429    #[tokio::test]
11430    async fn route_open_without_consumer_capabilities_relays_none() {
11431        let registry = Arc::new(Registry::default());
11432        let forwarding = Arc::new(ForwardingTable::default());
11433        let handler =
11434            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11435        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11436        hello_via_sink(
11437            &handler,
11438            &module_ctx,
11439            &mut module_rx,
11440            hello_frame("aft", PROTOCOL_VERSION, 7),
11441        )
11442        .await;
11443
11444        let project_root = unique_project_root("consumer-capabilities-absent");
11445        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11446        let route_handler = handler.clone();
11447        let route_task = tokio::spawn(async move {
11448            route_handler
11449                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11450                .await
11451                .unwrap()
11452        });
11453        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11454            .await
11455            .unwrap()
11456            .unwrap();
11457        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11458        let ModuleControlRequest::RouteBind {
11459            consumer_capabilities,
11460            ..
11461        } = bind
11462        else {
11463            panic!("expected route.bind request, got {bind:?}");
11464        };
11465        assert_eq!(consumer_capabilities, None);
11466
11467        handler
11468            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11469            .await
11470            .unwrap();
11471        let route_response = route_task.await.unwrap();
11472        assert!(route_response.is_empty());
11473        let published = client_rx.recv().await.unwrap();
11474        assert!(matches!(
11475            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11476            ClientControlResponse::RouteOpen { .. }
11477        ));
11478    }
11479
11480    /// Opens a route to a freshly registered `aft` with `sent` as the
11481    /// route.open's role_versions, acks the bind, and returns the role_versions
11482    /// the module's bind carried.
11483    async fn bind_role_versions_for(
11484        sent: Option<BTreeMap<String, String>>,
11485        connection: u64,
11486    ) -> Option<BTreeMap<String, String>> {
11487        let registry = Arc::new(Registry::default());
11488        let forwarding = Arc::new(ForwardingTable::default());
11489        let handler =
11490            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11491        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11492        hello_via_sink(
11493            &handler,
11494            &module_ctx,
11495            &mut module_rx,
11496            hello_frame("aft", PROTOCOL_VERSION, 7),
11497        )
11498        .await;
11499        let project_root = unique_project_root("role-versions");
11500        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11501        let route_handler = handler.clone();
11502        let route_task = tokio::spawn(async move {
11503            route_handler
11504                .handle_control_frame(
11505                    &client_ctx,
11506                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
11507                )
11508                .await
11509                .unwrap()
11510        });
11511        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11512            .await
11513            .expect("a well-formed route.open reaches the module as a bind")
11514            .unwrap();
11515        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11516        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11517            panic!("expected route.bind request, got {bind:?}");
11518        };
11519        handler
11520            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11521            .await
11522            .unwrap();
11523        assert!(route_task.await.unwrap().is_empty());
11524        let published = client_rx.recv().await.unwrap();
11525        assert!(matches!(
11526            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11527            ClientControlResponse::RouteOpen { .. }
11528        ));
11529        role_versions
11530    }
11531
11532    #[tokio::test]
11533    async fn route_open_relays_role_versions_verbatim() {
11534        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11535        assert_eq!(
11536            bind_role_versions_for(Some(sent.clone()), 141).await,
11537            Some(sent)
11538        );
11539    }
11540
11541    /// An empty map declares nothing, so the provider sees no field rather
11542    /// than an empty object it would have to treat as a second "none".
11543    #[tokio::test]
11544    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11545        assert_eq!(bind_role_versions_for(None, 143).await, None);
11546        assert_eq!(
11547            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11548            None
11549        );
11550    }
11551
11552    #[tokio::test]
11553    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11554        let registry = Arc::new(Registry::default());
11555        let forwarding = Arc::new(ForwardingTable::default());
11556        let handler =
11557            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11558        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11559        hello_via_sink(
11560            &handler,
11561            &module_ctx,
11562            &mut module_rx,
11563            hello_frame("aft", PROTOCOL_VERSION, 7),
11564        )
11565        .await;
11566
11567        let nine: BTreeMap<String, String> = (0..9)
11568            .map(|index| (format!("role-{index}"), "v1".to_string()))
11569            .collect();
11570        for (label, malformed) in [
11571            (
11572                "invalid role name",
11573                role_versions(&[("Tool_Provider", "v1")]),
11574            ),
11575            ("invalid version", role_versions(&[("tool-provider", "v0")])),
11576            ("nine entries", nine),
11577        ] {
11578            // A refused open answers at once; one that reached the module
11579            // would wait for its bind ack and trip this timeout.
11580            let responses = tokio::time::timeout(
11581                Duration::from_secs(1),
11582                handler.handle_control_frame(
11583                    &route_ctx(ConnectionId::new(148)).0,
11584                    route_open_frame_with_role_versions(
11585                        404,
11586                        "aft",
11587                        unique_project_root("role-versions-malformed"),
11588                        Some(malformed),
11589                    ),
11590                ),
11591            )
11592            .await
11593            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11594            .unwrap();
11595            assert_eq!(responses.len(), 1, "{label}");
11596            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11597            let error = parse_error(&responses[0]);
11598            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11599            assert_eq!(
11600                error["detail"]["field"], "role_versions",
11601                "{label}: {error}"
11602            );
11603            assert!(
11604                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11605                "{label}: a malformed declaration is terminal"
11606            );
11607            assert!(
11608                module_rx.try_recv().is_err(),
11609                "{label}: the module must never see a bind"
11610            );
11611        }
11612    }
11613
11614    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
11615    /// consumer can tell this daemon forwards the field from one that would
11616    /// drop it.
11617    #[tokio::test]
11618    async fn route_role_versions_capability_is_advertised() {
11619        let handler = ControlHandler::new(Arc::new(Registry::default()));
11620        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11621        let ack = hello_via_sink(
11622            &handler,
11623            &ctx,
11624            &mut rx,
11625            hello_frame("m", PROTOCOL_VERSION, 1),
11626        )
11627        .await;
11628        let ack = parse_ack(&ack);
11629        assert!(
11630            ack.subc_capabilities
11631                .iter()
11632                .any(|c| c == "route-role-versions/v1"),
11633            "{:?}",
11634            ack.subc_capabilities
11635        );
11636
11637        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11638        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11639        let reply = handler
11640            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11641            .await
11642            .unwrap()
11643            .pop()
11644            .unwrap();
11645        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11646            serde_json::from_slice(&reply.body).unwrap()
11647        else {
11648            panic!("not a server.describe reply");
11649        };
11650        assert!(
11651            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11652            "{capabilities:?}"
11653        );
11654    }
11655
11656    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
11657    /// catalog entry without `events` (the module declares none) from a daemon
11658    /// that never reports declared events.
11659    #[tokio::test]
11660    async fn catalog_events_capability_is_advertised() {
11661        let handler = ControlHandler::new(Arc::new(Registry::default()));
11662        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11663        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11664        let reply = handler
11665            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11666            .await
11667            .unwrap()
11668            .pop()
11669            .unwrap();
11670        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11671            serde_json::from_slice(&reply.body).unwrap()
11672        else {
11673            panic!("not a server.describe reply");
11674        };
11675        assert!(
11676            capabilities
11677                .iter()
11678                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11679            "{capabilities:?}"
11680        );
11681    }
11682
11683    #[tokio::test]
11684    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11685        let handler = ControlHandler::new(Arc::new(Registry::default()));
11686        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11687        let ack = hello_via_sink(
11688            &handler,
11689            &module,
11690            &mut module_rx,
11691            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11692        )
11693        .await;
11694        let ack = parse_ack(&ack);
11695        assert!(
11696            ack.subc_capabilities
11697                .iter()
11698                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11699            "{:?}",
11700            ack.subc_capabilities
11701        );
11702        assert!(ack
11703            .subc_capabilities
11704            .iter()
11705            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11706
11707        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11708        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11709        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11710        let reply = handler
11711            .handle_control_frame(&client, frame)
11712            .await
11713            .unwrap()
11714            .pop()
11715            .unwrap();
11716        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11717            serde_json::from_slice(&reply.body).unwrap()
11718        else {
11719            panic!("not a server.describe reply");
11720        };
11721        assert!(
11722            capabilities
11723                .iter()
11724                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11725            "{capabilities:?}"
11726        );
11727        assert!(capabilities
11728            .iter()
11729            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11730    }
11731
11732    #[tokio::test]
11733    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
11734        let handler = ControlHandler::new(Arc::new(Registry::default()));
11735        let (module, mut rx) = route_ctx(ConnectionId::new(156));
11736        let ack = hello_via_sink(
11737            &handler,
11738            &module,
11739            &mut rx,
11740            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
11741        )
11742        .await;
11743        assert!(parse_ack(&ack)
11744            .subc_capabilities
11745            .iter()
11746            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11747        assert!(parse_ack(&ack)
11748            .subc_capabilities
11749            .iter()
11750            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11751        let (client, _rx) = route_ctx(ConnectionId::new(157));
11752        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11753        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11754        let reply = handler
11755            .handle_control_frame(&client, frame)
11756            .await
11757            .unwrap()
11758            .pop()
11759            .unwrap();
11760        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11761            serde_json::from_slice(&reply.body).unwrap()
11762        else {
11763            panic!("server.describe reply")
11764        };
11765        assert!(capabilities
11766            .iter()
11767            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11768        assert!(capabilities
11769            .iter()
11770            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11771    }
11772
11773    #[tokio::test]
11774    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11775        let handler = ControlHandler::new(Arc::new(Registry::default()));
11776        let (module, mut rx) = route_ctx(ConnectionId::new(154));
11777        let ack = hello_via_sink(
11778            &handler,
11779            &module,
11780            &mut rx,
11781            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11782        )
11783        .await;
11784        assert!(parse_ack(&ack)
11785            .subc_capabilities
11786            .iter()
11787            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11788        let (client, _rx) = route_ctx(ConnectionId::new(155));
11789        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11790        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11791        let reply = handler
11792            .handle_control_frame(&client, frame)
11793            .await
11794            .unwrap()
11795            .pop()
11796            .unwrap();
11797        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11798            serde_json::from_slice(&reply.body).unwrap()
11799        else {
11800            panic!("not a server.describe reply")
11801        };
11802        assert!(capabilities
11803            .iter()
11804            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11805    }
11806
11807    #[tokio::test]
11808    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11809        let registry = Arc::new(Registry::default());
11810        let forwarding = Arc::new(ForwardingTable::default());
11811        let handler =
11812            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11813                .with_health_probe_timeout(Duration::from_secs(5));
11814        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11815        hello_via_sink(
11816            &handler,
11817            &module_ctx,
11818            &mut module_rx,
11819            non_routable_hello_frame_with_control_ops(
11820                "mcp",
11821                300,
11822                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11823            ),
11824        )
11825        .await;
11826        assert!(registry
11827            .get_module("mcp")
11828            .unwrap()
11829            .unwrap()
11830            .manifest
11831            .provides
11832            .is_empty());
11833
11834        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11835        let route_response = handler
11836            .handle_control_frame(
11837                &route_client_ctx,
11838                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11839            )
11840            .await
11841            .unwrap();
11842        assert_eq!(route_response[0].header.ty, FrameType::Error);
11843        assert_eq!(
11844            parse_error(&route_response[0])["code"],
11845            "target_unavailable"
11846        );
11847        assert!(parse_error(&route_response[0])["message"]
11848            .as_str()
11849            .unwrap()
11850            .contains("does not provide the requested target"));
11851        assert!(module_rx.try_recv().is_err());
11852
11853        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11854        let health_handler = handler.clone();
11855        let health_task = tokio::spawn(async move {
11856            health_handler
11857                .handle_control_frame(
11858                    &health_client_ctx,
11859                    supervisor_health_probe_frame(302, "mcp"),
11860                )
11861                .await
11862                .unwrap()
11863        });
11864        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11865            .await
11866            .unwrap()
11867            .unwrap();
11868        assert_eq!(
11869            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11870            ModuleControlRequest::HealthCheck {}
11871        );
11872        handler
11873            .handle_control_frame(
11874                &module_ctx,
11875                health_response(health_frame.header.corr, HealthStatus::Ok),
11876            )
11877            .await
11878            .unwrap();
11879        let health_response = health_task.await.unwrap();
11880        assert_eq!(health_response[0].header.ty, FrameType::Response);
11881        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11882            ClientControlResponse::SupervisorHealthProbe {
11883                module_id, status, ..
11884            } => {
11885                assert_eq!(module_id, "mcp");
11886                assert_eq!(status, HealthStatus::Ok);
11887            }
11888            other => panic!("unexpected health response: {other:?}"),
11889        }
11890
11891        // Exercise the forwarding cleanup path directly while leaving the registry
11892        // advertisement in place. If cleanup leaves a stale control sink behind,
11893        // the next probe will enqueue onto it and wait for the long probe timeout
11894        // instead of returning an immediate no-connection error.
11895        forwarding
11896            .cleanup_connection(module_ctx.connection_id)
11897            .unwrap();
11898        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11899        let cleanup_response = tokio::time::timeout(
11900            Duration::from_millis(200),
11901            handler.handle_control_frame(
11902                &cleanup_probe_ctx,
11903                supervisor_health_probe_frame(303, "mcp"),
11904            ),
11905        )
11906        .await
11907        .expect("probe should fail immediately when the control lane is gone")
11908        .unwrap();
11909        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11910        assert_eq!(
11911            parse_error(&cleanup_response[0])["code"],
11912            "target_unavailable"
11913        );
11914        assert!(parse_error(&cleanup_response[0])["message"]
11915            .as_str()
11916            .unwrap()
11917            .contains("no module connection"));
11918
11919        handler
11920            .cleanup_connection(module_ctx.connection_id)
11921            .unwrap();
11922    }
11923
11924    #[tokio::test]
11925    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11926        let registry = Arc::new(Registry::default());
11927        let supervisor_handle = SupervisorHandle::new();
11928        let supervisor =
11929            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
11930                .with_handle(supervisor_handle.clone())
11931                .with_connection_file_path(
11932                    std::env::temp_dir()
11933                        .join(format!("subc-route-open-warming-{}", std::process::id())),
11934                );
11935        let module = supervisor
11936            .supervise_configured(
11937                ModuleSpec {
11938                    module_id: "warming".to_string(),
11939                    program: fake_aft_stub_path(),
11940                    args: Vec::new(),
11941                    env: Vec::new(),
11942                    reserved: false,
11943                    reserved_prefixes: Vec::new(),
11944                    protocol: ModuleProtocol::Subc,
11945                    overlap: Default::default(),
11946                },
11947                true,
11948            )
11949            .unwrap();
11950        assert_eq!(module.state().unwrap(), ModuleState::Running);
11951
11952        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11953        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11954        let response = handler
11955            .handle_control_frame(
11956                &ctx,
11957                route_open_frame(304, "warming", unique_project_root("warming")),
11958            )
11959            .await
11960            .unwrap();
11961        module.stop().await.unwrap();
11962
11963        assert_eq!(response[0].header.ty, FrameType::Error);
11964        let error = parse_error(&response[0]);
11965        assert_eq!(error["code"], "module_warming");
11966        assert!(error["message"]
11967            .as_str()
11968            .unwrap()
11969            .contains("state=running, enabled=true, live=false"));
11970    }
11971
11972    #[test]
11973    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11974        let handler = ControlHandler::new(Arc::new(Registry::default()));
11975        let capture = EventCapture::default();
11976        let _subscriber =
11977            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11978        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11979        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11980        let pending = (0..limit).collect::<Vec<_>>();
11981        let response = handler
11982            .route_open_capacity_refusal(
11983                &ctx,
11984                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11985                "busy",
11986                pending.len(),
11987                limit,
11988            )
11989            .unwrap();
11990        assert_eq!(parse_error(&response)["code"], "target_unavailable");
11991        let event = capture
11992            .events()
11993            .into_iter()
11994            .find(|event| {
11995                event.target == "control"
11996                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11997            })
11998            .expect("connection admission refusal event");
11999        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12000        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12001    }
12002
12003    #[test]
12004    fn route_open_target_cap_logs_admission_reason_and_capacity() {
12005        let handler = ControlHandler::new(Arc::new(Registry::default()));
12006        let capture = EventCapture::default();
12007        let _subscriber =
12008            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12009        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12010        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12011        let guards = (0..limit)
12012            .map(|_| {
12013                handler
12014                    .route_bind_concurrency
12015                    .try_admit("busy", limit)
12016                    .unwrap()
12017            })
12018            .collect::<Vec<_>>();
12019        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12020            Err(in_flight) => in_flight,
12021            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12022        };
12023        let response = handler
12024            .route_open_target_capacity_refusal(
12025                &ctx,
12026                &route_open_frame(397, "busy", unique_project_root("target-cap")),
12027                "busy",
12028                in_flight,
12029            )
12030            .unwrap();
12031        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12032        let event = capture
12033            .events()
12034            .into_iter()
12035            .find(|event| {
12036                event.target == "control"
12037                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12038            })
12039            .expect("target admission refusal event");
12040        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12041        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12042        drop(guards);
12043    }
12044
12045    #[tokio::test(flavor = "current_thread")]
12046    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12047        let handler = ControlHandler::new(Arc::new(Registry::default()));
12048        let capture = EventCapture::default();
12049        let _subscriber =
12050            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12051        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12052        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12053
12054        let _ = handler
12055            .handle_client_control_request(
12056                &ctx,
12057                frame,
12058                ClientControlRequest::SupervisorSetEnabled {
12059                    module_id: "broca".to_string(),
12060                    enabled: false,
12061                },
12062                &mut None,
12063            )
12064            .await
12065            .unwrap();
12066
12067        let events = capture
12068            .events()
12069            .into_iter()
12070            .filter(|event| {
12071                event.target == "control"
12072                    && event.fields.get("message").map(String::as_str)
12073                        == Some("supervisor request received")
12074            })
12075            .collect::<Vec<_>>();
12076        assert_eq!(events.len(), 1, "one supervisor request log line");
12077        let event = &events[0];
12078        assert_eq!(event.level, tracing::Level::INFO);
12079        assert_eq!(
12080            event.fields.get("op"),
12081            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12082        );
12083        assert_eq!(
12084            event.fields.get("module_id"),
12085            Some(&"\"broca\"".to_string())
12086        );
12087        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12088        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12089        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12090    }
12091
12092    #[tokio::test(flavor = "current_thread")]
12093    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12094        let registry = Arc::new(Registry::default());
12095        let module_connection = ConnectionId::new(303);
12096        registry
12097            .register_with_control_ops(
12098                manifest("aft", PROTOCOL_VERSION),
12099                PROTOCOL_VERSION,
12100                module_connection,
12101                module_baseline_control_ops(),
12102            )
12103            .unwrap();
12104        let handler = ControlHandler::new(registry);
12105        let capture = EventCapture::default();
12106        let _subscriber =
12107            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12108        let (ctx, _rx) = route_ctx(module_connection);
12109        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12110
12111        let _ = handler
12112            .handle_client_control_request(
12113                &ctx,
12114                frame,
12115                ClientControlRequest::SupervisorSetEnabled {
12116                    module_id: "broca".to_string(),
12117                    enabled: false,
12118                },
12119                &mut None,
12120            )
12121            .await
12122            .unwrap();
12123
12124        let callers = capture
12125            .events()
12126            .into_iter()
12127            .filter(|event| {
12128                event.fields.get("message").map(String::as_str)
12129                    == Some("supervisor request received")
12130            })
12131            .map(|event| event.fields.get("caller").cloned())
12132            .collect::<Vec<_>>();
12133        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12134    }
12135
12136    #[tokio::test(flavor = "current_thread")]
12137    async fn supervisor_list_does_not_log_request_received() {
12138        let handler = ControlHandler::new(Arc::new(Registry::default()));
12139        let capture = EventCapture::default();
12140        let _subscriber =
12141            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12142        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12143        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12144
12145        handler
12146            .handle_client_control_request(
12147                &ctx,
12148                frame,
12149                ClientControlRequest::SupervisorList {},
12150                &mut None,
12151            )
12152            .await
12153            .unwrap();
12154
12155        assert!(capture.events().into_iter().all(|event| {
12156            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12157        }));
12158    }
12159
12160    #[tokio::test(flavor = "current_thread")]
12161    async fn supervisor_rescan_preview_does_not_log_request_received() {
12162        let handler = ControlHandler::new(Arc::new(Registry::default()));
12163        let capture = EventCapture::default();
12164        let _subscriber =
12165            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12166        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12167        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12168
12169        handler
12170            .handle_client_control_request(
12171                &ctx,
12172                frame,
12173                ClientControlRequest::SupervisorRescan { preview: true },
12174                &mut None,
12175            )
12176            .await
12177            .unwrap();
12178
12179        assert!(capture.events().into_iter().all(|event| {
12180            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12181        }));
12182    }
12183
12184    struct ApproveOperator;
12185    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12186        fn prompt(
12187            &self,
12188            _: &str,
12189            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12190        ) -> crate::operator_confirm::ProviderResult {
12191            crate::operator_confirm::ProviderResult::Approved
12192        }
12193    }
12194
12195    #[tokio::test]
12196    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12197        assert_eq!(
12198            ControlHandler::default().privileged_operator_prompt,
12199            PrivilegedOperatorPrompt::Log
12200        );
12201        for (initial, changed, running) in [
12202            ("log", "enforce", PrivilegedOperatorPrompt::Log),
12203            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12204        ] {
12205            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12206            let path = dir.join("subc.jsonc");
12207            std::fs::write(
12208                &path,
12209                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12210            )
12211            .unwrap();
12212            let config = crate::daemon_config::load(&path).unwrap().unwrap();
12213            let registry = Arc::new(Registry::default());
12214            let handle = SupervisorHandle::new();
12215            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12216                .with_handle(handle.clone());
12217            let handler = ControlHandler::new(registry)
12218                .with_supervisor(handle)
12219                .with_privileged_operator_prompt(config.privileged_operator_prompt)
12220                .with_supervisor_rescan(supervisor, &path, config.port);
12221            handler.forwarding.operator_confirms().configure(
12222                Arc::new(ApproveOperator),
12223                crate::operator_confirm::OperatorLimits::default(),
12224            );
12225            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12226
12227            for (disk_mode, needs_restart) in [
12228                (Some(initial), false),
12229                (Some(changed), true),
12230                (Some(changed), true),
12231                (None, running == PrivilegedOperatorPrompt::Enforce),
12232                (Some(initial), false),
12233            ] {
12234                let mut doc = json!({"version":1});
12235                if let Some(mode) = disk_mode {
12236                    doc["privileged_operator_prompt"] = json!(mode);
12237                }
12238                std::fs::write(&path, doc.to_string()).unwrap();
12239                for preview in [true, false] {
12240                    let request = ClientControlRequest::SupervisorRescan { preview };
12241                    let frame = Frame::build(
12242                        FrameType::Request,
12243                        control_flags(),
12244                        0,
12245                        0,
12246                        1,
12247                        serde_json::to_vec(&request).unwrap(),
12248                    )
12249                    .unwrap();
12250                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12251                    assert_eq!(frames.len(), 1);
12252                    let ClientControlResponse::SupervisorRescan { result } =
12253                        serde_json::from_slice(&frames[0].body).unwrap()
12254                    else {
12255                        panic!("expected a rescan response");
12256                    };
12257                    let expected = if needs_restart {
12258                        vec!["privileged_operator_prompt"]
12259                    } else {
12260                        Vec::new()
12261                    };
12262                    assert_eq!(result.restart_required, expected);
12263                    assert_eq!(handler.privileged_operator_prompt, running);
12264                    assert_eq!(
12265                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12266                        running
12267                    );
12268                }
12269            }
12270        }
12271    }
12272
12273    #[tokio::test]
12274    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12275        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12276        let path = dir.join("subc.jsonc");
12277        let registry = Arc::new(Registry::default());
12278        let handle = SupervisorHandle::new();
12279        let supervisor =
12280            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12281        let handler = ControlHandler::new(registry)
12282            .with_supervisor(handle)
12283            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12284            .with_supervisor_rescan(supervisor, &path, None);
12285        handler.forwarding.operator_confirms().configure(
12286            Arc::new(ApproveOperator),
12287            crate::operator_confirm::OperatorLimits::default(),
12288        );
12289        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12290        for preview in [true, false] {
12291            std::fs::write(
12292                &path,
12293                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12294            )
12295            .unwrap();
12296            let request = ClientControlRequest::SupervisorRescan { preview };
12297            let frame = Frame::build(
12298                FrameType::Request,
12299                control_flags(),
12300                0,
12301                0,
12302                1,
12303                serde_json::to_vec(&request).unwrap(),
12304            )
12305            .unwrap();
12306            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12307            assert_eq!(frames.len(), 1);
12308            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12309            assert_eq!(
12310                handler.privileged_operator_prompt,
12311                PrivilegedOperatorPrompt::Enforce
12312            );
12313        }
12314    }
12315
12316    /// One wire code has several senders, so the refusal line names the check
12317    /// that refused. This drives the shared refusal path for ordinary refusals
12318    /// with an unregistered
12319    /// target and requires the branch label on the event.
12320    #[tokio::test]
12321    async fn route_open_refusal_names_the_check_that_refused() {
12322        let handler = ControlHandler::new(Arc::new(Registry::default()));
12323        let capture = EventCapture::default();
12324        let _subscriber =
12325            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12326        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12327        let response = handler
12328            .handle_control_frame(
12329                &ctx,
12330                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12331            )
12332            .await
12333            .unwrap();
12334
12335        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12336        let event = capture
12337            .events()
12338            .into_iter()
12339            .find(|event| {
12340                event.target == "control"
12341                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12342            })
12343            .expect("route.open refusal event");
12344        assert_eq!(
12345            event.fields.get("reason"),
12346            Some(&"\"not_registered\"".to_string())
12347        );
12348    }
12349
12350    #[tokio::test]
12351    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12352        let registry = Arc::new(Registry::default());
12353        let supervisor_handle = SupervisorHandle::new();
12354        let supervisor =
12355            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12356                .with_handle(supervisor_handle.clone())
12357                .with_connection_file_path(std::env::temp_dir().join(format!(
12358                    "subc-route-open-refusal-info-{}",
12359                    std::process::id()
12360                )));
12361        let module = supervisor
12362            .supervise_configured(
12363                ModuleSpec {
12364                    module_id: "warming".to_string(),
12365                    program: fake_aft_stub_path(),
12366                    args: Vec::new(),
12367                    env: Vec::new(),
12368                    reserved: false,
12369                    reserved_prefixes: Vec::new(),
12370                    protocol: ModuleProtocol::Subc,
12371                    overlap: Default::default(),
12372                },
12373                true,
12374            )
12375            .unwrap();
12376        assert_eq!(module.state().unwrap(), ModuleState::Running);
12377
12378        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12379        assert!(handler
12380            .counters()
12381            .snapshot()
12382            .get("route_open_refused_by_code")
12383            .is_none());
12384        let capture = EventCapture::default();
12385        let _subscriber =
12386            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12387        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12388        let response = handler
12389            .handle_control_frame(
12390                &ctx,
12391                route_open_frame(394, "warming", unique_project_root("refusal-info")),
12392            )
12393            .await
12394            .unwrap();
12395        module.stop().await.unwrap();
12396
12397        assert_eq!(parse_error(&response[0])["code"], "module_warming");
12398        let event = capture
12399            .events()
12400            .into_iter()
12401            .find(|event| {
12402                event.target == "control"
12403                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12404            })
12405            .expect("route.open refusal event");
12406        assert_eq!(
12407            event.fields.get("module_id"),
12408            Some(&"\"warming\"".to_string())
12409        );
12410        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12411        assert_eq!(
12412            event.fields.get("reason"),
12413            Some(&"\"supervised_not_registered\"".to_string())
12414        );
12415        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12416        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12417        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12418        assert_eq!(
12419            handler.counters().snapshot()["route_open_refused_by_code"],
12420            json!({ "module_warming": 1 })
12421        );
12422    }
12423
12424    const OUTAGE_START: &str = "route.open refusing module: not serving";
12425    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12426
12427    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12428        capture
12429            .events()
12430            .into_iter()
12431            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12432            .collect()
12433    }
12434
12435    fn supervise_stub(
12436        registry: &Arc<Registry>,
12437        module_id: &str,
12438        enabled: bool,
12439    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12440        let supervisor_handle = SupervisorHandle::new();
12441        let supervisor =
12442            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12443                .with_handle(supervisor_handle.clone())
12444                .with_connection_file_path(std::env::temp_dir().join(format!(
12445                    "subc-route-outage-{module_id}-{}",
12446                    std::process::id()
12447                )));
12448        let module = supervisor
12449            .supervise_configured(
12450                ModuleSpec {
12451                    module_id: module_id.to_string(),
12452                    program: fake_aft_stub_path(),
12453                    args: Vec::new(),
12454                    env: Vec::new(),
12455                    reserved: false,
12456                    reserved_prefixes: Vec::new(),
12457                    protocol: ModuleProtocol::Subc,
12458                    overlap: Default::default(),
12459                },
12460                enabled,
12461            )
12462            .unwrap();
12463        (supervisor_handle, module)
12464    }
12465
12466    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12467        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12468            module_id: module_id.to_string(),
12469            drain_timeout_ms: Some(50),
12470        })
12471        .unwrap();
12472        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12473    }
12474
12475    /// Two handlers built over one forwarding table must share one outage
12476    /// tracker; separate trackers would each log their own opening line for
12477    /// the same outage.
12478    #[test]
12479    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12480        let registry = Arc::new(Registry::default());
12481        let forwarding = Arc::new(ForwardingTable::default());
12482        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12483        let second = ControlHandler::with_forwarding(registry, forwarding);
12484        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12485    }
12486
12487    /// A client can name any module id it likes. Refusing an unknown one,
12488    /// however often, must not create outage state or outage lines, or the
12489    /// tracker would be a memory sink any client could fill.
12490    #[tokio::test(flavor = "current_thread")]
12491    async fn route_open_unknown_module_refusals_add_no_outage_state() {
12492        let handler = ControlHandler::new(Arc::new(Registry::default()));
12493        let capture = EventCapture::default();
12494        let _subscriber =
12495            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12496        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12497        for corr in 0..8 {
12498            let response = handler
12499                .handle_control_frame(
12500                    &ctx,
12501                    route_open_frame(
12502                        380 + corr,
12503                        &format!("nobody-{corr}"),
12504                        unique_project_root("outage-unknown"),
12505                    ),
12506                )
12507                .await
12508                .unwrap();
12509            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12510        }
12511
12512        assert_eq!(handler.route_outages.tracked_module_count(), 0);
12513        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12514        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12515    }
12516
12517    /// Drives the refusal path end to end: a supervised module that served
12518    /// before and stopped being registered with no instruction to stop is a
12519    /// WARN, and the same module refused after an operator `supervisor.restart`
12520    /// is an INFO.
12521    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12522    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12523        let registry = Arc::new(Registry::default());
12524        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
12525        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12526        let capture = EventCapture::default();
12527        let _subscriber =
12528            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12529        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12530        // The stub never registers, so pretend it served once: otherwise every
12531        // refusal would fall in its startup window.
12532        handler.route_outages.record_accepted("outage-restart");
12533
12534        let response = handler
12535            .handle_control_frame(
12536                &ctx,
12537                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12538            )
12539            .await
12540            .unwrap();
12541        assert_eq!(response[0].header.ty, FrameType::Error);
12542        let starts = outage_lines(&capture, OUTAGE_START);
12543        assert_eq!(starts.len(), 1, "{starts:?}");
12544        assert_eq!(starts[0].level, tracing::Level::WARN);
12545        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12546        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12547        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12548        handler.route_outages.record_accepted("outage-restart");
12549        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12550
12551        let restart = handler
12552            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12553            .await
12554            .unwrap();
12555        assert_eq!(
12556            restart[0].header.ty,
12557            FrameType::Response,
12558            "{:?}",
12559            parse_error(&restart[0])
12560        );
12561        handler
12562            .handle_control_frame(
12563                &ctx,
12564                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12565            )
12566            .await
12567            .unwrap();
12568        module.stop().await.unwrap();
12569
12570        let starts = outage_lines(&capture, OUTAGE_START);
12571        assert_eq!(starts.len(), 2, "{starts:?}");
12572        assert_eq!(starts[1].level, tracing::Level::INFO);
12573        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12574    }
12575
12576    /// A restart refused before it touched the module (here: the module is
12577    /// disabled) must clear its operator mark, so the next real outage is
12578    /// still reported as a warning.
12579    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12580    async fn failed_operator_restart_leaves_no_operator_mark() {
12581        let registry = Arc::new(Registry::default());
12582        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
12583        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12584        let capture = EventCapture::default();
12585        let _subscriber =
12586            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12587        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12588        handler.route_outages.record_accepted("outage-disabled");
12589
12590        let restart = handler
12591            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12592            .await
12593            .unwrap();
12594        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12595        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12596
12597        handler
12598            .handle_control_frame(
12599                &ctx,
12600                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12601            )
12602            .await
12603            .unwrap();
12604        let starts = outage_lines(&capture, OUTAGE_START);
12605        assert_eq!(starts.len(), 1, "{starts:?}");
12606        assert_eq!(starts[0].level, tracing::Level::WARN);
12607    }
12608
12609    #[tokio::test(flavor = "current_thread")]
12610    async fn route_open_unknown_module_escapes_target_module_id() {
12611        let handler = ControlHandler::new(Arc::new(Registry::default()));
12612        let capture = EventCapture::default();
12613        let _subscriber =
12614            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12615        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12616        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12617        let response = handler
12618            .handle_control_frame(
12619                &ctx,
12620                route_open_frame(
12621                    395,
12622                    hostile_module_id,
12623                    unique_project_root("hostile-target-module-id"),
12624                ),
12625            )
12626            .await
12627            .unwrap();
12628
12629        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12630        let event = capture
12631            .events()
12632            .into_iter()
12633            .find(|event| {
12634                event.target == "control"
12635                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12636            })
12637            .expect("route.open unknown-module refusal event");
12638        let logged = event.fields.get("module_id").expect("module_id field");
12639        assert!(!logged.bytes().any(|byte| byte < 0x20));
12640        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12641    }
12642
12643    #[tokio::test(flavor = "current_thread")]
12644    async fn route_open_module_rejection_uses_daemon_counter_key() {
12645        let registry = Arc::new(Registry::default());
12646        let forwarding = Arc::new(ForwardingTable::default());
12647        let handler =
12648            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12649        let module_connection = ConnectionId::new(95);
12650        let (module_ctx, mut module_rx) = route_ctx(module_connection);
12651        hello_via_sink(
12652            &handler,
12653            &module_ctx,
12654            &mut module_rx,
12655            hello_frame("aft", PROTOCOL_VERSION, 395),
12656        )
12657        .await;
12658
12659        let client_connection = ConnectionId::new(96);
12660        let (client_ctx, _client_rx) = route_ctx(client_connection);
12661        let capture = EventCapture::default();
12662        let _subscriber =
12663            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12664        let (route_task, bind) = relay_route_open(
12665            &handler,
12666            client_connection,
12667            &client_ctx.egress,
12668            &mut module_rx,
12669            396,
12670            "aft",
12671            "hostile-module-code",
12672        )
12673        .await;
12674        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12675        let rejection = Frame::build(
12676            FrameType::Error,
12677            control_flags(),
12678            0,
12679            0,
12680            bind.header.corr,
12681            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12682        )
12683        .unwrap();
12684        handler
12685            .handle_control_frame(&module_ctx, rejection)
12686            .await
12687            .unwrap();
12688
12689        let response = route_task.await.unwrap();
12690        assert_eq!(parse_error(&response[0])["code"], hostile_code);
12691        let counters = handler.counters().snapshot();
12692        assert_eq!(
12693            counters["route_open_refused_by_code"],
12694            json!({ "module_rejected": 1 })
12695        );
12696        assert!(counters["route_open_refused_by_code"]
12697            .get(hostile_code)
12698            .is_none());
12699
12700        let event = capture
12701            .events()
12702            .into_iter()
12703            .find(|event| {
12704                event.target == "control"
12705                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12706            })
12707            .expect("route.open module-rejection refusal event");
12708        let logged = event.fields.get("module_code").expect("module_code field");
12709        assert!(!logged.bytes().any(|byte| byte < 0x20));
12710        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12711    }
12712
12713    #[tokio::test]
12714    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12715        let registry = Arc::new(Registry::default());
12716        let supervisor_handle = SupervisorHandle::new();
12717        let missing_program = std::env::temp_dir().join(format!(
12718            "subc-route-open-missing-program-{}",
12719            std::process::id()
12720        ));
12721        let supervisor =
12722            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12723                .with_handle(supervisor_handle.clone());
12724        let module = supervisor
12725            .supervise_configured(
12726                ModuleSpec {
12727                    module_id: "failed".to_string(),
12728                    program: missing_program,
12729                    args: Vec::new(),
12730                    env: Vec::new(),
12731                    reserved: false,
12732                    reserved_prefixes: Vec::new(),
12733                    protocol: ModuleProtocol::Subc,
12734                    overlap: Default::default(),
12735                },
12736                true,
12737            )
12738            .unwrap();
12739        assert_eq!(module.state().unwrap(), ModuleState::Failed);
12740
12741        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12742        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12743        let response = handler
12744            .handle_control_frame(
12745                &ctx,
12746                route_open_frame(305, "failed", unique_project_root("failed")),
12747            )
12748            .await
12749            .unwrap();
12750
12751        assert_eq!(response[0].header.ty, FrameType::Error);
12752        let error = parse_error(&response[0]);
12753        assert_eq!(error["code"], "target_unavailable");
12754        assert!(error["message"]
12755            .as_str()
12756            .unwrap()
12757            .contains("state=failed, enabled=true, live=false"));
12758    }
12759
12760    #[tokio::test]
12761    async fn route_open_role_mismatch_remains_target_unavailable() {
12762        let registry = Arc::new(Registry::default());
12763        let handler = ControlHandler::new(Arc::clone(&registry));
12764        handler
12765            .handle_control(
12766                ConnectionId::new(41),
12767                non_routable_hello_frame_with_control_ops("health-only", 306, None),
12768            )
12769            .unwrap();
12770
12771        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12772        let response = handler
12773            .handle_control_frame(
12774                &ctx,
12775                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12776            )
12777            .await
12778            .unwrap();
12779
12780        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12781        assert!(parse_error(&response[0])["message"]
12782            .as_str()
12783            .unwrap()
12784            .contains("does not provide the requested target"));
12785    }
12786
12787    #[tokio::test]
12788    async fn route_open_inactive_registration_remains_target_unavailable() {
12789        let registry = Arc::new(Registry::default());
12790        let handler = ControlHandler::new(Arc::clone(&registry));
12791        handler
12792            .handle_control(
12793                ConnectionId::new(43),
12794                hello_frame("inactive", PROTOCOL_VERSION, 308),
12795            )
12796            .unwrap();
12797        assert!(registry
12798            .set_module_state_for_test("inactive", ChannelState::Closed)
12799            .unwrap());
12800
12801        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12802        let response = handler
12803            .handle_control_frame(
12804                &ctx,
12805                route_open_frame(309, "inactive", unique_project_root("inactive")),
12806            )
12807            .await
12808            .unwrap();
12809
12810        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12811        assert!(parse_error(&response[0])["message"]
12812            .as_str()
12813            .unwrap()
12814            .contains("is not active"));
12815    }
12816
12817    #[tokio::test]
12818    async fn late_health_reply_is_recorded_through_the_module_response_path() {
12819        let registry = Arc::new(Registry::default());
12820        let forwarding = Arc::new(ForwardingTable::default());
12821        let supervisor_handle = SupervisorHandle::new();
12822        let supervisor =
12823            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
12824                .with_forwarding(Arc::clone(&forwarding))
12825                .with_handle(supervisor_handle.clone());
12826        let module = supervisor
12827            .supervise_configured(
12828                crate::ModuleSpec {
12829                    module_id: "late-health-response".to_string(),
12830                    program: PathBuf::from("disabled-module"),
12831                    args: Vec::new(),
12832                    env: Vec::new(),
12833                    reserved: false,
12834                    reserved_prefixes: Vec::new(),
12835                    protocol: ModuleProtocol::Subc,
12836                    overlap: Default::default(),
12837                },
12838                false,
12839            )
12840            .unwrap();
12841        let handler =
12842            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12843                .with_supervisor(supervisor_handle);
12844        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12845        handler
12846            .handle_control_frame(
12847                &module_ctx,
12848                hello_frame_with_control_ops(
12849                    "late-health-response",
12850                    PROTOCOL_VERSION,
12851                    7,
12852                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12853                ),
12854            )
12855            .await
12856            .unwrap();
12857        let probe_started_at = Instant::now() - Duration::from_millis(80);
12858        let pending = forwarding
12859            .begin_health_probe_rpc_for(
12860                "late-health-response",
12861                MODULE_CONTROL_OP_HEALTH_CHECK,
12862                probe_started_at,
12863                Instant::now() - Duration::from_millis(1),
12864            )
12865            .unwrap();
12866        assert!(forwarding
12867            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12868            .unwrap());
12869
12870        let responses = handler
12871            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12872            .await
12873            .unwrap();
12874
12875        assert!(responses.is_empty());
12876        let health = module.status().unwrap().health;
12877        assert_eq!(health.late_answer_count, 1);
12878        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12879    }
12880
12881    #[tokio::test]
12882    async fn health_probe_timeout_and_module_death_are_typed() {
12883        let registry = Arc::new(Registry::default());
12884        let forwarding = Arc::new(ForwardingTable::default());
12885        let handler =
12886            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12887                .with_health_probe_timeout(Duration::from_millis(50));
12888        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12889        hello_via_sink(
12890            &handler,
12891            &module_ctx,
12892            &mut module_rx,
12893            hello_frame_with_control_ops(
12894                "aft",
12895                PROTOCOL_VERSION,
12896                7,
12897                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12898            ),
12899        )
12900        .await;
12901
12902        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12903        let responses = handler
12904            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12905            .await
12906            .unwrap();
12907        assert_eq!(responses[0].header.ty, FrameType::Error);
12908        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12909        let _ = module_rx.try_recv();
12910
12911        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12912        let health_handler = handler.clone();
12913        let death_task = tokio::spawn(async move {
12914            health_handler
12915                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12916                .await
12917                .unwrap()
12918        });
12919        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12920            .await
12921            .unwrap()
12922            .unwrap();
12923        handler
12924            .cleanup_connection(module_ctx.connection_id)
12925            .unwrap();
12926        let responses = death_task.await.unwrap();
12927        assert_eq!(responses[0].header.ty, FrameType::Error);
12928        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12929    }
12930
12931    #[test]
12932    fn hello_requires_exact_protocol_version() {
12933        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12934            let registry = Arc::new(Registry::default());
12935            let handler = ControlHandler::new(Arc::clone(&registry));
12936            let responses = handler
12937                .handle_control(
12938                    ConnectionId::new(connection),
12939                    hello_frame("aft", offered, 9),
12940                )
12941                .unwrap();
12942
12943            assert_eq!(responses.len(), 1);
12944            assert_eq!(responses[0].header.ty, FrameType::Error);
12945            let error = parse_error(&responses[0]);
12946            assert_eq!(error["code"], "version_unsupported");
12947            assert!(registry.get_module("aft").unwrap().is_none());
12948            assert_eq!(registry.active_registration_count().unwrap(), 0);
12949        }
12950    }
12951
12952    #[test]
12953    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12954        let registry = Arc::new(Registry::default());
12955        let forwarding = Arc::new(ForwardingTable::default());
12956        let handler =
12957            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12958        let module_connection = ConnectionId::new(301);
12959        let registration = registry
12960            .register_with_control_ops(
12961                manifest("aft-push", PROTOCOL_VERSION),
12962                PROTOCOL_VERSION,
12963                module_connection,
12964                module_baseline_control_ops(),
12965            )
12966            .unwrap();
12967        let (module_tx, _module_rx) = mpsc::channel(8);
12968        let endpoint = forwarding
12969            .register_module_connection(
12970                module_connection,
12971                "aft-push".to_string(),
12972                PROTOCOL_VERSION,
12973                manifest_concurrency(&registration.manifest),
12974                FrameSink::new(module_tx),
12975            )
12976            .unwrap();
12977
12978        // A push op this version does not know is ignored (forward-compat), not errored.
12979        let unknown = Frame::build(
12980            FrameType::Push,
12981            control_flags(),
12982            0,
12983            0,
12984            5,
12985            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12986        )
12987        .unwrap();
12988        let out = handler.handle_status_update(endpoint, unknown).unwrap();
12989        assert!(
12990            out.is_empty(),
12991            "unknown push op must be ignored, got {out:?}"
12992        );
12993
12994        // A malformed body for a KNOWN op is a real error worth surfacing.
12995        let malformed = Frame::build(
12996            FrameType::Push,
12997            control_flags(),
12998            0,
12999            0,
13000            6,
13001            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13002        )
13003        .unwrap();
13004        let out = handler.handle_status_update(endpoint, malformed).unwrap();
13005        assert_eq!(out.len(), 1);
13006        assert_eq!(out[0].header.ty, FrameType::Error);
13007        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13008    }
13009
13010    #[test]
13011    fn hello_rejected_when_connection_already_owns_client_routes() {
13012        let registry = Arc::new(Registry::default());
13013        let forwarding = Arc::new(ForwardingTable::default());
13014        let handler =
13015            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13016        // Commits a client route on connection 202 (bound to a module on conn 101).
13017        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
13018        let client_connection = ConnectionId::new(202);
13019
13020        // That same connection now tries to register as a module: rejected, so one
13021        // connection never holds both client-route and module-endpoint state.
13022        let responses = handler
13023            .handle_control(
13024                client_connection,
13025                hello_frame("aft-second", PROTOCOL_VERSION, 9),
13026            )
13027            .unwrap();
13028        assert_eq!(responses[0].header.ty, FrameType::Error);
13029        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13030        assert!(registry.get_module("aft-second").unwrap().is_none());
13031    }
13032
13033    #[tokio::test]
13034    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13035        let registry = Arc::new(Registry::default());
13036        let forwarding = Arc::new(ForwardingTable::default());
13037        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13038        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13039        hello_via_sink(
13040            &handler,
13041            &module_ctx,
13042            &mut module_rx,
13043            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13044        )
13045        .await;
13046        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13047        let pending = forwarding
13048            .begin_route_bind_relay_for_test(
13049                client_ctx.connection_id,
13050                client_ctx.egress.clone(),
13051                2,
13052                "alpha",
13053            )
13054            .unwrap();
13055        forwarding
13056            .complete_pending_relay(
13057                module_ctx.connection_id,
13058                pending.corr,
13059                RouteBindRelayOutcome::Accepted,
13060            )
13061            .unwrap();
13062        client_rx.try_recv().unwrap();
13063        for module_id in ["beta", "alpha"] {
13064            let replies = handler
13065                .handle_control_frame(
13066                    &module_ctx,
13067                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13068                )
13069                .await
13070                .unwrap();
13071            assert_eq!(replies.len(), 1, "second HELLO must be refused");
13072            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13073        }
13074        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13075        assert!(registry.get_module("beta").unwrap().is_none());
13076        assert!(matches!(
13077            forwarding
13078                .lookup_data_route(
13079                    client_ctx.connection_id,
13080                    pending.client_channel,
13081                    pending.client_epoch,
13082                )
13083                .unwrap(),
13084            DataRoute::Client(DataRouteState::Bound(_))
13085        ));
13086        assert!(handler
13087            .hello_launch_nonces
13088            .lock()
13089            .unwrap()
13090            .presented(module_ctx.connection_id, Some("alpha-nonce")));
13091        assert!(module_rx.try_recv().is_err());
13092    }
13093
13094    #[test]
13095    fn reserved_module_hello_requires_matching_launch_nonce() {
13096        let registry = Arc::new(Registry::default());
13097        let supervisor = SupervisorHandle::new();
13098        // The supervisor recorded the nonce it injected when it spawned the reserved
13099        // module; the HELLO verifier checks against the same shared handle.
13100        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13101        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13102
13103        // A HELLO with NO nonce is rejected.
13104        let no_nonce = handler
13105            .handle_control(
13106                ConnectionId::new(1),
13107                hello_frame("vault", PROTOCOL_VERSION, 1),
13108            )
13109            .unwrap();
13110        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13111        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13112        assert!(registry.get_module("vault").unwrap().is_none());
13113
13114        // A HELLO with the WRONG nonce is rejected.
13115        let wrong = handler
13116            .handle_control(
13117                ConnectionId::new(2),
13118                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13119            )
13120            .unwrap();
13121        assert_eq!(wrong[0].header.ty, FrameType::Error);
13122        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13123        assert!(registry.get_module("vault").unwrap().is_none());
13124
13125        // A HELLO with the CORRECT nonce registers.
13126        let ok = handler
13127            .handle_control(
13128                ConnectionId::new(3),
13129                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13130            )
13131            .unwrap();
13132        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13133        assert!(registry.get_module("vault").unwrap().is_some());
13134    }
13135
13136    #[test]
13137    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13138        let registry = Arc::new(Registry::default());
13139        let supervisor = SupervisorHandle::new();
13140        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13141        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13142        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13143
13144        let squat = handler
13145            .handle_control(
13146                ConnectionId::new(1),
13147                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13148            )
13149            .unwrap();
13150        assert_eq!(squat[0].header.ty, FrameType::Error);
13151        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13152        assert!(parse_error(&squat[0])["message"]
13153            .as_str()
13154            .unwrap()
13155            .contains("fed:"));
13156
13157        let accepted_peer = handler
13158            .handle_control(
13159                ConnectionId::new(2),
13160                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13161            )
13162            .unwrap();
13163        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13164
13165        let accepted_short = handler
13166            .handle_control(
13167                ConnectionId::new(3),
13168                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13169            )
13170            .unwrap();
13171        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13172
13173        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13174            let response = handler
13175                .handle_control(
13176                    ConnectionId::new(conn),
13177                    hello_frame(module_id, PROTOCOL_VERSION, conn),
13178                )
13179                .unwrap();
13180            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13181        }
13182    }
13183
13184    #[test]
13185    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13186        let registry = Arc::new(Registry::default());
13187        let supervisor = SupervisorHandle::new();
13188        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13189        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13190        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13191        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13192
13193        let owner_nonce = handler
13194            .handle_control(
13195                ConnectionId::new(1),
13196                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13197            )
13198            .unwrap();
13199        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13200        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13201        assert!(registry.get_module("fed:special").unwrap().is_none());
13202
13203        let exact_nonce = handler
13204            .handle_control(
13205                ConnectionId::new(2),
13206                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13207            )
13208            .unwrap();
13209        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13210        assert!(registry.get_module("fed:special").unwrap().is_some());
13211    }
13212
13213    #[test]
13214    fn non_reserved_module_ignores_launch_nonce() {
13215        let registry = Arc::new(Registry::default());
13216        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
13217        // registration succeeds whether a spawned process echoes a nonce or not.
13218        let handler = ControlHandler::new(Arc::clone(&registry));
13219        let no_nonce = handler
13220            .handle_control(
13221                ConnectionId::new(1),
13222                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13223            )
13224            .unwrap();
13225        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13226        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13227
13228        let echoed_nonce = handler
13229            .handle_control(
13230                ConnectionId::new(2),
13231                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13232            )
13233            .unwrap();
13234        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13235        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13236    }
13237
13238    #[test]
13239    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13240        let handler = ControlHandler::default();
13241        let conn = ConnectionId::new(1);
13242        let malformed = Frame::build(
13243            FrameType::Hello,
13244            control_flags(),
13245            0,
13246            0,
13247            3,
13248            b"{not json".to_vec(),
13249        )
13250        .unwrap();
13251
13252        let error = handler.handle_control(conn, malformed).unwrap();
13253        assert_eq!(error[0].header.ty, FrameType::Error);
13254        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13255
13256        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13257        let pong = handler.handle_control(conn, ping).unwrap();
13258        assert_eq!(pong[0].header.ty, FrameType::Pong);
13259        assert_eq!(pong[0].header.corr, 4);
13260    }
13261
13262    #[test]
13263    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13264        let registry = Arc::new(Registry::default());
13265        let handler = ControlHandler::new(Arc::clone(&registry));
13266
13267        handler
13268            .handle_control(
13269                ConnectionId::new(1),
13270                hello_frame("aft", PROTOCOL_VERSION, 1),
13271            )
13272            .unwrap();
13273        let duplicate = handler
13274            .handle_control(
13275                ConnectionId::new(2),
13276                hello_frame("aft", PROTOCOL_VERSION, 2),
13277            )
13278            .unwrap();
13279
13280        assert_eq!(duplicate[0].header.ty, FrameType::Error);
13281        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13282        let registration = registry.get_module("aft").unwrap().unwrap();
13283        assert_eq!(registration.connection_id, ConnectionId::new(1));
13284    }
13285
13286    #[test]
13287    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13288        let registry = Arc::new(Registry::default());
13289        let forwarding = Arc::new(ForwardingTable::default());
13290        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13291        let handler =
13292            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13293                .with_process_liveness(process_liveness);
13294        let (ctx, route_channel, route_epoch) =
13295            bind_liveness_route(&registry, &forwarding, "aft-dead");
13296        let responses = handler
13297            .handle_route_poll(
13298                &ctx,
13299                route_poll_frame(41, PollKind::Liveness, route_channel),
13300                route_channel,
13301                route_epoch,
13302                PollKind::Liveness,
13303            )
13304            .unwrap();
13305
13306        assert_eq!(responses.len(), 1);
13307        assert_eq!(responses[0].header.ty, FrameType::Response);
13308        assert_route_poll_liveness(&responses[0], false);
13309    }
13310
13311    #[test]
13312    fn liveness_poll_without_process_source_uses_bound_route() {
13313        let registry = Arc::new(Registry::default());
13314        let forwarding = Arc::new(ForwardingTable::default());
13315        let handler =
13316            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13317        let (ctx, route_channel, route_epoch) =
13318            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
13319        let responses = handler
13320            .handle_route_poll(
13321                &ctx,
13322                route_poll_frame(42, PollKind::Liveness, route_channel),
13323                route_channel,
13324                route_epoch,
13325                PollKind::Liveness,
13326            )
13327            .unwrap();
13328
13329        assert_route_poll_liveness(&responses[0], true);
13330    }
13331
13332    #[test]
13333    fn liveness_poll_untracked_process_source_uses_bound_route() {
13334        let registry = Arc::new(Registry::default());
13335        let forwarding = Arc::new(ForwardingTable::default());
13336        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13337        let handler =
13338            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13339                .with_process_liveness(process_liveness);
13340        let (ctx, route_channel, route_epoch) =
13341            bind_liveness_route(&registry, &forwarding, "aft-untracked");
13342        let responses = handler
13343            .handle_route_poll(
13344                &ctx,
13345                route_poll_frame(43, PollKind::Liveness, route_channel),
13346                route_channel,
13347                route_epoch,
13348                PollKind::Liveness,
13349            )
13350            .unwrap();
13351
13352        assert_route_poll_liveness(&responses[0], true);
13353    }
13354
13355    #[tokio::test]
13356    async fn unknown_op_returns_unknown_control_op() {
13357        let handler = ControlHandler::default();
13358        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13359        let request = Frame::build(
13360            FrameType::Request,
13361            control_flags(),
13362            0,
13363            0,
13364            55,
13365            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13366        )
13367        .unwrap();
13368
13369        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13370
13371        assert_eq!(response.len(), 1);
13372        assert_eq!(response[0].header.ty, FrameType::Error);
13373        assert_eq!(response[0].header.corr, 55);
13374        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13375    }
13376
13377    #[tokio::test]
13378    async fn supervisor_provenance_rejects_unknown_exact_module() {
13379        let handler = ControlHandler::default();
13380        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13381        let request = Frame::build(
13382            FrameType::Request,
13383            control_flags(),
13384            0,
13385            0,
13386            57,
13387            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13388        )
13389        .unwrap();
13390
13391        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13392
13393        assert_eq!(response.len(), 1);
13394        assert_eq!(response[0].header.ty, FrameType::Error);
13395        assert_eq!(response[0].header.corr, 57);
13396        let error = parse_error(&response[0]);
13397        assert_eq!(error["code"], "unknown_module");
13398        assert_eq!(error["message"], "module_id 'missing' is not supervised");
13399    }
13400
13401    #[test]
13402    fn provenance_probe_override_keeps_handler_tests_deterministic() {
13403        let expected = subc_control::RunningImageAgreement::Unavailable {
13404            reason: subc_control::RunningImageUnavailableReason::HashFailed,
13405        };
13406        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13407        assert_eq!(handler.provenance_probe_override, Some(expected));
13408    }
13409
13410    #[test]
13411    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13412        let verdict = reload_verdict(
13413            std::path::Path::new("/bin/new"),
13414            Some(std::path::Path::new("/bin/old")),
13415            subc_control::RunningImageAgreement::Unavailable {
13416                reason: subc_control::RunningImageUnavailableReason::HashFailed,
13417            },
13418        );
13419        assert!(matches!(
13420            verdict.path,
13421            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13422                if configured == std::path::Path::new("/bin/new")
13423                    && spawned_from == std::path::Path::new("/bin/old")
13424        ));
13425    }
13426
13427    #[test]
13428    fn launch_spec_without_running_digest_is_unavailable_not_match() {
13429        use subc_control::{
13430            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13431        };
13432        let configured = crate::supervise::ModuleSpec {
13433            module_id: "legacy".into(),
13434            program: "/module".into(),
13435            args: Vec::new(),
13436            env: Vec::new(),
13437            reserved: false,
13438            reserved_prefixes: Vec::new(),
13439            protocol: subc_control::ModuleProtocol::Subc,
13440            overlap: Default::default(),
13441        };
13442        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13443        assert_eq!(
13444            launch_spec_agreement(&configured, false, Some(&digest)),
13445            Agreement::Unavailable {
13446                reason: Reason::NotRunning
13447            }
13448        );
13449        assert_eq!(
13450            launch_spec_agreement(&configured, true, None),
13451            Agreement::Unavailable {
13452                reason: Reason::SpawnedSpecUnavailable
13453            }
13454        );
13455        assert_eq!(
13456            launch_spec_agreement(&configured, true, Some(&digest)),
13457            Agreement::Match
13458        );
13459    }
13460
13461    #[cfg(windows)]
13462    #[test]
13463    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13464        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13465        let configured = dir.join("module.exe");
13466        std::fs::write(&configured, b"executable path fixture").unwrap();
13467        let spawned = std::fs::canonicalize(&configured).unwrap();
13468        assert_ne!(
13469            configured, spawned,
13470            "fixture must exercise different path spellings"
13471        );
13472        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13473        let lower_case = configured.to_string_lossy().to_lowercase();
13474        for spelling in [
13475            &configured,
13476            Path::new(&forward_slashes),
13477            Path::new(&lower_case),
13478        ] {
13479            let verdict = reload_verdict(
13480                spelling,
13481                Some(&spawned),
13482                subc_control::RunningImageAgreement::Unavailable {
13483                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13484                },
13485            );
13486            assert_eq!(
13487                verdict.path,
13488                ReloadPathAgreement::Match,
13489                "configured {spelling:?}, spawned {spawned:?}"
13490            );
13491        }
13492    }
13493
13494    #[cfg(windows)]
13495    #[test]
13496    fn windows_reload_path_keeps_different_existing_program_pending() {
13497        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13498        let old = dir.join("old.exe");
13499        let new = dir.join("new.exe");
13500        std::fs::write(&old, b"same executable content").unwrap();
13501        std::fs::copy(&old, &new).unwrap();
13502        let spawned = std::fs::canonicalize(&old).unwrap();
13503        let image = subc_control::RunningImageAgreement::Unavailable {
13504            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13505        };
13506        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13507        assert!(
13508            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13509            if configured == new && spawned_from == spawned)
13510        );
13511        let missing = dir.join("missing.exe");
13512        assert!(matches!(
13513            reload_verdict(&missing, Some(&spawned), image).path,
13514            ReloadPathAgreement::Mismatch { .. }
13515        ));
13516    }
13517
13518    #[test]
13519    fn reload_verdict_detects_replaced_image_at_same_path() {
13520        let image = subc_control::RunningImageAgreement::Mismatch {
13521            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13522                digest: "old".into(),
13523            },
13524            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13525                digest: "new".into(),
13526            },
13527        };
13528        let verdict = reload_verdict(
13529            std::path::Path::new("/bin/same"),
13530            Some(std::path::Path::new("/bin/same")),
13531            image.clone(),
13532        );
13533        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13534        assert_eq!(verdict.image, image);
13535    }
13536
13537    #[test]
13538    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13539        let image = subc_control::RunningImageAgreement::Unavailable {
13540            reason: subc_control::RunningImageUnavailableReason::NotRunning,
13541        };
13542        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13543        assert_eq!(
13544            verdict.path,
13545            subc_control::ReloadPathAgreement::Unavailable {
13546                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13547            }
13548        );
13549        assert_eq!(verdict.image, image);
13550
13551        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13552            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13553        };
13554        let verdict = reload_verdict(
13555            std::path::Path::new("/bin/same"),
13556            Some(std::path::Path::new("/bin/same")),
13557            unconfirmed.clone(),
13558        );
13559        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13560        assert_eq!(verdict.image, unconfirmed);
13561    }
13562
13563    #[test]
13564    fn reload_verdict_preserves_each_image_unavailability_reason() {
13565        use subc_control::RunningImageUnavailableReason as Reason;
13566
13567        for reason in [
13568            Reason::NotRunning,
13569            Reason::UnsupportedPlatform,
13570            Reason::RunningExecutableUnreadable,
13571            Reason::SpawnedPathUnreadable,
13572            Reason::HashFailed,
13573            Reason::ProcessIdentityUnconfirmed,
13574            Reason::Unknown("future_probe_reason".to_string()),
13575        ] {
13576            let image = subc_control::RunningImageAgreement::Unavailable {
13577                reason: reason.clone(),
13578            };
13579            let verdict = reload_verdict(
13580                std::path::Path::new("/bin/same"),
13581                Some(std::path::Path::new("/bin/same")),
13582                image.clone(),
13583            );
13584            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13585            assert_eq!(verdict.image, image, "{reason:?}");
13586        }
13587    }
13588
13589    #[tokio::test]
13590    async fn malformed_control_bodies_return_invalid_control_body() {
13591        let handler = ControlHandler::default();
13592        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13593
13594        for (corr, body) in [
13595            (56, br#"{"route_channel":1}"#.as_slice()),
13596            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13597            (
13598                58,
13599                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13600            ),
13601        ] {
13602            let request = Frame::build(
13603                FrameType::Request,
13604                control_flags(),
13605                0,
13606                0,
13607                corr,
13608                body.to_vec(),
13609            )
13610            .unwrap();
13611            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13612
13613            assert_eq!(response.len(), 1);
13614            assert_eq!(response[0].header.ty, FrameType::Error);
13615            assert_eq!(response[0].header.corr, corr);
13616            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13617        }
13618    }
13619
13620    #[tokio::test]
13621    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13622        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13623        let registry = Arc::new(Registry::default());
13624        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
13625        let router = Router::with_control_handler(Arc::clone(&control));
13626        let connection = router.begin_connection();
13627        let (ctx, mut rx) = route_ctx(connection.id());
13628
13629        router
13630            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13631            .await
13632            .unwrap();
13633        let response = rx.recv().await.unwrap();
13634        let ack = parse_ack(&response);
13635        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13636        let channel = 1;
13637
13638        let goodbye =
13639            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13640        router.route_for_connection(&ctx, goodbye).await.unwrap();
13641        assert!(rx.try_recv().is_err());
13642        assert!(registry.get_module("aft").unwrap().is_none());
13643
13644        router
13645            .route_for_connection(&ctx, channel_request(channel, 13))
13646            .await
13647            .unwrap();
13648        let error_frame = rx.recv().await.unwrap();
13649        assert_eq!(error_frame.header.ty, FrameType::Error);
13650        assert_eq!(error_frame.header.channel, channel);
13651        let captured = crate::router::test_log::captured_logs(&logs);
13652        assert_eq!(
13653            captured
13654                .lines()
13655                .filter(|line| {
13656                    line.contains("module_id=aft")
13657                        && line.contains("reason=explicit_goodbye")
13658                        && line.contains("module registration ended")
13659                })
13660                .count(),
13661            1,
13662            "unexpected GOODBYE registry log: {captured}"
13663        );
13664    }
13665
13666    #[tokio::test]
13667    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13668        let registry = Arc::new(Registry::default());
13669        let handler = ControlHandler::new(registry).with_capability_config(
13670            [("prov".to_string(), true), ("cons".to_string(), true)],
13671            BTreeMap::new(),
13672        );
13673        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13674        register_capability_manifest(
13675            &handler,
13676            &provider_ctx,
13677            &mut provider_rx,
13678            capability_manifest("prov", &["thing/v1"], &[]),
13679            1,
13680        )
13681        .await;
13682        let mut consumer = capability_manifest("cons", &[], &[]);
13683        consumer.capabilities.as_mut().unwrap().requires.push(
13684            subc_protocol::manifest::CapabilityRequirement {
13685                capability: "thing/v1".to_string(),
13686                need: subc_protocol::manifest::CapabilityNeed::Required,
13687            },
13688        );
13689        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13690        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13691        assert_eq!(
13692            handler.capability_evaluator.verdict("cons", "thing/v1"),
13693            Some(CapabilityVerdict::Provided)
13694        );
13695        let (mut client_rx, _) = open_route_for_capability_test(
13696            &handler,
13697            &provider_ctx,
13698            &mut provider_rx,
13699            703,
13700            3,
13701            "prov",
13702            None,
13703        )
13704        .await;
13705        let goodbye =
13706            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13707        handler
13708            .handle_control_frame(&provider_ctx, goodbye)
13709            .await
13710            .unwrap();
13711        assert_eq!(
13712            handler.capability_evaluator.verdict("cons", "thing/v1"),
13713            Some(CapabilityVerdict::NeverProvided)
13714        );
13715        let closed = client_rx
13716            .try_recv()
13717            .expect("GOODBYE pushes route.closed before route GOODBYE");
13718        assert!(
13719            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13720            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13721        );
13722        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13723        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13724    }
13725
13726    struct RouteClosedFixture {
13727        handler: ControlHandler,
13728        #[cfg(unix)]
13729        supervisor: Supervisor,
13730        module_ctx: RouteCtx,
13731        #[cfg(unix)]
13732        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13733        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13734        route: Arc<crate::forwarding::RouteBinding>,
13735    }
13736
13737    async fn route_closed_fixture() -> RouteClosedFixture {
13738        let registry = Arc::new(Registry::default());
13739        let handler = ControlHandler::new(Arc::clone(&registry));
13740        #[cfg(unix)]
13741        let supervisor = Supervisor::new(registry, RestartPolicy::default())
13742            .with_forwarding(Arc::clone(&handler.forwarding));
13743        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13744        register_capability_manifest(
13745            &handler,
13746            &module_ctx,
13747            &mut module_rx,
13748            capability_manifest("prov", &[], &[]),
13749            1,
13750        )
13751        .await;
13752        let (client_rx, bind) = open_route_for_capability_test(
13753            &handler,
13754            &module_ctx,
13755            &mut module_rx,
13756            702,
13757            2,
13758            "prov",
13759            None,
13760        )
13761        .await;
13762        let ModuleControlRequest::RouteBind {
13763            route_channel: channel,
13764            epoch: route_epoch,
13765            ..
13766        } = bind
13767        else {
13768            panic!("expected route.bind");
13769        };
13770        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13771            handler
13772                .forwarding
13773                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13774                .unwrap()
13775        else {
13776            panic!("the acknowledged route must be bound");
13777        };
13778        RouteClosedFixture {
13779            handler,
13780            #[cfg(unix)]
13781            supervisor,
13782            module_ctx,
13783            #[cfg(unix)]
13784            module_rx,
13785            client_rx,
13786            route,
13787        }
13788    }
13789
13790    impl RouteClosedFixture {
13791        #[cfg(unix)]
13792        async fn cut(&mut self) {
13793            let notices = async {
13794                let mut draining = self.module_rx.recv().await.unwrap();
13795                assert!(matches!(
13796                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13797                        &draining.body
13798                    )
13799                    .unwrap(),
13800                    subc_protocol::session::ModuleControlCommand::Draining {
13801                        reason: RouteCloseReason::Restart,
13802                        ..
13803                    }
13804                ));
13805                draining.flushed.take().unwrap().send(()).unwrap();
13806                let mut closing = self.client_rx.recv().await.unwrap();
13807                assert!(matches!(
13808                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13809                    ClientControlPush::RouteClosing {
13810                        reason: RouteCloseReason::Restart,
13811                        ..
13812                    }
13813                ));
13814                closing.flushed.take().unwrap().send(()).unwrap();
13815            };
13816            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13817            result.unwrap();
13818        }
13819
13820        fn disconnect(&mut self) -> ClientControlPush {
13821            self.handler
13822                .cleanup_connection(self.module_ctx.connection_id)
13823                .unwrap();
13824            let closed = self.client_rx.try_recv().unwrap();
13825            assert_eq!(closed.header.ty, FrameType::Push);
13826            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13827            assert!(matches!(
13828                &push,
13829                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13830                    if channels == &[self.route.client_channel]
13831            ));
13832            assert_eq!(
13833                self.client_rx.try_recv().unwrap().header.ty,
13834                FrameType::Goodbye
13835            );
13836            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13837            push
13838        }
13839    }
13840
13841    #[cfg(unix)]
13842    #[tokio::test(start_paused = true)]
13843    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13844        let mut fixture = route_closed_fixture().await;
13845        fixture.cut().await;
13846        assert!(matches!(
13847            fixture.disconnect(),
13848            ClientControlPush::RouteClosed {
13849                reason: RouteCloseReason::Restart,
13850                drained: true,
13851                ..
13852            }
13853        ));
13854    }
13855
13856    #[cfg(unix)]
13857    #[tokio::test(start_paused = true)]
13858    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13859        let mut fixture = route_closed_fixture().await;
13860        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13861        let started = tokio::time::Instant::now();
13862        fixture.cut().await;
13863        assert!(started.elapsed() >= Duration::from_secs(2));
13864        assert_eq!(fixture.route.flow.in_flight(), 1);
13865        assert!(matches!(
13866            fixture.disconnect(),
13867            ClientControlPush::RouteClosed {
13868                reason: RouteCloseReason::Restart,
13869                drained: false,
13870                ..
13871            }
13872        ));
13873    }
13874
13875    #[tokio::test]
13876    async fn module_connection_crash_reports_crash_and_not_drained() {
13877        let mut fixture = route_closed_fixture().await;
13878        // The server calls cleanup_connection on module EOF. Without a planned
13879        // daemon shutdown, that disconnect must remain a crash with no prior
13880        // route.closing notice, even when the route has no in-flight calls.
13881        assert!(matches!(
13882            fixture.disconnect(),
13883            ClientControlPush::RouteClosed {
13884                reason: RouteCloseReason::Crash,
13885                drained: false,
13886                ..
13887            }
13888        ));
13889    }
13890
13891    #[tokio::test]
13892    async fn dropping_router_connection_releases_registration() {
13893        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13894        let registry = Arc::new(Registry::default());
13895        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
13896        let router = Router::with_control_handler(Arc::clone(&control));
13897        let connection = router.begin_connection();
13898        let connection_id = connection.id();
13899        let (ctx, mut rx) = route_ctx(connection_id);
13900
13901        router
13902            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13903            .await
13904            .unwrap();
13905        let response = rx.recv().await.unwrap();
13906        let ack = parse_ack(&response);
13907        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13908        assert!(registry.get_module("aft").unwrap().is_some());
13909
13910        drop(connection);
13911
13912        assert!(registry.get_module("aft").unwrap().is_none());
13913        assert_eq!(registry.active_registration_count().unwrap(), 0);
13914
13915        control
13916            .cleanup_connection(ConnectionId::new(u64::MAX))
13917            .unwrap();
13918        let captured = crate::router::test_log::captured_logs(&logs);
13919        println!("captured registry lifecycle logs:\n{captured}");
13920        let events: Vec<_> = captured
13921            .lines()
13922            .filter(|line| {
13923                line.contains("module registered module_id=aft ")
13924                    || line.contains("module registration ended")
13925            })
13926            .collect();
13927        assert_eq!(
13928            events.len(),
13929            2,
13930            "unexpected registry lifecycle logs: {captured}"
13931        );
13932        assert!(events[0].contains("module registered module_id=aft "));
13933        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13934        assert!(events[1].contains(&format!(
13935            "module_id=aft connection_id={}",
13936            connection_id.get()
13937        )));
13938        assert!(events[1].contains("reason=connection_closed"));
13939        assert!(events[1].contains("module registration ended"));
13940        assert_eq!(
13941            captured
13942                .lines()
13943                .filter(|line| line.contains("module registered module_id=aft "))
13944                .count(),
13945            1,
13946            "legacy registration admission line must appear once: {captured}"
13947        );
13948    }
13949
13950    #[tokio::test]
13951    async fn hello_registration_keeps_legacy_module_registered_line_once() {
13952        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13953        let registry = Arc::new(Registry::default());
13954        let control = ControlHandler::new(Arc::clone(&registry));
13955        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13956        hello_via_sink(
13957            &control,
13958            &ctx,
13959            &mut rx,
13960            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13961        )
13962        .await;
13963
13964        let captured = crate::router::test_log::captured_logs(&logs);
13965        let admissions: Vec<_> = captured
13966            .lines()
13967            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13968            .collect();
13969        assert_eq!(
13970            admissions.len(),
13971            1,
13972            "legacy admission line must remain exactly once: {captured}"
13973        );
13974        assert!(admissions[0].contains("routable_provider=true"));
13975        assert!(admissions[0].contains("connection_id=777"));
13976    }
13977
13978    fn capability_manifest(
13979        module_id: &str,
13980        provides: &[&str],
13981        must_never_reach: &[&str],
13982    ) -> ModuleManifest {
13983        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13984        manifest.capabilities = Some(CapabilityDeclarations {
13985            provides: provides
13986                .iter()
13987                .map(|capability| (*capability).to_string())
13988                .collect(),
13989            requires: Vec::new(),
13990            must_never_reach: must_never_reach
13991                .iter()
13992                .map(|capability| (*capability).to_string())
13993                .collect(),
13994        });
13995        manifest
13996    }
13997
13998    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13999        Frame::build(
14000            FrameType::Hello,
14001            control_flags(),
14002            0,
14003            0,
14004            corr,
14005            serde_json::to_vec(&ModuleHelloBody {
14006                protocol_ver: manifest.protocol_ver,
14007                manifest,
14008                control_ops: None,
14009                launch_nonce: None,
14010            })
14011            .expect("capability test HELLO serializes"),
14012        )
14013        .expect("capability test HELLO frame builds")
14014    }
14015
14016    fn catalog_update_with_capabilities_frame(
14017        corr: u64,
14018        capabilities: CapabilityDeclarations,
14019    ) -> Frame {
14020        Frame::build(
14021            FrameType::Request,
14022            control_flags(),
14023            0,
14024            0,
14025            corr,
14026            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14027                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14028                capabilities: Some(capabilities),
14029                ready: None,
14030            })
14031            .expect("capability catalog.update serializes"),
14032        )
14033        .expect("capability catalog.update frame builds")
14034    }
14035
14036    async fn register_capability_manifest(
14037        handler: &ControlHandler,
14038        ctx: &RouteCtx,
14039        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14040        manifest: ModuleManifest,
14041        corr: u64,
14042    ) {
14043        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14044    }
14045
14046    async fn open_route_for_capability_test(
14047        handler: &ControlHandler,
14048        target_ctx: &RouteCtx,
14049        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14050        client_connection_id: u64,
14051        corr: u64,
14052        target_module_id: &str,
14053        consumer_identity: Option<ConsumerIdentity>,
14054    ) -> (
14055        mpsc::Receiver<crate::router::OutboundFrame>,
14056        ModuleControlRequest,
14057    ) {
14058        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14059        let route_handler = handler.clone();
14060        let target_module_id = target_module_id.to_string();
14061        let route_task = tokio::spawn(async move {
14062            route_handler
14063                .handle_control_frame(
14064                    &client_ctx,
14065                    route_open_frame_with_admission_facts(
14066                        corr,
14067                        &target_module_id,
14068                        unique_project_root("admission-facts"),
14069                        consumer_identity,
14070                        None,
14071                    ),
14072                )
14073                .await
14074                .expect("capability test route.open succeeds")
14075        });
14076        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14077            .await
14078            .expect("capability test route.open must reach route.bind")
14079            .expect("target control receiver stays open");
14080        let bind_request: ModuleControlRequest =
14081            serde_json::from_slice(&bind.body).expect("route.bind decodes");
14082        handler
14083            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14084            .await
14085            .expect("capability test route.bind ACK succeeds");
14086        assert!(route_task.await.expect("route.open task joins").is_empty());
14087        let opened = client_rx
14088            .recv()
14089            .await
14090            .expect("successful route.open publishes a response");
14091        assert!(matches!(
14092            serde_json::from_slice::<ClientControlResponse>(&opened.body),
14093            Ok(ClientControlResponse::RouteOpen { .. })
14094        ));
14095        (client_rx, bind_request)
14096    }
14097
14098    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14099        assert_eq!(frame.header.ty, FrameType::Push);
14100        assert_eq!(frame.header.channel, 0);
14101        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14102            .expect("route.closed control push decodes");
14103        let ClientControlPush::RouteClosed { channels, .. } = &push else {
14104            panic!("expected route.closed");
14105        };
14106        assert_eq!(channels.len(), 1, "exactly one violating route closed");
14107        let channels = channels.clone();
14108        assert_eq!(
14109            push,
14110            ClientControlPush::RouteClosed {
14111                module_id: target_module_id.to_string(),
14112                channels,
14113                reason: RouteCloseReason::CapabilityDenied,
14114                drained: false,
14115                abandoned: 0,
14116                excluded_subscriptions: 0,
14117                terminal: Some(false),
14118            }
14119        );
14120    }
14121
14122    #[tokio::test]
14123    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14124        let registry = Arc::new(Registry::default());
14125        let forwarding = Arc::new(ForwardingTable::default());
14126        let supervisor = SupervisorHandle::new();
14127        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14128        let handler =
14129            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14130                .with_supervisor(supervisor);
14131        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14132        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14133        register_capability_manifest(
14134            &handler,
14135            &target_ctx,
14136            &mut target_rx,
14137            capability_manifest("target", &["credentials-provider/v1"], &[]),
14138            1,
14139        )
14140        .await;
14141        register_capability_manifest(
14142            &handler,
14143            &opener_ctx,
14144            &mut opener_rx,
14145            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14146            2,
14147        )
14148        .await;
14149
14150        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14151        let replies = handler
14152            .handle_control_frame(
14153                &client_ctx,
14154                route_open_frame_with_admission_facts(
14155                    3,
14156                    "target",
14157                    unique_project_root("admission-facts"),
14158                    Some(ConsumerIdentity {
14159                        module_id: "opener".to_string(),
14160                        launch_nonce: "opener-nonce".to_string(),
14161                    }),
14162                    None,
14163                ),
14164            )
14165            .await
14166            .expect("denied route.open returns a typed frame");
14167        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14168        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14169        assert!(
14170            target_rx.try_recv().is_err(),
14171            "forbidden route.open must not relay route.bind"
14172        );
14173    }
14174
14175    #[tokio::test]
14176    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14177        let registry = Arc::new(Registry::default());
14178        let forwarding = Arc::new(ForwardingTable::default());
14179        let supervisor = SupervisorHandle::new();
14180        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14181        let handler =
14182            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14183                .with_supervisor(supervisor);
14184        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14185        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14186        register_capability_manifest(
14187            &handler,
14188            &target_ctx,
14189            &mut target_rx,
14190            capability_manifest("target", &["credentials-provider/v1"], &[]),
14191            1,
14192        )
14193        .await;
14194        register_capability_manifest(
14195            &handler,
14196            &old_opener_ctx,
14197            &mut old_opener_rx,
14198            capability_manifest("opener", &[], &[]),
14199            2,
14200        )
14201        .await;
14202        let (mut client_rx, _) = open_route_for_capability_test(
14203            &handler,
14204            &target_ctx,
14205            &mut target_rx,
14206            712,
14207            3,
14208            "target",
14209            Some(ConsumerIdentity {
14210                module_id: "opener".to_string(),
14211                launch_nonce: "opener-nonce".to_string(),
14212            }),
14213        )
14214        .await;
14215        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14216
14217        handler
14218            .cleanup_connection(old_opener_ctx.connection_id)
14219            .expect("old opener registration cleans up");
14220        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14221        register_capability_manifest(
14222            &handler,
14223            &new_opener_ctx,
14224            &mut new_opener_rx,
14225            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14226            4,
14227        )
14228        .await;
14229
14230        assert_capability_denied_push(
14231            client_rx
14232                .try_recv()
14233                .expect("HELLO deny addition must emit route.closed")
14234                .frame,
14235            "target",
14236        );
14237        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14238        assert!(matches!(
14239            target_rx.try_recv(),
14240            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14241        ));
14242    }
14243
14244    #[tokio::test]
14245    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14246        let registry = Arc::new(Registry::default());
14247        let forwarding = Arc::new(ForwardingTable::default());
14248        let supervisor = SupervisorHandle::new();
14249        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14250        let handler =
14251            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14252                .with_supervisor(supervisor);
14253        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14254        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14255        register_capability_manifest(
14256            &handler,
14257            &target_ctx,
14258            &mut target_rx,
14259            capability_manifest("target", &[], &[]),
14260            1,
14261        )
14262        .await;
14263        register_capability_manifest(
14264            &handler,
14265            &opener_ctx,
14266            &mut opener_rx,
14267            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14268            2,
14269        )
14270        .await;
14271        let (mut client_rx, _) = open_route_for_capability_test(
14272            &handler,
14273            &target_ctx,
14274            &mut target_rx,
14275            722,
14276            3,
14277            "target",
14278            Some(ConsumerIdentity {
14279                module_id: "opener".to_string(),
14280                launch_nonce: "opener-nonce".to_string(),
14281            }),
14282        )
14283        .await;
14284        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14285
14286        let replies = handler
14287            .handle_control_frame(
14288                &target_ctx,
14289                catalog_update_with_capabilities_frame(
14290                    4,
14291                    CapabilityDeclarations {
14292                        provides: vec!["credentials-provider/v1".to_string()],
14293                        requires: Vec::new(),
14294                        must_never_reach: Vec::new(),
14295                    },
14296                ),
14297            )
14298            .await
14299            .expect("claim catalog.update succeeds");
14300        assert!(matches!(
14301            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14302            Ok(ModuleControlResponseToModule::CatalogUpdate {})
14303        ));
14304        assert_capability_denied_push(
14305            client_rx
14306                .try_recv()
14307                .expect("claim addition must emit route.closed")
14308                .frame,
14309            "target",
14310        );
14311        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14312        assert!(matches!(
14313            target_rx.try_recv(),
14314            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14315        ));
14316    }
14317
14318    #[tokio::test]
14319    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14320        let registry = Arc::new(Registry::default());
14321        let forwarding = Arc::new(ForwardingTable::default());
14322        let supervisor = SupervisorHandle::new();
14323        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14324        let handler =
14325            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14326                .with_supervisor(supervisor);
14327        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14328        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14329        register_capability_manifest(
14330            &handler,
14331            &target_ctx,
14332            &mut target_rx,
14333            capability_manifest("target", &["credentials-provider/v1"], &[]),
14334            1,
14335        )
14336        .await;
14337        register_capability_manifest(
14338            &handler,
14339            &opener_ctx,
14340            &mut opener_rx,
14341            capability_manifest("opener", &[], &[]),
14342            2,
14343        )
14344        .await;
14345        let (mut client_rx, _) = open_route_for_capability_test(
14346            &handler,
14347            &target_ctx,
14348            &mut target_rx,
14349            732,
14350            3,
14351            "target",
14352            Some(ConsumerIdentity {
14353                module_id: "opener".to_string(),
14354                launch_nonce: "opener-nonce".to_string(),
14355            }),
14356        )
14357        .await;
14358        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14359
14360        handler
14361            .handle_control_frame(
14362                &target_ctx,
14363                catalog_update_with_capabilities_frame(
14364                    4,
14365                    CapabilityDeclarations {
14366                        provides: Vec::new(),
14367                        requires: Vec::new(),
14368                        must_never_reach: Vec::new(),
14369                    },
14370                ),
14371            )
14372            .await
14373            .expect("claim removal catalog.update succeeds");
14374        assert_eq!(
14375            forwarding.active_binding_count().unwrap(),
14376            1,
14377            "removing an attested target claim must leave the route census unchanged"
14378        );
14379        assert!(
14380            client_rx.try_recv().is_err(),
14381            "claim removal must not emit route.closed capability_denied"
14382        );
14383        assert!(
14384            target_rx.try_recv().is_err(),
14385            "claim removal must not send the target a route GOODBYE"
14386        );
14387    }
14388
14389    /// A direct client may open a route to a denied capability provider; this
14390    /// policy applies only to attested supervised module origins, not to direct clients.
14391    #[tokio::test]
14392    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14393        let registry = Arc::new(Registry::default());
14394        let forwarding = Arc::new(ForwardingTable::default());
14395        let supervisor = SupervisorHandle::new();
14396        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14397        let handler =
14398            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14399                .with_supervisor(supervisor);
14400        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14401        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14402        register_capability_manifest(
14403            &handler,
14404            &target_ctx,
14405            &mut target_rx,
14406            capability_manifest("target", &["credentials-provider/v1"], &[]),
14407            1,
14408        )
14409        .await;
14410        register_capability_manifest(
14411            &handler,
14412            &opener_ctx,
14413            &mut opener_rx,
14414            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14415            2,
14416        )
14417        .await;
14418
14419        let (_client_rx, bind) = open_route_for_capability_test(
14420            &handler,
14421            &target_ctx,
14422            &mut target_rx,
14423            742,
14424            3,
14425            "target",
14426            None,
14427        )
14428        .await;
14429        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14430            panic!("direct scope-honesty route must bind");
14431        };
14432        assert_eq!(principal, Some(Principal::Direct));
14433        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14434    }
14435
14436    /// A module that denies a capability receives no self-route exemption when it
14437    /// also attestedly provides that capability.
14438    #[tokio::test]
14439    async fn must_never_reach_self_route_is_capability_forbidden() {
14440        let registry = Arc::new(Registry::default());
14441        let forwarding = Arc::new(ForwardingTable::default());
14442        let supervisor = SupervisorHandle::new();
14443        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14444        let handler =
14445            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14446                .with_supervisor(supervisor);
14447        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14448        register_capability_manifest(
14449            &handler,
14450            &self_ctx,
14451            &mut self_rx,
14452            capability_manifest(
14453                "self-provider",
14454                &["credentials-provider/v1"],
14455                &["credentials-provider/v1"],
14456            ),
14457            1,
14458        )
14459        .await;
14460
14461        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14462        let replies = handler
14463            .handle_control_frame(
14464                &client_ctx,
14465                route_open_frame_with_admission_facts(
14466                    2,
14467                    "self-provider",
14468                    unique_project_root("admission-facts"),
14469                    Some(ConsumerIdentity {
14470                        module_id: "self-provider".to_string(),
14471                        launch_nonce: "self-nonce".to_string(),
14472                    }),
14473                    None,
14474                ),
14475            )
14476            .await
14477            .expect("self-route refusal returns a typed frame");
14478        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14479        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14480        assert!(
14481            self_rx.try_recv().is_err(),
14482            "self denial must not relay route.bind"
14483        );
14484    }
14485
14486    #[test]
14487    fn unsupported_channel_zero_frame_returns_error() {
14488        let handler = ControlHandler::default();
14489        let request = Frame::build(
14490            FrameType::Request,
14491            control_flags(),
14492            0,
14493            0,
14494            21,
14495            b"opaque".to_vec(),
14496        )
14497        .unwrap();
14498
14499        let response = handler
14500            .handle_control(ConnectionId::new(1), request)
14501            .unwrap();
14502
14503        assert_eq!(response[0].header.ty, FrameType::Error);
14504        assert_eq!(
14505            parse_error(&response[0])["code"],
14506            "unsupported_control_frame"
14507        );
14508    }
14509
14510    /// Blue/green swap at the control-plane boundary. The supervisor that opens
14511    /// a swap is not wired yet, so the candidate is registered here directly
14512    /// into the registry and forwarding candidate slots, the way the swap's
14513    /// HELLO admission will.
14514    mod swap {
14515        use super::*;
14516
14517        const INCUMBENT: ConnectionId = ConnectionId::new(30);
14518        const CANDIDATE: ConnectionId = ConnectionId::new(40);
14519
14520        struct Swap {
14521            registry: Arc<Registry>,
14522            forwarding: Arc<ForwardingTable>,
14523            handler: ControlHandler,
14524            incumbent_ctx: RouteCtx,
14525            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14526            candidate_ctx: RouteCtx,
14527            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14528        }
14529
14530        async fn swap_with_incumbent() -> Swap {
14531            let registry = Arc::new(Registry::default());
14532            let forwarding = Arc::new(ForwardingTable::default());
14533            let handler =
14534                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14535            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14536            hello_via_sink(
14537                &handler,
14538                &incumbent_ctx,
14539                &mut incumbent_rx,
14540                hello_frame("aft", PROTOCOL_VERSION, 7),
14541            )
14542            .await;
14543            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14544            Swap {
14545                registry,
14546                forwarding,
14547                handler,
14548                incumbent_ctx,
14549                incumbent_rx,
14550                candidate_ctx,
14551                candidate_rx,
14552            }
14553        }
14554
14555        fn register_candidate(swap: &Swap, ready: Option<bool>) {
14556            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14557            candidate_manifest.ready = ready;
14558            let registration = swap
14559                .registry
14560                .register_candidate_with_control_ops(
14561                    candidate_manifest,
14562                    PROTOCOL_VERSION,
14563                    CANDIDATE,
14564                    module_baseline_control_ops(),
14565                )
14566                .unwrap();
14567            swap.forwarding
14568                .register_candidate_module_connection(
14569                    CANDIDATE,
14570                    "aft".to_string(),
14571                    PROTOCOL_VERSION,
14572                    manifest_concurrency(&registration.manifest),
14573                    swap.candidate_ctx.egress.clone(),
14574                )
14575                .unwrap();
14576        }
14577
14578        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14579            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14580            swap.registry.promote_candidate("aft").unwrap().unwrap();
14581            cutover.incumbent.unwrap()
14582        }
14583
14584        /// A swap candidate's HELLO reserves its candidate registry entry
14585        /// before installing its candidate forwarding endpoint. The swap
14586        /// supervisor promotes a ready candidate as soon as it reads it as
14587        /// registered, and promotion cuts forwarding over to the candidate
14588        /// endpoint, so the lookup the supervisor reads
14589        /// (`Registry::published_registration`) must not return the candidate
14590        /// until that endpoint exists. Otherwise the cutover finds no candidate
14591        /// endpoint and the swap fails as if the candidate had disconnected.
14592        #[tokio::test]
14593        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14594            let mut swap = swap_with_incumbent().await;
14595            let hold = Arc::new(std::sync::Barrier::new(2));
14596            let handler = swap
14597                .handler
14598                .clone()
14599                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14600            handler
14601                .supervisor
14602                .open_swap("aft", "candidate-nonce".to_string());
14603            let hello = {
14604                let handler = handler.clone();
14605                let candidate_ctx = swap.candidate_ctx.clone();
14606                std::thread::spawn(move || {
14607                    handler
14608                        .handle_hello(
14609                            candidate_ctx.connection_id,
14610                            Some(candidate_ctx.egress.clone()),
14611                            hello_frame_with_nonce(
14612                                "aft",
14613                                PROTOCOL_VERSION,
14614                                8,
14615                                Some("candidate-nonce"),
14616                            ),
14617                        )
14618                        .unwrap()
14619                })
14620            };
14621
14622            hold.wait();
14623            // Parked: the candidate slot is claimed but not published, and
14624            // forwarding has no candidate endpoint.
14625            assert!(swap
14626                .registry
14627                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14628                .unwrap()
14629                .is_some());
14630            assert!(swap
14631                .registry
14632                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14633                .unwrap()
14634                .is_none());
14635            assert!(swap
14636                .forwarding
14637                .module_endpoint_for_connection(CANDIDATE)
14638                .unwrap()
14639                .is_none());
14640
14641            hold.wait();
14642            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14643            assert_eq!(
14644                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14645                FrameType::HelloAck
14646            );
14647            let published = swap
14648                .registry
14649                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14650                .unwrap()
14651                .expect("candidate is published once its HELLO completes");
14652            assert_eq!(published.connection_id, CANDIDATE);
14653            assert!(swap
14654                .forwarding
14655                .module_endpoint_for_connection(CANDIDATE)
14656                .unwrap()
14657                .is_some());
14658        }
14659
14660        fn keyed_total(counters: &Value, key: &str) -> u64 {
14661            counters[key]
14662                .as_object()
14663                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14664                .unwrap_or(0)
14665        }
14666
14667        #[tokio::test]
14668        async fn replacement_logs_old_end_and_new_admission_once() {
14669            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14670            let mut swap = swap_with_incumbent().await;
14671            swap.handler
14672                .supervisor
14673                .open_swap("aft", "candidate-nonce".to_string());
14674            hello_via_sink(
14675                &swap.handler,
14676                &swap.candidate_ctx,
14677                &mut swap.candidate_rx,
14678                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14679            )
14680            .await;
14681            cutover(&swap);
14682            swap.handler.cleanup_connection(INCUMBENT).unwrap();
14683
14684            let captured = crate::router::test_log::captured_logs(&logs);
14685            println!("captured replacement registry logs:\n{captured}");
14686            let new_admissions: Vec<_> = captured
14687                .lines()
14688                .filter(|line| {
14689                    line.contains("connection_id=40")
14690                        && line.contains("swap candidate registered; not routable until cutover")
14691                })
14692                .collect();
14693            assert_eq!(
14694                new_admissions.len(),
14695                1,
14696                "unexpected admission logs: {captured}"
14697            );
14698            assert!(new_admissions[0].contains("module_id=aft"));
14699            assert!(new_admissions[0].contains("connection_id=40"));
14700            assert!(
14701                new_admissions[0].contains("swap candidate registered; not routable until cutover")
14702            );
14703            assert!(new_admissions[0].contains("ready=true"));
14704
14705            let old_admissions: Vec<_> = captured
14706                .lines()
14707                .filter(|line| {
14708                    line.contains("module registered module_id=aft ")
14709                        && line.contains("connection_id=30")
14710                })
14711                .collect();
14712            assert_eq!(
14713                old_admissions.len(),
14714                1,
14715                "unexpected incumbent admission: {captured}"
14716            );
14717
14718            let old_ends: Vec<_> = captured
14719                .lines()
14720                .filter(|line| {
14721                    line.contains("connection_id=30") && line.contains("module registration ended")
14722                })
14723                .collect();
14724            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14725            assert!(old_ends[0].contains("module_id=aft"));
14726            assert!(old_ends[0].contains("connection_id=30"));
14727            assert!(old_ends[0].contains("reason=replaced"));
14728            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14729            assert_eq!(
14730                captured
14731                    .lines()
14732                    .filter(|line| line.contains("module registration promoted"))
14733                    .count(),
14734                1,
14735                "unexpected promotion logs: {captured}"
14736            );
14737        }
14738
14739        /// An ack from the incumbent for a bind it was sent before cutover,
14740        /// arriving before the incumbent is drained. The incumbent is the live
14741        /// connection carrying every other client's routes, so the ack must
14742        /// not end it: the waiting client is told to retry, the reservation is
14743        /// given back, and the incumbent is told to drop just that binding.
14744        #[tokio::test]
14745        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14746            let mut swap = swap_with_incumbent().await;
14747            let handler = swap.handler.clone();
14748
14749            // A co-tenant route, bound on the incumbent before the swap.
14750            let cotenant = ConnectionId::new(31);
14751            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14752            let (cotenant_task, cotenant_bind) = relay_route_open(
14753                &handler,
14754                cotenant,
14755                &cotenant_ctx.egress,
14756                &mut swap.incumbent_rx,
14757                100,
14758                "aft",
14759                "swap-cotenant",
14760            )
14761            .await;
14762            handler
14763                .handle_control_frame(
14764                    &swap.incumbent_ctx,
14765                    route_bind_ack(cotenant_bind.header.corr),
14766                )
14767                .await
14768                .unwrap();
14769            assert!(cotenant_task.await.unwrap().is_empty());
14770            let (cotenant_channel, cotenant_epoch) =
14771                published_route(&cotenant_rx.recv().await.unwrap());
14772
14773            // A second route.open, relayed to the incumbent and not yet acked.
14774            let caller = ConnectionId::new(32);
14775            let (caller_ctx, mut caller_rx) = route_ctx(caller);
14776            let (caller_task, caller_bind) = relay_route_open(
14777                &handler,
14778                caller,
14779                &caller_ctx.egress,
14780                &mut swap.incumbent_rx,
14781                101,
14782                "aft",
14783                "swap-caller",
14784            )
14785            .await;
14786            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14787
14788            register_candidate(&swap, None);
14789            cutover(&swap);
14790
14791            // The incumbent acks after promotion and before any drain.
14792            let ack = handler
14793                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14794                .await;
14795            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14796            if module_loop_error.is_some() {
14797                // What the connection loop does with an untranslated router
14798                // error: end the connection, releasing every route on it.
14799                handler.cleanup_connection(INCUMBENT).unwrap();
14800            }
14801
14802            // 1. The incumbent's other routes survive.
14803            assert!(
14804                cotenant_rx.try_recv().is_err(),
14805                "the co-tenant route on the incumbent was torn down by one late ack: \
14806                 {module_loop_error:?}"
14807            );
14808            assert!(matches!(
14809                swap.forwarding
14810                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14811                    .unwrap(),
14812                DataRoute::Client(DataRouteState::Bound(_))
14813            ));
14814            assert_eq!(module_loop_error, None);
14815            assert!(swap
14816                .registry
14817                .get_module_by_connection(INCUMBENT)
14818                .unwrap()
14819                .is_some());
14820
14821            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
14822            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14823                .await
14824                .expect("the incumbent is told to drop the abandoned binding")
14825                .unwrap()
14826                .frame;
14827            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14828            assert_eq!(goodbye.header.channel, abandoned_channel);
14829            assert_eq!(goodbye.header.epoch, abandoned_epoch);
14830            assert!(swap.incumbent_rx.try_recv().is_err());
14831
14832            // 3. The waiting client gets a retryable refusal and no route.
14833            let response = caller_task.await.unwrap();
14834            assert_eq!(response.len(), 1);
14835            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14836            assert!(caller_rx.try_recv().is_err());
14837
14838            // 4. The reservation pair is given back, and the pending bind
14839            //    settled exactly once: one accepted open (the co-tenant) and one
14840            //    refused open (the caller), nothing counted twice.
14841            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14842            let counters = handler.counters().snapshot();
14843            assert_eq!(
14844                keyed_total(&counters, "route_open_accepted_by_principal"),
14845                1
14846            );
14847            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14848            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14849        }
14850
14851        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
14852        /// id would resolve to the promoted candidate and every new route.open
14853        /// would be refused as reloading, leaving neither process routable.
14854        #[tokio::test]
14855        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14856            let mut swap = swap_with_incumbent().await;
14857            register_candidate(&swap, None);
14858            let incumbent = cutover(&swap);
14859            swap.forwarding
14860                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14861                .unwrap()
14862                .expect("the incumbent is still registered");
14863
14864            let client = ConnectionId::new(33);
14865            let (client_ctx, mut client_rx) = route_ctx(client);
14866            let route_handler = swap.handler.clone();
14867            let open_ctx = RouteCtx {
14868                connection_id: client,
14869                egress: client_ctx.egress.clone(),
14870            };
14871            let mut route_task = tokio::spawn(async move {
14872                route_handler
14873                    .handle_control_frame(
14874                        &open_ctx,
14875                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14876                    )
14877                    .await
14878                    .unwrap()
14879            });
14880            let bind = tokio::select! {
14881                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14882                response = &mut route_task => {
14883                    let response = response.unwrap();
14884                    panic!(
14885                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
14886                        parse_error(&response[0])["code"]
14887                    );
14888                }
14889            };
14890            swap.handler
14891                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14892                .await
14893                .unwrap();
14894            assert!(route_task.await.unwrap().is_empty());
14895            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14896            match swap
14897                .forwarding
14898                .lookup_data_route(client, channel, epoch)
14899                .unwrap()
14900            {
14901                DataRoute::Client(DataRouteState::Bound(route)) => {
14902                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14903                }
14904                other => panic!("expected a bound route on the candidate, got {other:?}"),
14905            }
14906            assert!(swap.incumbent_rx.try_recv().is_err());
14907        }
14908
14909        /// A candidate declares itself ready with `catalog.update` on its own
14910        /// connection. If the connection-keyed registry lookups searched only the
14911        /// active slot, this would answer `not_registered` and the candidate
14912        /// would never become ready.
14913        #[tokio::test]
14914        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14915            let swap = swap_with_incumbent().await;
14916            register_candidate(&swap, Some(false));
14917            let update = Frame::build(
14918                FrameType::Request,
14919                control_flags(),
14920                0,
14921                0,
14922                55,
14923                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14924                    provides: manifest("aft", PROTOCOL_VERSION).provides,
14925                    capabilities: None,
14926                    ready: Some(true),
14927                })
14928                .unwrap(),
14929            )
14930            .unwrap();
14931
14932            let replies = swap
14933                .handler
14934                .handle_control_frame(&swap.candidate_ctx, update)
14935                .await
14936                .unwrap();
14937
14938            assert_eq!(replies.len(), 1);
14939            assert_eq!(
14940                replies[0].header.ty,
14941                FrameType::Response,
14942                "candidate catalog.update was refused: {:?}",
14943                serde_json::from_slice::<Value>(&replies[0].body).ok()
14944            );
14945            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14946            assert_eq!(
14947                swap.registry
14948                    .get_module("aft")
14949                    .unwrap()
14950                    .unwrap()
14951                    .connection_id,
14952                INCUMBENT
14953            );
14954        }
14955    }
14956
14957    /// The HELLO gate while the supervisor has a swap open: only the nonce it
14958    /// minted for the candidate admits a second process, into the candidate
14959    /// slot, and that check runs ahead of the reserved-module gate.
14960    mod swap_admission {
14961        use super::*;
14962
14963        const INCUMBENT_NONCE: &str = "incumbent-nonce";
14964        const CANDIDATE_NONCE: &str = "candidate-nonce";
14965
14966        fn handler_with_incumbent(
14967            module_id: &str,
14968            reserved: bool,
14969        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14970            let registry = Arc::new(Registry::default());
14971            let supervisor = SupervisorHandle::new();
14972            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14973            if reserved {
14974                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14975            }
14976            let handler =
14977                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
14978            let incumbent = handler
14979                .handle_control(
14980                    ConnectionId::new(1),
14981                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14982                )
14983                .unwrap();
14984            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14985            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14986            (registry, supervisor, handler)
14987        }
14988
14989        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
14990        /// admits every nonce, so while a swap is open the swap gate is the only
14991        /// thing between a key-holder and the candidate slot. A nonce the
14992        /// supervisor did not mint, or none at all, is refused, and neither the
14993        /// incumbent's registration nor the candidate slot moves.
14994        #[test]
14995        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14996            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14997
14998            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14999                let replies = handler
15000                    .handle_control(
15001                        ConnectionId::new(connection),
15002                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15003                    )
15004                    .unwrap();
15005                assert_eq!(replies[0].header.ty, FrameType::Error);
15006                assert_eq!(
15007                    parse_error(&replies[0])["code"],
15008                    "swap_token_invalid",
15009                    "nonce {nonce:?}"
15010                );
15011            }
15012            assert!(registry.get_candidate("aft").unwrap().is_none());
15013            assert_eq!(
15014                registry.get_module("aft").unwrap().unwrap().connection_id,
15015                ConnectionId::new(1)
15016            );
15017
15018            // Control: the minted token is admitted, into the candidate slot,
15019            // and only once.
15020            let admitted = handler
15021                .handle_control(
15022                    ConnectionId::new(4),
15023                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15024                )
15025                .unwrap();
15026            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15027            assert_eq!(
15028                registry
15029                    .get_candidate("aft")
15030                    .unwrap()
15031                    .unwrap()
15032                    .connection_id,
15033                ConnectionId::new(4)
15034            );
15035            assert_eq!(
15036                registry.get_module("aft").unwrap().unwrap().connection_id,
15037                ConnectionId::new(1),
15038                "the candidate must not take the active slot"
15039            );
15040            let replayed = handler
15041                .handle_control(
15042                    ConnectionId::new(5),
15043                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15044                )
15045                .unwrap();
15046            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15047
15048            // The case only this gate covers: the incumbent has died mid-swap,
15049            // so its duplicate refusal is gone too, and without the gate a
15050            // key-holder would take the id's ACTIVE slot.
15051            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15052            let squatter = handler
15053                .handle_control(
15054                    ConnectionId::new(6),
15055                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15056                )
15057                .unwrap();
15058            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15059            assert!(
15060                registry.get_module("aft").unwrap().is_none(),
15061                "a squatter took the active slot of an id being swapped"
15062            );
15063        }
15064
15065        /// Design mutation arm (iii). A reserved module's candidate presents a
15066        /// nonce the reserved gate has never seen (that gate holds the
15067        /// incumbent's), so the swap gate must run first or the candidate is
15068        /// refused `reserved_module` and a reserved module can never be swapped.
15069        #[test]
15070        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15071            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15072
15073            let replies = handler
15074                .handle_control(
15075                    ConnectionId::new(2),
15076                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15077                )
15078                .unwrap();
15079
15080            assert_eq!(
15081                replies[0].header.ty,
15082                FrameType::HelloAck,
15083                "reserved candidate refused: {:?}",
15084                serde_json::from_slice::<Value>(&replies[0].body).ok()
15085            );
15086            assert_eq!(
15087                registry
15088                    .get_candidate("vault")
15089                    .unwrap()
15090                    .unwrap()
15091                    .connection_id,
15092                ConnectionId::new(2)
15093            );
15094        }
15095
15096        /// With no swap open the gate is inert: the incumbent's reserved gate
15097        /// and duplicate refusal behave exactly as before.
15098        #[test]
15099        fn without_an_open_swap_the_ordinary_gates_decide() {
15100            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15101            supervisor.close_swap("vault");
15102
15103            let candidate = handler
15104                .handle_control(
15105                    ConnectionId::new(2),
15106                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15107                )
15108                .unwrap();
15109            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15110            let duplicate = handler
15111                .handle_control(
15112                    ConnectionId::new(3),
15113                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15114                )
15115                .unwrap();
15116            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15117            assert!(registry.get_candidate("vault").unwrap().is_none());
15118        }
15119    }
15120
15121    /// `scope.sync` and `scope.describe` through the real control handler: who
15122    /// may sync is decided by the registration and launch nonce of the module
15123    /// connection, never by the request body.
15124    mod scopes {
15125        mod idle_tests {
15126            use super::*;
15127            include!("scope_idle_tests.rs");
15128        }
15129        mod route_open_timings {
15130            use super::*;
15131            include!("route_open_timing_tests.rs");
15132        }
15133        mod activity {
15134            include!("activity_scope_tests.rs");
15135        }
15136        mod delta {
15137            include!("scope_handler_delta_tests.rs");
15138        }
15139        mod call_modes {
15140            include!("scope_call_mode_tests.rs");
15141        }
15142        #[tokio::test]
15143        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15144            let rig = rig().await;
15145            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15146                "s".into(),
15147                1,
15148                "tool".into(),
15149            ))
15150            .unwrap();
15151            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15152            let (direct, _rx) = wide_ctx(999);
15153            for (ctx, expected) in [
15154                (&rig.owner, FrameType::Response),
15155                (&direct, FrameType::Error),
15156            ] {
15157                let frame =
15158                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15159                        .unwrap();
15160                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15161                assert_eq!(replies[0].header.ty, expected);
15162                if expected == FrameType::Response {
15163                    let reply: subc_control::ScopeServedByReply =
15164                        serde_json::from_slice(&replies[0].body).unwrap();
15165                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15166                } else {
15167                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15168                }
15169            }
15170            let reply = call(
15171                &rig.handler,
15172                &rig.owner,
15173                &ModuleControlRequestFromModule::ScopeDescribe {
15174                    owner: Principal::Reserved {
15175                        module_id: OWNER.into(),
15176                    },
15177                    scope_ref: "s".into(),
15178                },
15179            )
15180            .await;
15181            assert!(matches!(
15182                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15183                ModuleControlResponseToModule::ScopeDescribe { .. }
15184            ));
15185        }
15186
15187        use subc_protocol::scope::{
15188            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15189            ScopeStatus,
15190        };
15191
15192        use super::*;
15193
15194        const OWNER: &str = "prefrontal-core";
15195
15196        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15197            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15198        }
15199
15200        async fn call(
15201            handler: &ControlHandler,
15202            ctx: &RouteCtx,
15203            request: &ModuleControlRequestFromModule,
15204        ) -> Frame {
15205            let body = serde_json::to_vec(request).unwrap();
15206            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15207            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15208            assert_eq!(replies.len(), 1, "{replies:?}");
15209            replies.pop().unwrap()
15210        }
15211
15212        async fn sync(
15213            handler: &ControlHandler,
15214            ctx: &RouteCtx,
15215            generation: u64,
15216            scopes: Vec<ScopeRecord>,
15217        ) -> Result<ModuleControlResponseToModule, String> {
15218            let reply = call(
15219                handler,
15220                ctx,
15221                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15222            )
15223            .await;
15224            match reply.header.ty {
15225                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15226                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15227            }
15228        }
15229
15230        async fn describe(
15231            handler: &ControlHandler,
15232            ctx: &RouteCtx,
15233            owner: &str,
15234            scope_ref: &str,
15235        ) -> ModuleControlResponseToModule {
15236            let reply = call(
15237                handler,
15238                ctx,
15239                &ModuleControlRequestFromModule::ScopeDescribe {
15240                    owner: Principal::Reserved {
15241                        module_id: owner.to_string(),
15242                    },
15243                    scope_ref: scope_ref.to_string(),
15244                },
15245            )
15246            .await;
15247            assert_eq!(
15248                reply.header.ty,
15249                FrameType::Response,
15250                "{:?}",
15251                parse_error(&reply)
15252            );
15253            serde_json::from_slice(&reply.body).unwrap()
15254        }
15255
15256        #[tokio::test]
15257        async fn scope_describe_never_echoes_bind_proof_or_hash() {
15258            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15259            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15260            let mut rig = rig().await;
15261            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15262                .await;
15263            let _bound = rig
15264                .bound(
15265                    Some(AFT),
15266                    PLEXUS,
15267                    Some(
15268                        rig.selector("s", Some(1))
15269                            .with_bind_proof(Some(PROOF.into())),
15270                    ),
15271                )
15272                .await;
15273            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15274            let output = serde_json::to_string(&reply).unwrap();
15275            assert!(output.contains("\"status\":\"live\""));
15276            assert!(!output.contains(PROOF));
15277            assert!(!output.contains(HASH));
15278        }
15279
15280        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
15281        async fn module(
15282            handler: &ControlHandler,
15283            connection: u64,
15284            module_id: &str,
15285            nonce: Option<&str>,
15286        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15287            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15288            hello_via_sink(
15289                handler,
15290                &ctx,
15291                &mut rx,
15292                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15293            )
15294            .await;
15295            (ctx, rx)
15296        }
15297
15298        /// `direct` and every other client connection has no registration, so
15299        /// it can neither sync nor own a scope.
15300        #[tokio::test]
15301        async fn a_client_connection_cannot_sync_or_describe() {
15302            let handler = ControlHandler::new(Arc::new(Registry::default()));
15303            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15304            for request in [
15305                ModuleControlRequestFromModule::ScopeSync {
15306                    generation: 1,
15307                    scopes: vec![head("s", 1)],
15308                },
15309                ModuleControlRequestFromModule::ScopeDescribe {
15310                    owner: Principal::Direct,
15311                    scope_ref: "s".to_string(),
15312                },
15313            ] {
15314                let reply = call(&handler, &ctx, &request).await;
15315                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15316            }
15317            assert!(
15318                !handler
15319                    .scopes
15320                    .read()
15321                    .unwrap()
15322                    .describe(
15323                        &Principal::Reserved {
15324                            module_id: OWNER.to_string()
15325                        },
15326                        "s"
15327                    )
15328                    .owner_synced
15329            );
15330        }
15331
15332        /// A module the supervisor did not spawn registers without a launch
15333        /// nonce, so it is never an owner's current launch.
15334        #[tokio::test]
15335        async fn a_module_without_a_supervised_launch_cannot_sync() {
15336            let handler = ControlHandler::new(Arc::new(Registry::default()));
15337            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15338            assert_eq!(
15339                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15340                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15341            );
15342        }
15343
15344        #[tokio::test]
15345        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15346            let supervisor = SupervisorHandle::new();
15347            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15348            let handler = ControlHandler::new(Arc::new(Registry::default()))
15349                .with_supervisor(supervisor.clone());
15350            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15351            sync(&handler, &incumbent, 1, vec![head("s", 1)])
15352                .await
15353                .expect("the current launch syncs");
15354
15355            // A swap candidate registers with the swap token and is refused
15356            // while the incumbent keeps syncing.
15357            supervisor.open_swap(OWNER, "n2".to_string());
15358            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15359            assert_eq!(
15360                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15361                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15362            );
15363            sync(&handler, &incumbent, 2, vec![head("s", 1)])
15364                .await
15365                .expect("the serving owner syncs during the swap");
15366
15367            // The swap fails and is rolled back. The candidate never held sync
15368            // authority, and still cannot sync.
15369            supervisor.close_swap(OWNER);
15370            assert_eq!(
15371                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15372                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15373            );
15374            sync(&handler, &incumbent, 3, vec![head("s", 1)])
15375                .await
15376                .expect("the serving owner syncs after the rollback");
15377            handler.cleanup_connection(candidate.connection_id).unwrap();
15378
15379            // A swap that cuts over. Promotion records the candidate's nonce as
15380            // the module's spawn nonce, which is what `set_spawn_nonce` does
15381            // here; the promoted connection then takes authority at any
15382            // generation and the superseded incumbent is refused.
15383            supervisor.open_swap(OWNER, "n3".to_string());
15384            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15385            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15386            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15387                .await
15388                .expect("the promoted launch takes authority");
15389            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15390                panic!("unexpected reply {reply:?}");
15391            };
15392            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15393            assert_eq!(
15394                sync(&handler, &incumbent, 4, Vec::new()).await,
15395                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15396            );
15397        }
15398
15399        /// Authority dies with its connection: the cleanup path releases it,
15400        /// so the owner's next connection takes it at any generation.
15401        #[tokio::test]
15402        async fn closing_the_authority_connection_frees_sync_authority() {
15403            let supervisor = SupervisorHandle::new();
15404            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15405            let handler = ControlHandler::new(Arc::new(Registry::default()))
15406                .with_supervisor(supervisor.clone());
15407            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15408            sync(&handler, &first, 10, vec![head("s", 1)])
15409                .await
15410                .unwrap();
15411            handler.cleanup_connection(first.connection_id).unwrap();
15412
15413            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15414            sync(&handler, &second, 1, vec![head("s", 1)])
15415                .await
15416                .expect("the next connection takes the released authority");
15417        }
15418
15419        #[tokio::test]
15420        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15421            let supervisor = SupervisorHandle::new();
15422            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15423            let handler =
15424                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15425            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15426            sync(&handler, &first, 10, vec![head("s", 1)])
15427                .await
15428                .unwrap();
15429            handler
15430                .handle_control_frame(
15431                    &first,
15432                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15433                )
15434                .await
15435                .unwrap();
15436            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15437            sync(&handler, &second, 1, vec![head("s", 1)])
15438                .await
15439                .expect("GOODBYE releases authority even if the old socket remains open");
15440        }
15441
15442        #[tokio::test]
15443        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15444            let registry = Arc::new(Registry::default());
15445            let supervisor_handle = SupervisorHandle::new();
15446            let supervisor =
15447                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15448                    .with_handle(supervisor_handle.clone())
15449                    .with_daemon_incarnation("incarnation-7".to_string());
15450            // Configured with enabled: false, so the supervisor lists the
15451            // module without spawning a process for it.
15452            supervisor
15453                .supervise_configured(
15454                    ModuleSpec {
15455                        module_id: OWNER.to_string(),
15456                        program: PathBuf::from("/nonexistent/prefrontal-core"),
15457                        args: Vec::new(),
15458                        env: Vec::new(),
15459                        reserved: false,
15460                        reserved_prefixes: Vec::new(),
15461                        protocol: ModuleProtocol::Subc,
15462                        overlap: Default::default(),
15463                    },
15464                    false,
15465                )
15466                .unwrap();
15467            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15468            let handler =
15469                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15470            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15471
15472            // Configured but not yet synced: a reader waits for the owner.
15473            let ModuleControlResponseToModule::ScopeDescribe {
15474                status,
15475                daemon_incarnation,
15476                owner_synced,
15477                owner_configured,
15478                scope,
15479                ..
15480            } = describe(&handler, &reader, OWNER, "s").await
15481            else {
15482                panic!("not a describe reply");
15483            };
15484            assert_eq!(status, ScopeStatus::NotLive);
15485            assert_eq!(daemon_incarnation, "incarnation-7");
15486            assert!(!owner_synced);
15487            assert!(owner_configured);
15488            assert!(scope.is_none());
15489
15490            // Not a supervised module: the owner will never sync, and a reader
15491            // refuses rather than waits.
15492            let ModuleControlResponseToModule::ScopeDescribe {
15493                status,
15494                owner_configured,
15495                ..
15496            } = describe(&handler, &reader, "ghost", "s").await
15497            else {
15498                panic!("not a describe reply");
15499            };
15500            assert_eq!(status, ScopeStatus::NotLive);
15501            assert!(!owner_configured);
15502
15503            // Live, with the stamp fields and the computed owner_authorized.
15504            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15505            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15506            let ModuleControlResponseToModule::ScopeDescribe {
15507                status,
15508                scope_epoch,
15509                owner_synced,
15510                scope,
15511                ..
15512            } = describe(&handler, &reader, OWNER, "s").await
15513            else {
15514                panic!("not a describe reply");
15515            };
15516            assert_eq!(status, ScopeStatus::Live);
15517            assert_eq!(scope_epoch, Some(4));
15518            assert!(owner_synced);
15519            let stamp = scope.expect("a live scope carries its stamp");
15520            assert!(
15521                stamp.owner_authorized,
15522                "prefrontal-core is the default authority"
15523            );
15524            assert_eq!(stamp.kind, ScopeKind::Head);
15525        }
15526
15527        #[tokio::test]
15528        async fn scope_authority_owners_decides_owner_authorized() {
15529            let supervisor = SupervisorHandle::new();
15530            supervisor.set_spawn_nonce("broca", "b1".to_string());
15531            let handler = ControlHandler::new(Arc::new(Registry::default()))
15532                .with_supervisor(supervisor)
15533                .with_scope_authority_owners(vec!["broca".to_string()]);
15534            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15535            let mut gated = head("s", 1);
15536            gated.attributes.agent_id = Some("agent".to_string());
15537            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15538            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15539                describe(&handler, &broca, "broca", "s").await
15540            else {
15541                panic!("not a describe reply");
15542            };
15543            assert!(scope.unwrap().owner_authorized);
15544        }
15545
15546        /// With route admission, the stamp, the commit re-check and drains in
15547        /// place, the feature is advertised: the module ops in HELLO_ACK, and
15548        /// `scopes/v1` in HELLO_ACK and `server.describe`.
15549        #[tokio::test]
15550        async fn scope_ops_and_the_scopes_capability_are_advertised() {
15551            let handler = ControlHandler::new(Arc::new(Registry::default()));
15552            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15553            let ack = hello_via_sink(
15554                &handler,
15555                &ctx,
15556                &mut rx,
15557                hello_frame("m", PROTOCOL_VERSION, 1),
15558            )
15559            .await;
15560            let ack = parse_ack(&ack);
15561            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15562                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15563            }
15564            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15565
15566            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15567            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15568            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15569            let reply = handler
15570                .handle_control_frame(&client, frame)
15571                .await
15572                .unwrap()
15573                .pop()
15574                .unwrap();
15575            let ClientControlResponse::ServerDescribe { capabilities, .. } =
15576                serde_json::from_slice(&reply.body).unwrap()
15577            else {
15578                panic!("not a server.describe reply");
15579            };
15580            assert!(
15581                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15582                "{capabilities:?}"
15583            );
15584        }
15585
15586        // ---- route admission, stamps, commit re-check and drains ----------
15587
15588        const PLEXUS: &str = "plexus";
15589        const OTHER: &str = "other";
15590        const AFT: &str = "aft";
15591        const BROCA: &str = "broca";
15592        const MAGIC: &str = "magic-context";
15593
15594        fn nonce(module_id: &str) -> String {
15595            format!("nonce-{module_id}")
15596        }
15597
15598        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15599            let (tx, rx) = mpsc::channel(64);
15600            (
15601                RouteCtx {
15602                    connection_id: ConnectionId::new(connection),
15603                    egress: FrameSink::new(tx),
15604                },
15605                rx,
15606            )
15607        }
15608
15609        /// A daemon with a configured owner (prefrontal-core) registered on its
15610        /// own module connection, two routable targets (plexus, other), and
15611        /// launch nonces minted for the modules that open routes as carriers.
15612        struct Rig {
15613            handler: ControlHandler,
15614            forwarding: Arc<ForwardingTable>,
15615            owner: RouteCtx,
15616            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15617            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15618            generation: u64,
15619            next_connection: u64,
15620            _supervisor: Supervisor,
15621        }
15622
15623        async fn rig() -> Rig {
15624            rig_with_flow_support(true).await
15625        }
15626
15627        async fn rig_with_flow_support(flow_support: bool) -> Rig {
15628            rig_with_scope_capabilities(flow_support, false).await
15629        }
15630
15631        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15632            let mut provides = Vec::new();
15633            if flow_support {
15634                provides.push("flow-scopes/v1");
15635            }
15636            if run_support {
15637                provides.push("agent-run-scopes/v1");
15638            }
15639            rig_with_target_capabilities(&provides).await
15640        }
15641
15642        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15643            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15644            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15645            // Decoder versions alone cannot admit scoped calls: only a capability
15646            // declaration promises the required run or flow behaviour.
15647            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15648            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15649            Frame::build(
15650                FrameType::Hello,
15651                control_flags(),
15652                0,
15653                0,
15654                connection,
15655                serde_json::to_vec(&body).unwrap(),
15656            )
15657            .unwrap()
15658        }
15659
15660        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15661            let registry = Arc::new(Registry::default());
15662            let forwarding = Arc::new(ForwardingTable::default());
15663            let supervisor_handle = SupervisorHandle::new();
15664            let supervisor =
15665                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15666                    .with_handle(supervisor_handle.clone());
15667            supervisor
15668                .supervise_configured(
15669                    ModuleSpec {
15670                        module_id: OWNER.to_string(),
15671                        program: PathBuf::from("/nonexistent/prefrontal-core"),
15672                        args: Vec::new(),
15673                        env: Vec::new(),
15674                        reserved: false,
15675                        reserved_prefixes: Vec::new(),
15676                        protocol: ModuleProtocol::Subc,
15677                        overlap: Default::default(),
15678                    },
15679                    false,
15680                )
15681                .unwrap();
15682            for module_id in [OWNER, AFT, BROCA, MAGIC] {
15683                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15684            }
15685            let handler =
15686                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15687                    .with_supervisor(supervisor_handle);
15688            let (owner, mut owner_rx) = wide_ctx(1);
15689            hello_via_sink(
15690                &handler,
15691                &owner,
15692                &mut owner_rx,
15693                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15694            )
15695            .await;
15696            let mut modules = BTreeMap::new();
15697            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15698                let (ctx, mut rx) = wide_ctx(connection);
15699                let hello = hello_with_capabilities(module_id, connection, provides);
15700                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15701                modules.insert(module_id.to_string(), (ctx, rx));
15702            }
15703            Rig {
15704                handler,
15705                forwarding,
15706                owner,
15707                _owner_rx: owner_rx,
15708                modules,
15709                generation: 0,
15710                next_connection: 100,
15711                _supervisor: supervisor,
15712            }
15713        }
15714
15715        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15716            ScopeCarrier::new(Principal::Reserved {
15717                module_id: module_id.to_string(),
15718            })
15719            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15720        }
15721
15722        /// The scope most tests open under: aft carries to any module, broca
15723        /// only to plexus and other, and the owner delegates as agent-1.
15724        fn session(scope_epoch: u64) -> ScopeRecord {
15725            let mut record = head("s", scope_epoch);
15726            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15727            record.attributes.agent_id = Some("agent-1".to_string());
15728            record.attributes.delegates = true;
15729            record
15730        }
15731
15732        impl Rig {
15733            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15734                self.generation += 1;
15735                sync(&self.handler, &self.owner, self.generation, scopes)
15736                    .await
15737                    .expect("the owner's sync is accepted");
15738            }
15739
15740            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15741                ScopeSelector::new(
15742                    Principal::Reserved {
15743                        module_id: OWNER.to_string(),
15744                    },
15745                    scope_ref.to_string(),
15746                    scope_epoch,
15747                )
15748            }
15749
15750            fn open_frame(
15751                &mut self,
15752                opener: Option<&str>,
15753                target: &str,
15754                scope: Option<ScopeSelector>,
15755            ) -> (
15756                RouteCtx,
15757                mpsc::Receiver<crate::router::OutboundFrame>,
15758                Frame,
15759            ) {
15760                self.next_connection += 1;
15761                let (ctx, rx) = wide_ctx(self.next_connection);
15762                let root = unique_project_root("scoped-open");
15763                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15764                    target: RouteTarget::ToolProvider {
15765                        module_id: target.to_string(),
15766                    },
15767                    identity: BindIdentity::new(
15768                        root.path().to_path_buf(),
15769                        "unit".to_string(),
15770                        "session".to_string(),
15771                    ),
15772                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
15773                        module_id: module_id.to_string(),
15774                        launch_nonce: nonce(module_id),
15775                    }),
15776                    consumer_capabilities: None,
15777                    role_versions: None,
15778                    admission_facts: None,
15779                    scope,
15780                })
15781                .unwrap();
15782                let frame = Frame::build(
15783                    FrameType::Request,
15784                    control_flags(),
15785                    0,
15786                    0,
15787                    self.next_connection,
15788                    body,
15789                )
15790                .unwrap();
15791                (ctx, rx, frame)
15792            }
15793
15794            /// Open and expect a refusal before anything is relayed.
15795            async fn refused(
15796                &mut self,
15797                opener: Option<&str>,
15798                target: &str,
15799                scope: Option<ScopeSelector>,
15800            ) -> String {
15801                self.refusal_body(opener, target, scope).await["code"]
15802                    .as_str()
15803                    .unwrap()
15804                    .to_string()
15805            }
15806
15807            async fn refusal_body(
15808                &mut self,
15809                opener: Option<&str>,
15810                target: &str,
15811                scope: Option<ScopeSelector>,
15812            ) -> Value {
15813                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15814                let replies = tokio::time::timeout(
15815                    Duration::from_secs(2),
15816                    self.handler.handle_control_frame(&ctx, frame),
15817                )
15818                .await
15819                .expect("the open must be refused before waiting for a bind ack")
15820                .unwrap();
15821                assert_eq!(replies.len(), 1, "{replies:?}");
15822                assert_eq!(replies[0].header.ty, FrameType::Error);
15823                let (_, module_rx) = self.modules.get_mut(target).unwrap();
15824                assert!(
15825                    module_rx.try_recv().is_err(),
15826                    "a refused open relays nothing"
15827                );
15828                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15829                parse_error(&replies[0])
15830            }
15831
15832            /// Start an open and return its task and the bind the target got.
15833            async fn relayed(
15834                &mut self,
15835                opener: Option<&str>,
15836                target: &str,
15837                scope: Option<ScopeSelector>,
15838            ) -> Relayed {
15839                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15840                let handler = self.handler.clone();
15841                let task_ctx = ctx.clone();
15842                let task = tokio::spawn(async move {
15843                    handler
15844                        .handle_control_frame(&task_ctx, frame)
15845                        .await
15846                        .unwrap()
15847                });
15848                let (_, module_rx) = self.modules.get_mut(target).unwrap();
15849                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15850                    .await
15851                    .expect("the target receives the relayed route.bind")
15852                    .unwrap()
15853                    .frame;
15854                Relayed {
15855                    target: target.to_string(),
15856                    client: ctx,
15857                    client_rx: rx,
15858                    task,
15859                    bind,
15860                }
15861            }
15862
15863            async fn ack(&self, relayed: &Relayed) {
15864                let (module, _) = &self.modules[&relayed.target];
15865                self.handler
15866                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15867                    .await
15868                    .unwrap();
15869            }
15870
15871            /// Open, ack and return the bound route.
15872            async fn bound(
15873                &mut self,
15874                opener: Option<&str>,
15875                target: &str,
15876                scope: Option<ScopeSelector>,
15877            ) -> Bound {
15878                let relayed = self.relayed(opener, target, scope).await;
15879                self.ack(&relayed).await;
15880                let Relayed {
15881                    target,
15882                    client,
15883                    mut client_rx,
15884                    task,
15885                    bind,
15886                } = relayed;
15887                assert!(
15888                    task.await.unwrap().is_empty(),
15889                    "the open is answered by commit"
15890                );
15891                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15892                Bound {
15893                    target,
15894                    client,
15895                    client_rx,
15896                    channel,
15897                    epoch,
15898                    bind,
15899                }
15900            }
15901
15902            fn live(&self, route: &Bound) -> bool {
15903                matches!(
15904                    self.forwarding
15905                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15906                        .unwrap(),
15907                    DataRoute::Client(DataRouteState::Bound(_))
15908                )
15909            }
15910        }
15911
15912        struct Relayed {
15913            target: String,
15914            client: RouteCtx,
15915            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15916            task: tokio::task::JoinHandle<Vec<Frame>>,
15917            bind: Frame,
15918        }
15919
15920        struct Bound {
15921            target: String,
15922            client: RouteCtx,
15923            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15924            channel: u16,
15925            epoch: u32,
15926            bind: Frame,
15927        }
15928
15929        impl Bound {
15930            /// The reason of the `route.closed` this client was sent, after
15931            /// checking it also got a GOODBYE on exactly this route.
15932            fn closed_reason(&mut self) -> RouteCloseReason {
15933                let mut reason = None;
15934                let mut goodbye = false;
15935                while let Ok(outbound) = self.client_rx.try_recv() {
15936                    let frame = outbound.frame;
15937                    match frame.header.ty {
15938                        FrameType::Goodbye => {
15939                            assert_eq!(
15940                                (frame.header.channel, frame.header.epoch),
15941                                (self.channel, self.epoch)
15942                            );
15943                            goodbye = true;
15944                        }
15945                        FrameType::Push => {
15946                            let ClientControlPush::RouteClosed {
15947                                reason: r,
15948                                module_id,
15949                                ..
15950                            } = serde_json::from_slice(&frame.body).unwrap()
15951                            else {
15952                                panic!("unexpected push");
15953                            };
15954                            assert_eq!(module_id, self.target);
15955                            reason = Some(r);
15956                        }
15957                        other => panic!("unexpected frame {other:?}"),
15958                    }
15959                }
15960                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15961                reason.expect("the client is told why the route closed")
15962            }
15963
15964            fn untouched(&mut self) -> bool {
15965                self.client_rx.try_recv().is_err()
15966            }
15967
15968            fn stamp(&self) -> Option<ScopeStamp> {
15969                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15970                    ModuleControlRequest::RouteBind { scope, .. } => scope,
15971                    other => panic!("expected a route.bind, got {other:?}"),
15972                }
15973            }
15974        }
15975
15976        #[tokio::test]
15977        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15978        ) {
15979            let mut rig = rig().await;
15980            let mut record = session(1);
15981            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15982            record.child_owners = vec![Principal::Reserved {
15983                module_id: MAGIC.to_string(),
15984            }];
15985            rig.sync(vec![record]).await;
15986            let scope = || Some(rig_selector("s", Some(1)));
15987
15988            // Admitted: the owner, a bare carrier to any module, a targeted
15989            // carrier to its listed module.
15990            rig.bound(Some(OWNER), PLEXUS, scope()).await;
15991            rig.bound(Some(AFT), OTHER, scope()).await;
15992            rig.bound(Some(BROCA), PLEXUS, scope()).await;
15993
15994            // Refused scope_not_carrier: a targeted carrier to an unlisted
15995            // module, a module that is not listed at all (a child owner is not
15996            // a carrier), and a direct key-holder.
15997            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15998                assert_eq!(
15999                    rig.refused(opener, target, scope()).await,
16000                    error_codes::SCOPE_NOT_CARRIER,
16001                    "{opener:?} -> {target}"
16002                );
16003            }
16004        }
16005
16006        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16007            ScopeSelector::new(
16008                Principal::Reserved {
16009                    module_id: OWNER.to_string(),
16010                },
16011                scope_ref.to_string(),
16012                scope_epoch,
16013            )
16014        }
16015
16016        #[tokio::test]
16017        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16018            let mut rig = rig().await;
16019            rig.sync(vec![session(1)]).await;
16020            for opener in [OWNER, AFT] {
16021                assert_eq!(
16022                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16023                        .await,
16024                    error_codes::SCOPE_EPOCH_REQUIRED,
16025                    "{opener}"
16026                );
16027            }
16028        }
16029
16030        #[tokio::test]
16031        async fn admission_separates_not_synced_not_live_and_ended() {
16032            let mut rig = rig().await;
16033            // Before the configured owner's first sync: retryable.
16034            let code = rig
16035                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16036                .await;
16037            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16038            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16039
16040            // An owner that is not configured will never sync: terminal.
16041            let ghost = ScopeSelector::new(
16042                Principal::Reserved {
16043                    module_id: "ghost".to_string(),
16044                },
16045                "s".to_string(),
16046                Some(1),
16047            );
16048            assert_eq!(
16049                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16050                error_codes::SCOPE_NOT_LIVE
16051            );
16052
16053            rig.sync(vec![session(2)]).await;
16054            assert_eq!(
16055                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16056                    .await,
16057                error_codes::SCOPE_NOT_LIVE
16058            );
16059            for epoch in [1, 3] {
16060                assert_eq!(
16061                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16062                        .await,
16063                    error_codes::SCOPE_ENDED,
16064                    "epoch {epoch}"
16065                );
16066            }
16067            // Control: the live epoch is admitted.
16068            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16069                .await;
16070        }
16071
16072        #[tokio::test]
16073        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16074            let mut rig = rig().await;
16075            rig.sync(vec![session(1)]).await;
16076            let route = rig
16077                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16078                .await;
16079            let stamp = route.stamp().expect("a scoped bind carries the stamp");
16080            assert_eq!(stamp.scope_ref, "s");
16081            assert_eq!(stamp.scope_epoch, 1);
16082            assert_eq!(stamp.kind, ScopeKind::Head);
16083            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16084            assert!(stamp.attributes.delegates);
16085            assert!(stamp.owner_authorized);
16086            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16087            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16088
16089            // broca owns a scope of its own on its own module connection; it is
16090            // not in scope_authority_owners, so its stamp is not authorized.
16091            let (broca, mut broca_rx) = wide_ctx(50);
16092            hello_via_sink(
16093                &rig.handler,
16094                &broca,
16095                &mut broca_rx,
16096                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16097            )
16098            .await;
16099            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16100                .await
16101                .unwrap();
16102            let own = ScopeSelector::new(
16103                Principal::Reserved {
16104                    module_id: BROCA.to_string(),
16105                },
16106                "b".to_string(),
16107                Some(1),
16108            );
16109            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16110            assert!(!route.stamp().unwrap().owner_authorized);
16111        }
16112
16113        #[tokio::test]
16114        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16115            let mut rig = rig().await;
16116            let mut record = head("s", 1);
16117            record.carriers = vec![carrier(AFT, None)];
16118            let flow_id = "Flow:run-7/step_2!~";
16119            record.attributes.flow_id = Some(flow_id.to_string());
16120            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16121                .await
16122                .unwrap();
16123            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16124                panic!("not a sync reply");
16125            };
16126            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16127            let route = rig
16128                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16129                .await;
16130            assert!(rig.live(&route), "the stamped bind committed");
16131            let stamp = route.stamp().expect("a flow scope carries a stamp");
16132            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16133            assert_eq!(stamp.attributes.agent_id, None);
16134            assert!(!stamp.attributes.delegates);
16135            assert!(stamp.owner_authorized);
16136            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16137            assert_eq!(unscoped.stamp(), None);
16138        }
16139
16140        #[tokio::test]
16141        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16142            let mut rig = rig_with_flow_support(false).await;
16143            let mut record = session(1);
16144            record.attributes.flow_id = Some("flow:7".to_string());
16145            rig.sync(vec![record]).await;
16146            for opener in [OWNER, AFT] {
16147                let body = rig
16148                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16149                    .await;
16150                assert_eq!(body["code"], "target_flow_unsupported");
16151                let message = body["message"].as_str().unwrap();
16152                for required in [PLEXUS, "flow-scopes/v1"] {
16153                    assert!(message.contains(required), "{message}");
16154                }
16155            }
16156        }
16157
16158        #[tokio::test]
16159        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16160            let mut rig = rig_with_flow_support(true).await;
16161            let mut record = session(1);
16162            record.attributes.flow_id = Some("flow:7".to_string());
16163            rig.sync(vec![record]).await;
16164            for opener in [OWNER, AFT] {
16165                let route = rig
16166                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16167                    .await;
16168                assert!(rig.live(&route));
16169                assert_eq!(
16170                    route.stamp().unwrap().attributes.flow_id.as_deref(),
16171                    Some("flow:7")
16172                );
16173            }
16174        }
16175
16176        #[tokio::test]
16177        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16178            use std::future::Future;
16179
16180            let mut rig = rig_with_flow_support(true).await;
16181            let mut record = session(1);
16182            record.attributes.flow_id = Some("flow:7".to_string());
16183            rig.sync(vec![record]).await;
16184            let (client, mut client_rx, frame) =
16185                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16186            // Hold the route-open response permit so admission sees the first
16187            // target but relay reservation cannot capture an endpoint yet.
16188            for _ in 0..64 {
16189                client.egress.try_send(route_bind_ack(1)).unwrap();
16190            }
16191            let handler = rig.handler.clone();
16192            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16193            std::future::poll_fn(|cx| {
16194                assert!(open.as_mut().poll(cx).is_pending());
16195                std::task::Poll::Ready(())
16196            })
16197            .await;
16198            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16199
16200            let old_connection = rig.modules[PLEXUS].0.connection_id;
16201            rig.handler.cleanup_connection(old_connection).unwrap();
16202            let (replacement, mut replacement_rx) = wide_ctx(200);
16203            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16204            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16205            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16206            let hello = Frame::build(
16207                FrameType::Hello,
16208                control_flags(),
16209                0,
16210                0,
16211                200,
16212                serde_json::to_vec(&body).unwrap(),
16213            )
16214            .unwrap();
16215            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16216
16217            client_rx.try_recv().unwrap();
16218            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16219                .await
16220                .expect("the replacement is refused without waiting for a bind ack")
16221                .unwrap();
16222            assert_eq!(replies.len(), 1);
16223            let body = parse_error(&replies[0]);
16224            assert_eq!(body["code"], "target_flow_unsupported");
16225            for required in [PLEXUS, "flow-scopes/v1"] {
16226                assert!(body["message"].as_str().unwrap().contains(required));
16227            }
16228            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16229            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16230            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16231            assert!(rig
16232                .handler
16233                .registry
16234                .get_module_by_connection(replacement.connection_id)
16235                .unwrap()
16236                .is_some());
16237        }
16238
16239        #[tokio::test]
16240        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16241            use std::future::Future;
16242            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16243            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16244            const NEXT_HASH: &str =
16245                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16246            let mut rig = rig().await;
16247            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16248            record.carriers.push(ScopeCarrier::new(Principal::Direct));
16249            rig.sync(vec![record.clone()]).await;
16250            let (client, mut rx, frame) = rig.open_frame(
16251                None,
16252                PLEXUS,
16253                Some(
16254                    rig.selector("s", Some(1))
16255                        .with_bind_proof(Some(PROOF.into())),
16256                ),
16257            );
16258            // Hold egress reservation while the scope owner changes its bind-proof
16259            // hash, so a proof valid at admission must be checked again at relay.
16260            for _ in 0..64 {
16261                client.egress.try_send(route_bind_ack(1)).unwrap();
16262            }
16263            let handler = rig.handler.clone();
16264            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16265            std::future::poll_fn(|cx| {
16266                assert!(open.as_mut().poll(cx).is_pending());
16267                std::task::Poll::Ready(())
16268            })
16269            .await;
16270            record.bind_proof_sha256 = Some(NEXT_HASH.into());
16271            rig.sync(vec![record]).await;
16272            rx.try_recv().unwrap();
16273            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16274                .await
16275                .unwrap()
16276                .unwrap();
16277            assert_eq!(
16278                parse_error(&replies[0])["code"],
16279                error_codes::SCOPE_BIND_PROOF_MISMATCH
16280            );
16281            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16282            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16283            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16284        }
16285
16286        #[tokio::test]
16287        async fn scope_bind_proof_reserved_openers_ignore_proof() {
16288            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16289            let mut rig = rig().await;
16290            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16291                .await;
16292            for opener in [OWNER, AFT] {
16293                for proof in [None, Some("malformed+proof=")] {
16294                    let route = rig
16295                        .bound(
16296                            Some(opener),
16297                            PLEXUS,
16298                            Some(
16299                                rig.selector("s", Some(1))
16300                                    .with_bind_proof(proof.map(str::to_string)),
16301                            ),
16302                        )
16303                        .await;
16304                    assert!(rig.live(&route));
16305                }
16306            }
16307        }
16308
16309        #[tokio::test]
16310        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16311            use std::future::Future;
16312            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16313            const NEXT_HASH: &str =
16314                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16315            let mut rig = rig().await;
16316            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16317                .await;
16318            let (client, mut rx, frame) =
16319                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16320            for _ in 0..64 {
16321                client.egress.try_send(route_bind_ack(1)).unwrap();
16322            }
16323            let handler = rig.handler.clone();
16324            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16325            std::future::poll_fn(|cx| {
16326                assert!(open.as_mut().poll(cx).is_pending());
16327                std::task::Poll::Ready(())
16328            })
16329            .await;
16330            rig.sync(vec![
16331                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16332            ])
16333            .await;
16334            rx.try_recv().unwrap();
16335            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16336            let bind = std::future::poll_fn(|cx| {
16337                assert!(open.as_mut().poll(cx).is_pending());
16338                module_rx.poll_recv(cx)
16339            })
16340            .await
16341            .unwrap()
16342            .frame;
16343            handler
16344                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16345                .await
16346                .unwrap();
16347            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16348                .await
16349                .unwrap()
16350                .unwrap();
16351            assert_eq!(replies.len(), 1);
16352            // Proof rechecking allowed the relay. Independent tag settlement
16353            // still asks the opener to retry after any same-epoch content change.
16354            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16355            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16356            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16357            let route = rig
16358                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16359                .await;
16360            assert!(rig.live(&route));
16361        }
16362
16363        #[tokio::test]
16364        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16365        {
16366            let mut rig = rig_with_flow_support(false).await;
16367            rig.sync(vec![session(1)]).await;
16368            let route = rig
16369                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16370                .await;
16371            assert!(rig.live(&route));
16372            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16373            let mut record = session(1);
16374            record.attributes.flow_id = Some("flow:7".to_string());
16375            rig.sync(vec![record]).await;
16376            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16377            assert!(rig.live(&unscoped));
16378            assert_eq!(unscoped.stamp(), None);
16379        }
16380
16381        #[tokio::test]
16382        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16383            let mut rig = rig().await;
16384            let mut record = session(1);
16385            record.attributes.flow_id = Some("flow:7".to_string());
16386            rig.sync(vec![record.clone()]).await;
16387            let before = rig.forwarding.published_scope_tag(OWNER, "s");
16388            let mut owner_route = rig
16389                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16390                .await;
16391            let mut carrier_route = rig
16392                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16393                .await;
16394            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16395            rig.sync(vec![record.clone()]).await;
16396            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16397            assert!(rig.live(&owner_route) && owner_route.untouched());
16398            assert!(rig.live(&carrier_route) && carrier_route.untouched());
16399
16400            record.attributes.flow_id = Some("flow:8".to_string());
16401            rig.sync(vec![record]).await;
16402            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16403            let before = before.unwrap();
16404            assert_eq!(after.scope_epoch, before.scope_epoch);
16405            assert!(after.version > before.version);
16406            for route in [&mut owner_route, &mut carrier_route] {
16407                assert!(!rig.live(route));
16408                assert_eq!(
16409                    route.closed_reason(),
16410                    RouteCloseReason::ScopeDelegationChanged
16411                );
16412            }
16413            assert!(rig.live(&unscoped) && unscoped.untouched());
16414            // Each provider also receives a GOODBYE for its drained route;
16415            // consume it before expecting the next route.bind on that sink.
16416            for target in [PLEXUS, OTHER] {
16417                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16418                let goodbye = module_rx
16419                    .try_recv()
16420                    .expect("the provider sees the drain")
16421                    .frame;
16422                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16423                assert!(module_rx.try_recv().is_err());
16424            }
16425            let rebound = rig
16426                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16427                .await;
16428            assert_eq!(
16429                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16430                Some("flow:8")
16431            );
16432        }
16433
16434        /// The owner's sync lands between admission and the module's ack. The
16435        /// open is refused by name, the module's other routes stay up, and the
16436        /// reserved pair is released. Changed content is retryable; an ended
16437        /// scope is not.
16438        #[tokio::test]
16439        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16440            let mut rig = rig().await;
16441            rig.sync(vec![session(1)]).await;
16442            let mut cotenant = rig
16443                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16444                .await;
16445
16446            let mut changed = session(1);
16447            changed.child_owners.push(Principal::Reserved {
16448                module_id: MAGIC.to_string(),
16449            });
16450            let mut ended = None;
16451            for (code, next) in [
16452                (error_codes::SCOPE_CHANGED, vec![changed]),
16453                (error_codes::SCOPE_ENDED, Vec::new()),
16454            ] {
16455                let relayed = rig
16456                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16457                    .await;
16458                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16459                ended = Some(next.is_empty());
16460                rig.sync(next).await;
16461                rig.ack(&relayed).await;
16462                let replies = relayed.task.await.unwrap();
16463                assert_eq!(replies.len(), 1, "{replies:?}");
16464                assert_eq!(parse_error(&replies[0])["code"], code);
16465                assert_eq!(
16466                    subc_protocol::error_codes::is_retryable_route_open(code),
16467                    code == error_codes::SCOPE_CHANGED
16468                );
16469                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16470                // The module is told to drop just the binding it created.
16471                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16472                // Collected, because ending the scope also closes the co-tenant
16473                // route, whose GOODBYE comes first.
16474                let mut goodbyes = Vec::new();
16475                while let Ok(outbound) = plexus_rx.try_recv() {
16476                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16477                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16478                }
16479                assert!(
16480                    goodbyes.contains(&(bind_channel, bind_epoch)),
16481                    "{goodbyes:?}"
16482                );
16483                assert!(rig
16484                    .handler
16485                    .registry
16486                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16487                    .unwrap()
16488                    .is_some());
16489            }
16490            assert_eq!(ended, Some(true));
16491            // The co-tenant stayed up through the change, and closed only when
16492            // the scope ended, by the drain rule rather than by the commit.
16493            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16494        }
16495
16496        /// Each row of the drain table on one set of routes: the owner's, a
16497        /// bare carrier's, and a targeted carrier's to each of its targets.
16498        #[tokio::test]
16499        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16500            struct Case {
16501                name: &'static str,
16502                change: fn(&mut ScopeRecord),
16503                /// Closed routes by index: owner->plexus, aft->plexus,
16504                /// broca->plexus, broca->other.
16505                closed: [Option<RouteCloseReason>; 4],
16506            }
16507            use RouteCloseReason::*;
16508            let cases = [
16509                Case {
16510                    name: "a carrier entry removed",
16511                    change: |r| {
16512                        r.carriers.retain(|c| {
16513                            c.principal
16514                                != Principal::Reserved {
16515                                    module_id: AFT.to_string(),
16516                                }
16517                        })
16518                    },
16519                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16520                },
16521                Case {
16522                    name: "a target removed from a carrier",
16523                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16524                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
16525                },
16526                Case {
16527                    name: "a bare carrier narrowed to targets",
16528                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16529                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16530                },
16531                Case {
16532                    name: "delegates turned off",
16533                    change: |r| r.attributes.delegates = false,
16534                    closed: [Some(ScopeDelegationChanged); 4],
16535                },
16536                Case {
16537                    name: "agent_id changed",
16538                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16539                    closed: [Some(ScopeDelegationChanged); 4],
16540                },
16541                Case {
16542                    name: "a carrier added, child owners changed, the record re-sent",
16543                    change: |r| {
16544                        r.carriers.push(carrier(MAGIC, None));
16545                        r.child_owners.push(Principal::Reserved {
16546                            module_id: MAGIC.to_string(),
16547                        });
16548                    },
16549                    closed: [None; 4],
16550                },
16551                Case {
16552                    name: "a target added",
16553                    change: |r| {
16554                        r.carriers[1]
16555                            .targets
16556                            .as_mut()
16557                            .unwrap()
16558                            .push("third".to_string())
16559                    },
16560                    closed: [None; 4],
16561                },
16562                Case {
16563                    name: "delegates turned on",
16564                    change: |r| r.attributes.delegates = true,
16565                    closed: [None; 4],
16566                },
16567            ];
16568            for case in cases {
16569                let mut rig = rig().await;
16570                rig.sync(vec![session(1)]).await;
16571                let scope = || Some(rig_selector("s", Some(1)));
16572                let mut routes = [
16573                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
16574                    rig.bound(Some(AFT), PLEXUS, scope()).await,
16575                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
16576                    rig.bound(Some(BROCA), OTHER, scope()).await,
16577                ];
16578                let mut record = session(1);
16579                (case.change)(&mut record);
16580                rig.sync(vec![record]).await;
16581                for (index, expected) in case.closed.iter().enumerate() {
16582                    let route = &mut routes[index];
16583                    match expected {
16584                        Some(reason) => {
16585                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
16586                            assert_eq!(
16587                                route.closed_reason(),
16588                                *reason,
16589                                "{}: route {index}",
16590                                case.name
16591                            );
16592                        }
16593                        None => {
16594                            assert!(rig.live(route), "{}: route {index} closed", case.name);
16595                            assert!(
16596                                route.untouched(),
16597                                "{}: route {index} was told something",
16598                                case.name
16599                            );
16600                        }
16601                    }
16602                }
16603            }
16604        }
16605
16606        #[tokio::test]
16607        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16608            // Removed, and replaced by a higher epoch.
16609            for next in [Vec::new(), vec![session(2)]] {
16610                let mut rig = rig().await;
16611                rig.sync(vec![session(1)]).await;
16612                let mut route = rig
16613                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16614                    .await;
16615                rig.sync(next).await;
16616                assert!(!rig.live(&route));
16617                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16618            }
16619
16620            // A child whose parent ends: its routes close as parent-ended, the
16621            // child stays live, and routes under the parent close as ended.
16622            let mut rig = rig().await;
16623            let mut child = session(1);
16624            child.scope_ref = "child".to_string();
16625            child.kind = ScopeKind::Worker;
16626            child.parent = Some(ScopeParent::new(
16627                Principal::Reserved {
16628                    module_id: OWNER.to_string(),
16629                },
16630                "s".to_string(),
16631                1,
16632            ));
16633            rig.sync(vec![session(1), child.clone()]).await;
16634            let mut child_route = rig
16635                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16636                .await;
16637            assert_eq!(
16638                child_route.stamp().unwrap().parent_state,
16639                Some(ParentState::Linked)
16640            );
16641            rig.sync(vec![child]).await;
16642            assert!(!rig.live(&child_route));
16643            assert_eq!(
16644                child_route.closed_reason(),
16645                RouteCloseReason::ScopeParentEnded
16646            );
16647        }
16648
16649        #[tokio::test]
16650        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16651        ) {
16652            let mut rig = rig().await;
16653            rig.sync(vec![session(1)]).await;
16654            let mut route = rig
16655                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16656                .await;
16657            let before = rig.forwarding.published_scope_tag(OWNER, "s");
16658            rig.sync(vec![session(1)]).await;
16659            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16660            assert!(rig.live(&route) && route.untouched());
16661
16662            // A call in flight on the route when another carrier is added. A
16663            // forwarded REQUEST holds one credit on the route's flow until the
16664            // module answers; the router takes it exactly like this.
16665            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16666                .forwarding
16667                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16668                .unwrap()
16669            else {
16670                panic!("the route is bound");
16671            };
16672            binding.flow.acquire_tagged(9, false).await.unwrap();
16673            let mut widened = session(1);
16674            widened.carriers.push(carrier(MAGIC, None));
16675            rig.sync(vec![widened]).await;
16676            assert!(rig.live(&route) && route.untouched());
16677            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16678            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16679            // The call's credit is still held on an open flow, so its answer
16680            // will be delivered: closing the route would have closed the flow.
16681            assert_eq!(binding.flow.in_flight(), 1);
16682            binding
16683                .flow
16684                .acquire_tagged(10, false)
16685                .await
16686                .expect("the flow is still open");
16687        }
16688
16689        /// A swap's superseded endpoint keeps its routes until drained; ending
16690        /// the scope closes them there too.
16691        #[tokio::test]
16692        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16693            let mut rig = rig().await;
16694            rig.sync(vec![session(1)]).await;
16695            let mut on_incumbent = rig
16696                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16697                .await;
16698
16699            // Swap plexus: register a candidate and cut over, leaving the
16700            // incumbent superseded with the route still on it.
16701            let (candidate, _candidate_rx) = wide_ctx(9);
16702            let registration = rig
16703                .handler
16704                .registry
16705                .register_candidate_with_control_ops(
16706                    manifest(PLEXUS, PROTOCOL_VERSION),
16707                    PROTOCOL_VERSION,
16708                    candidate.connection_id,
16709                    module_baseline_control_ops(),
16710                )
16711                .unwrap();
16712            rig.forwarding
16713                .register_candidate_module_connection(
16714                    candidate.connection_id,
16715                    PLEXUS.to_string(),
16716                    PROTOCOL_VERSION,
16717                    manifest_concurrency(&registration.manifest),
16718                    candidate.egress.clone(),
16719                )
16720                .unwrap();
16721            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16722            rig.handler
16723                .registry
16724                .promote_candidate(PLEXUS)
16725                .unwrap()
16726                .unwrap();
16727            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16728
16729            rig.sync(Vec::new()).await;
16730            assert!(!rig.live(&on_incumbent));
16731            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16732            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16733            let goodbye = incumbent_rx
16734                .try_recv()
16735                .expect("the superseded endpoint is told")
16736                .frame;
16737            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16738        }
16739    }
16740}
16741
16742#[cfg(test)]
16743mod concurrency_default_exposure_tests {
16744    use super::*;
16745
16746    fn hello_body(role_json: &str) -> Vec<u8> {
16747        format!(
16748            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16749        )
16750        .into_bytes()
16751    }
16752
16753    fn manifest_from(body: &[u8]) -> ModuleManifest {
16754        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16755        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16756            .expect("manifest parses")
16757    }
16758
16759    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16760
16761    #[test]
16762    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16763        let body = hello_body(&format!(
16764            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16765        ));
16766        let manifest = manifest_from(&body);
16767        // Precondition: serde really resolved it to the default, so the typed
16768        // manifest alone cannot answer the question this probe exists for.
16769        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16770        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16771    }
16772
16773    #[test]
16774    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16775        let body = hello_body(&format!(
16776            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16777        ));
16778        let manifest = manifest_from(&body);
16779        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16780        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16781    }
16782
16783    #[test]
16784    fn non_management_roles_are_never_reported() {
16785        let body = hello_body(
16786            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16787        );
16788        let manifest = manifest_from(&body);
16789        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16790    }
16791}