Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27    },
28    scope::{
29        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31        SCOPE_SYNC_OP,
32    },
33    session::{
34        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37        ROLE_VERSIONS_FIELD,
38    },
39    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40    Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46    caller_identity::CallerIdentity,
47    capability_requirements::{
48        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50        RuntimeModule,
51    },
52    child_roster::DaemonShutdownFlag,
53    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
54    forwarding::{
55        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
56        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
57        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
58    },
59    observability::{
60        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
61    },
62    privileged_prompt::{prompt_text, PrivilegedOperation},
63    provenance::{
64        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
65    },
66    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
67    router::{RouteCtx, RouterError},
68    scopes::{HelloLaunchNonces, ScopeTable},
69    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
70    stderr_tail::{CaptureState, TailEntry},
71    supervise::{
72        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
73        SupervisorHandle, SwapHelloAdmission,
74    },
75    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
76};
77
78/// Lowest envelope version this subc build will negotiate.
79///
80/// Module HELLO negotiation is exact: peers must use the daemon's locked
81/// protocol version. Older and newer peers receive `version_unsupported` and
82/// are not registered.
83pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
84
85const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
86const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
87const CAP_PING_PONG: &str = "ping_pong_v1";
88const CAP_SESSION_ATTACH: &str = "session_attach_v1";
89const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
90
91const SUBC_CONTROL_OPS: &[&str] = &[
92    ops::ACTIVITY_RECORD,
93    ops::ACTIVITY_QUERY,
94    ops::ACTIVITY_SUBSCRIBE,
95    ops::SERVER_DESCRIBE,
96    ops::CATALOG_LIST,
97    ops::ROUTE_OPEN,
98    ops::ROUTE_POLL,
99    ops::ROUTE_CLOSING,
100    ops::ROUTE_CLOSED,
101    ops::SUPERVISOR_LIST,
102    ops::SUPERVISOR_RESTART,
103    ops::SUPERVISOR_SWAP,
104    ops::SUPERVISOR_RELOAD,
105    ops::SUPERVISOR_RESCAN,
106    ops::SUPERVISOR_RELEASE_RESERVED,
107    ops::SUPERVISOR_SET_ENABLED,
108    ops::SUPERVISOR_HEALTH_PROBE,
109    ops::SUPERVISOR_HEALTH,
110    ops::SUPERVISOR_STDERR_TAIL,
111    ops::SUPERVISOR_TERMINALS,
112    ops::SUPERVISOR_ROUTES,
113    ops::SUPERVISOR_PROVENANCE,
114    ops::SUPERVISOR_SPAWN_SNAPSHOT,
115    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
116];
117
118/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
119/// announced by a capability in `server.describe` instead, and clients check
120/// that capability before sending it. `ck daemon --json` prints `subc_ops`
121/// verbatim and scripts read that output, so a new name there would change it.
122/// A malformed body for one of these is still refused `invalid_control_body`
123/// rather than as an unknown op.
124const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
125
126mod rescan_checked;
127use rescan_checked::{RescanMode, RescanPlan};
128
129const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
130    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
131    "supervisor.live_roots",
132    SCOPE_SYNC_OP,
133    SCOPE_APPLY_OP,
134    SCOPE_DESCRIBE_OP,
135    "operator.confirm",
136];
137
138/// Module-originated ops the daemon answers but does not advertise in
139/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
140/// belongs to is incomplete, so no module is told it works before it does.
141const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
142
143const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
144
145/// How long subc waits for a module to ack a relayed route.bind before returning
146/// `module_timeout`. The ack waits on the module's own configure, which for AFT
147/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
148/// and DB-open work — on a cold page cache or a large repo that legitimately
149/// exceeds a couple of seconds. The default is generous because rejecting a VALID
150/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
151/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
152pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
153
154/// How many CONSECUTIVE full-budget relay timeouts against one target module
155/// open that module's bind-relay breaker.
156///
157/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
158/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
159/// so three consecutive full-budget timeouts take ~36s to observe: every client
160/// whose open contributed to opening the breaker had already given up on its
161/// own. That is what makes opening the breaker unable to turn a call that would
162/// have succeeded into a refusal — it can only make an already-failing module
163/// fail faster.
164///
165/// Two would be reachable inside one default deadline. One would convict a
166/// module on a single cold-cache bind, which is exactly the valid-but-slow case
167/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
168pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
169
170/// How long a module's bind-relay breaker stays open before exactly one
171/// `route.open` is let through as a probe.
172///
173/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
174/// re-pays a full-budget stall almost continuously, and the breaker stops being
175/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
176/// deadline: a client that starts retrying after the module recovers has to get
177/// a probe opportunity inside its own deadline, or the breaker converts a
178/// recovered module into a failed call — the failure it exists to prevent,
179/// pointed the other way.
180///
181/// 20s sits between those with room on both sides, and it caps what a wedged
182/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
183/// rather than one per `route.open` per connection. The stall that motivated
184/// this, with its measurements, is written up in
185/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
186/// waited the whole budget out.
187pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
188
189const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
190const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
191
192fn reload_verdict(
193    configured: &Path,
194    spawned_from: Option<&Path>,
195    image: subc_control::RunningImageAgreement,
196) -> PendingReloadVerdict {
197    let path = match spawned_from {
198        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
199            ReloadPathAgreement::Match
200        }
201        Some(spawned_from) => ReloadPathAgreement::Mismatch {
202            configured: configured.to_path_buf(),
203            spawned_from: spawned_from.to_path_buf(),
204        },
205        None => ReloadPathAgreement::Unavailable {
206            reason: if matches!(
207                image,
208                subc_control::RunningImageAgreement::Unavailable {
209                    reason: subc_control::RunningImageUnavailableReason::NotRunning
210                }
211            ) {
212                ReloadPathUnavailableReason::NotRunning
213            } else {
214                ReloadPathUnavailableReason::SpawnedPathUnavailable
215            },
216        },
217    };
218    PendingReloadVerdict {
219        path,
220        image,
221        spec: None,
222    }
223}
224
225fn launch_spec_agreement(
226    configured: &crate::supervise::ModuleSpec,
227    running: bool,
228    spawned: Option<&crate::supervise::LaunchSpecDigest>,
229) -> subc_control::LaunchSpecAgreement {
230    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
231    if !running {
232        return Agreement::Unavailable {
233            reason: Reason::NotRunning,
234        };
235    }
236    match spawned {
237        None => Agreement::Unavailable {
238            reason: Reason::SpawnedSpecUnavailable,
239        },
240        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
241            Agreement::Match
242        }
243        Some(_) => Agreement::Mismatch,
244    }
245}
246
247fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
248    if configured == spawned_from {
249        return true;
250    }
251    #[cfg(windows)]
252    {
253        // Windows spawn records the resolved verbatim path (\\?\...), while
254        // configuration keeps its original spelling. Resolve both with the OS
255        // instead of stripping prefixes or folding case, which could conflate
256        // distinct names in case-sensitive directories. Different resolved
257        // paths still differ even if their files have identical content.
258        match (
259            std::fs::canonicalize(configured),
260            std::fs::canonicalize(spawned_from),
261        ) {
262            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
263            _ => false,
264        }
265    }
266    #[cfg(not(windows))]
267    {
268        false
269    }
270}
271
272#[derive(Clone)]
273struct DaemonProvenanceFacts {
274    build: DaemonBuildProvenance,
275    pid: Option<u32>,
276    started_at_ms: Option<u64>,
277    start_clock: Option<crate::clock::StartClock>,
278    executable_path: Option<PathBuf>,
279    executable_identity: Option<SpawnedFileIdentity>,
280    process_start_time: Option<u64>,
281    probe: ExecutableIdentityProbe,
282}
283
284impl Default for DaemonProvenanceFacts {
285    fn default() -> Self {
286        Self {
287            build: DaemonBuildProvenance {
288                build_git_sha: None,
289                build_lock_digest: None,
290            },
291            pid: None,
292            started_at_ms: None,
293            start_clock: None,
294            executable_path: None,
295            executable_identity: None,
296            process_start_time: None,
297            probe: ExecutableIdentityProbe::default(),
298        }
299    }
300}
301
302#[derive(Debug, Clone)]
303struct SupervisorRescanContext {
304    supervisor: Supervisor,
305    config_path: PathBuf,
306    configured_port: Option<u16>,
307    storage_config: Option<crate::daemon_config::StorageConfig>,
308    admission_facts_carrier_module_id: Option<String>,
309    admission_facts_targets: Option<Vec<String>>,
310    scope_authority_owners: Vec<String>,
311    privileged_operator_prompt: PrivilegedOperatorPrompt,
312    /// Counts executed rescans that set out to change the supervised modules'
313    /// staged or applied configuration, counted before the first change so a
314    /// rescan that fails partway still counts. With the daemon incarnation it
315    /// names the state a `supervisor.rescan_checked` preview was computed
316    /// against.
317    generation: Arc<std::sync::atomic::AtomicU64>,
318}
319
320/// Refusal labels passed to `observe_route_open_refusal` that mean the target
321/// module is not serving right now, and so open or extend an outage in the
322/// route outage tracker. Every one of them is only reachable after the target
323/// was found in the registry, which is what keeps an arbitrary client-chosen
324/// id from ever creating tracker state.
325///
326/// Deliberately absent: `not_registered` and `removed` (the id may be
327/// anything a client sent, and a removed module is gone on purpose),
328/// `protocol_none` (such a module never serves routes, so nothing is out),
329/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
330/// capability and admission-facts refusals (they refuse the caller, not a
331/// module outage), and `relay_reservation_failed` (its code ranges over
332/// capacity limits as well as a vanished connection). Capacity, breaker,
333/// relay-timeout and module-rejection refusals do not pass through that
334/// function at all; the breaker logs its own transitions.
335///
336/// The two not-serving refusals that bypass that function record themselves
337/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
338/// `required_capability_unprovided` is not tracked: the module itself is up,
339/// and the outage belongs to the missing provider.
340const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
341    "reloading",
342    "supervisor_not_live",
343    "registration_not_active",
344    "no_forwarding_connection",
345    "relay_send_failed",
346];
347
348/// Real channel-0 control handler for subc itself.
349#[derive(Clone)]
350pub struct ControlHandler {
351    registry: Arc<Registry>,
352    activity: Option<crate::activity::ActivityLedger>,
353    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
354    forwarding: Arc<ForwardingTable>,
355    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
356    supervisor: SupervisorHandle,
357    subc_capabilities: Arc<[String]>,
358    /// Daemon-wide route.bind relay budget. Used as the fallback when the
359    /// target module has no per-module override in
360    /// `route_bind_relay_timeouts`.
361    route_bind_relay_timeout: Duration,
362    /// Per-module route.bind relay budget overrides, keyed by module id. When
363    /// `handle_route_open` resolves the deadline for a target module, a
364    /// per-module entry wins over the daemon-wide value above.
365    route_bind_relay_timeouts: BTreeMap<String, Duration>,
366    /// Per-target-module bind-relay breaker state. Shared with the forwarding
367    /// table, which is where a new module connection resets it.
368    route_bind_breakers: RouteBindBreakers,
369    /// Live relay admissions keyed by target module. Shared through the
370    /// forwarding table so cloned or separately built handlers enforce one cap.
371    route_bind_concurrency: RouteBindConcurrency,
372    /// Start and end of each module's not-serving period as seen by
373    /// `route.open`, so an outage gets one line at each edge instead of only
374    /// the per-refusal INFO lines. Taken from the forwarding table, so every
375    /// handler built over one table shares it.
376    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
377    /// Consecutive relay timeouts that open a module's breaker.
378    route_bind_breaker_threshold: u32,
379    /// How long a breaker stays open before one probe is admitted.
380    route_bind_breaker_cooldown: Duration,
381    health_probe_timeout: Duration,
382    /// Central storage policy. When set, each registering module receives its
383    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
384    storage_config: Option<crate::daemon_config::StorageConfig>,
385    /// The machine id established at boot, served on every HELLO_ACK and on
386    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
387    /// changes the file, never this value. `None` serves no id.
388    machine_id: Option<crate::machine_id::MachineId>,
389    admission_facts_carrier_module_id: Option<String>,
390    admission_facts_targets: Option<Vec<String>>,
391    /// Scope records with their sync authorities and tombstones; see
392    /// `crate::scopes`. Shared by clones of this handler, so every connection
393    /// reads and writes one table.
394    scopes: Arc<RwLock<ScopeTable>>,
395    /// Wakes the expiry loop when an accepted scope change may have added a
396    /// deadline, so the loop can wait without ticking while no live scope has
397    /// one. A permit is stored when nothing waits, so no wake is lost.
398    scope_deadline_added: Arc<tokio::sync::Notify>,
399    /// The configured `scope_authority_owners`, kept so a rescan can report a
400    /// changed value as needing a daemon restart; rescan never applies it.
401    scope_authority_owners: Vec<String>,
402    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
403    privileged_operator_prompt: PrivilegedOperatorPrompt,
404    /// The local and peer address of each accepted TCP client connection, read
405    /// from the daemon's own socket, so a privileged request can name its caller.
406    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
407    #[cfg(test)]
408    caller_lookup: Option<TestCallerLookup>,
409    /// The launch nonce each module connection presented at HELLO, which is how
410    /// a `scope.sync` is matched to the owner's current launch.
411    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
412    rescan: Option<SupervisorRescanContext>,
413    connected_clients: ConnectedClients,
414    counters: DaemonCounters,
415    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
416    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
417    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
418    /// their usual severity (see [`log_requirement_events`]). A handler built
419    /// without one has a flag that is never set.
420    daemon_shutdown: DaemonShutdownFlag,
421    daemon_provenance: DaemonProvenanceFacts,
422    #[cfg(test)]
423    control_dispatch_delay: Option<Duration>,
424    #[cfg(test)]
425    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
426    /// Holds `route.open` after the route.bind is queued to the module and
427    /// before it is marked enqueued, so a test can make the module's reply (or
428    /// a client teardown) land exactly inside that window.
429    #[cfg(test)]
430    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
431    /// Holds a HELLO after its registry entry is reserved and before its
432    /// forwarding endpoint is installed, so a test can read the catalog and
433    /// open routes while the module is registered but not yet routable. A
434    /// blocking barrier because the HELLO handler is synchronous; the test
435    /// runs the HELLO on its own thread.
436    #[cfg(test)]
437    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
438}
439
440impl fmt::Debug for ControlHandler {
441    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
442        f.debug_struct("ControlHandler")
443            .field("registry", &self.registry)
444            .field("forwarding", &self.forwarding)
445            .field("process_liveness", &self.process_liveness.is_some())
446            .field("supervisor", &self.supervisor)
447            .field("subc_capabilities", &self.subc_capabilities)
448            .finish()
449    }
450}
451
452struct RouteOpenRequest {
453    target: RouteTarget,
454    identity: BindIdentity,
455    consumer_identity: Option<ConsumerIdentity>,
456    consumer_capabilities: Option<Vec<String>>,
457    role_versions: Option<BTreeMap<String, String>>,
458    admission_facts: Option<serde_json::Value>,
459    scope: Option<ScopeSelector>,
460}
461
462#[derive(Clone, Copy)]
463enum RouteOpenPhase {
464    Admission,
465    ProjectRoot,
466    EgressReservation,
467    BindSend,
468    BindAckWait,
469}
470
471/// Store elapsed time for each route-open phase, including attempts that end
472/// in an early refusal or backend error. Admission also includes rechecks and
473/// commit bookkeeping performed between the other timed phases.
474struct RouteOpenTimings {
475    module_id: String,
476    principal: String,
477    admission: Duration,
478    project_root: Duration,
479    egress_reservation: Duration,
480    bind_send: Duration,
481    bind_ack_wait: Duration,
482    phase: Option<RouteOpenPhase>,
483    phase_started_at: Instant,
484}
485
486impl RouteOpenTimings {
487    fn new() -> Self {
488        Self {
489            module_id: String::new(),
490            // Refusals before attestation must not claim a verified principal.
491            principal: "unresolved".to_string(),
492            admission: Duration::ZERO,
493            project_root: Duration::ZERO,
494            egress_reservation: Duration::ZERO,
495            bind_send: Duration::ZERO,
496            bind_ack_wait: Duration::ZERO,
497            phase: Some(RouteOpenPhase::Admission),
498            phase_started_at: Instant::now(),
499        }
500    }
501
502    fn finish(&mut self) {
503        let elapsed = self.phase_started_at.elapsed();
504        match self.phase.take() {
505            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
506            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
507            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
508            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
509            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
510            None => {}
511        }
512    }
513
514    fn enter(&mut self, phase: RouteOpenPhase) {
515        self.finish();
516        self.phase = Some(phase);
517        self.phase_started_at = Instant::now();
518    }
519}
520
521struct RouteBindReservationGuard {
522    forwarding: Arc<ForwardingTable>,
523    endpoint: ModuleEndpointId,
524    relay_corr: u64,
525    armed: bool,
526}
527
528struct ModuleControlRpcGuard {
529    forwarding: Arc<ForwardingTable>,
530    endpoint: ModuleEndpointId,
531    corr: u64,
532    armed: bool,
533}
534
535impl ModuleControlRpcGuard {
536    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
537        Self {
538            forwarding,
539            endpoint,
540            corr,
541            armed: true,
542        }
543    }
544
545    fn disarm(&mut self) {
546        self.armed = false;
547    }
548}
549
550impl Drop for ModuleControlRpcGuard {
551    fn drop(&mut self) {
552        if self.armed {
553            let _ = self
554                .forwarding
555                .cancel_module_control_rpc(self.endpoint, self.corr);
556        }
557    }
558}
559
560impl RouteBindReservationGuard {
561    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
562        Self {
563            forwarding,
564            endpoint,
565            relay_corr,
566            armed: true,
567        }
568    }
569
570    fn release_and_disarm(&mut self) {
571        if !self.armed {
572            return;
573        }
574        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
575            self.endpoint,
576            self.relay_corr,
577            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
578        ) {
579            send_goodbye_target_best_effort(
580                &self.forwarding.counters(),
581                &target,
582                "canceled route.bind",
583            );
584        }
585        self.armed = false;
586    }
587
588    fn disarm(&mut self) {
589        self.armed = false;
590    }
591}
592
593impl Drop for RouteBindReservationGuard {
594    fn drop(&mut self) {
595        self.release_and_disarm();
596    }
597}
598
599/// Per-target-module circuit breaker around the `route.bind` relay.
600///
601/// The connection reader is serial per connection, so a module whose `on_bind`
602/// sits on the ack blocks every LATER frame on the connections that call it,
603/// including calls to unrelated modules. This does not make any module's bind
604/// fast; it stops the daemon paying the full budget again and again for a
605/// condition it has already observed.
606///
607/// State is keyed by TARGET MODULE and shared by every connection: a wedged
608/// module wedges everyone, so what one connection learned should protect the
609/// rest.
610///
611/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
612/// relay to that module has actually timed out, and is removed again when a
613/// relay is accepted or the module reconnects, so it cannot grow with traffic
614/// or with modules that behave.
615///
616/// # Why a `std` mutex here is not the head-of-line defect again
617///
618/// Acquisition never awaits. The critical section is a hash lookup plus a few
619/// integer updates, with no I/O and no `.await` inside it, so a reader task
620/// cannot be descheduled behind it the way it can behind
621/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
622/// primitive, held for the same kind of work, as the refusal counter this very
623/// path already increments.
624///
625/// It is also NOT on the data-plane splice path: only `route.open` and module
626/// registration touch it, so bound-route frames gain no state check and no
627/// contention.
628#[derive(Debug, Clone, Default)]
629pub(crate) struct RouteBindBreakers {
630    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
631}
632
633#[derive(Debug, Clone, Default)]
634pub(crate) struct RouteBindConcurrency {
635    modules: Arc<Mutex<HashMap<String, usize>>>,
636}
637
638struct RouteBindConcurrencyGuard {
639    concurrency: RouteBindConcurrency,
640    module_id: String,
641}
642
643impl RouteBindConcurrency {
644    /// Admit without waiting. Waiting here would move the bind stall from the
645    /// module reply to a semaphore and restore reader head-of-line blocking.
646    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
647        let mut modules = self
648            .modules
649            .lock()
650            .expect("route.bind concurrency mutex poisoned");
651        let in_flight = modules.entry(module_id.to_string()).or_default();
652        if *in_flight >= limit {
653            return Err(*in_flight);
654        }
655        *in_flight += 1;
656        Ok(RouteBindConcurrencyGuard {
657            concurrency: self.clone(),
658            module_id: module_id.to_string(),
659        })
660    }
661}
662
663impl Drop for RouteBindConcurrencyGuard {
664    fn drop(&mut self) {
665        let mut modules = self
666            .concurrency
667            .modules
668            .lock()
669            .expect("route.bind concurrency mutex poisoned");
670        let remove = {
671            let in_flight = modules
672                .get_mut(&self.module_id)
673                .expect("admitted route.bind has a concurrency entry");
674            *in_flight -= 1;
675            *in_flight == 0
676        };
677        if remove {
678            modules.remove(&self.module_id);
679        }
680    }
681}
682
683#[derive(Debug, Default)]
684struct ModuleBreakerState {
685    /// Relay timeouts observed with no accepted relay in between.
686    consecutive_timeouts: u32,
687    /// `Some` while the breaker is open: the instant the cooldown expires and
688    /// the next arrival may probe. `None` means closed.
689    cooldown_until: Option<Instant>,
690    /// A half-open probe has been admitted and has not settled yet. This is
691    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
692    /// that read the cooldown, so concurrent opens arriving at the moment the
693    /// cooldown expires cannot all decide that they are the probe.
694    probe_in_flight: Option<Arc<()>>,
695}
696
697/// What the breaker decided for one `route.open`, before any relay work.
698enum RouteBindAdmission<'a> {
699    Admitted {
700        guard: RouteBindBreakerGuard<'a>,
701        /// This open is the single half-open probe, so the transition is worth
702        /// one log line.
703        probe: bool,
704    },
705    Refused {
706        consecutive_timeouts: u32,
707        /// What is left of the cooldown. Zero when the refusal is because the
708        /// one probe is already in flight rather than because the cooldown has
709        /// not elapsed.
710        retry_in: Duration,
711        probe_in_flight: bool,
712    },
713}
714
715/// An outstanding admission, which must be told how its relay settled.
716///
717/// `Drop` settles it as inconclusive, so an early return between admission and
718/// the relay -- or the whole handler being cancelled when the client
719/// disconnects -- releases a half-open probe slot instead of leaving the
720/// breaker wedged half-open with no further probes.
721struct RouteBindBreakerGuard<'a> {
722    breakers: RouteBindBreakers,
723    module_id: &'a str,
724    probe_token: Option<Arc<()>>,
725    settled: bool,
726}
727
728impl RouteBindBreakerGuard<'_> {
729    /// The module answered within the budget and took the bind. THE ONLY
730    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
731    /// breaker, which is a transition worth logging.
732    fn record_accepted(&mut self) -> bool {
733        self.settled = true;
734        self.breakers.record_accepted(self.module_id)
735    }
736
737    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
738    /// COUNTS TOWARD OPENING.
739    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
740        self.settled = true;
741        self.breakers.record_timeout(
742            self.module_id,
743            self.probe_token.as_ref(),
744            threshold,
745            cooldown,
746        )
747    }
748
749    /// Everything else: the module REJECTED the bind, its connection went away
750    /// mid-relay, or the waiter was cancelled.
751    ///
752    /// None of these is evidence that a module is slow, and each already has
753    /// its own refusal with its own code. A module that rejects a bind in
754    /// microseconds is healthy and must never be convicted for it; a module
755    /// that died has said nothing about the module that replaces it. So these
756    /// neither increment nor reset the count -- they only release a probe slot.
757    fn record_inconclusive(&mut self) {
758        self.settled = true;
759        self.breakers
760            .record_inconclusive(self.module_id, self.probe_token.as_ref());
761    }
762}
763
764impl Drop for RouteBindBreakerGuard<'_> {
765    fn drop(&mut self) {
766        if !self.settled {
767            self.breakers
768                .record_inconclusive(self.module_id, self.probe_token.as_ref());
769        }
770    }
771}
772
773/// The breaker moved to open, reported so the caller can log it outside the
774/// lock. Opening is rare and load-bearing; the refusals that follow are
775/// frequent and are counted rather than logged.
776struct BreakerOpened {
777    consecutive_timeouts: u32,
778    /// True when a failed probe re-opened an already-open breaker, which reads
779    /// very differently in a log from a first opening.
780    reopened_after_probe: bool,
781}
782
783impl RouteBindBreakers {
784    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
785        self.modules
786            .lock()
787            .expect("route.bind breaker mutex poisoned")
788    }
789
790    /// Decide whether this `route.open` may attempt its relay. Takes the map
791    /// lock and nothing else, and never awaits.
792    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
793        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
794            probe: probe_token.is_some(),
795            guard: RouteBindBreakerGuard {
796                breakers: self.clone(),
797                module_id,
798                probe_token,
799                settled: false,
800            },
801        };
802
803        let mut modules = self.lock();
804        let Some(state) = modules.get_mut(module_id) else {
805            return admitted(None);
806        };
807        let Some(cooldown_until) = state.cooldown_until else {
808            return admitted(None);
809        };
810        if state.probe_in_flight.is_some() {
811            return RouteBindAdmission::Refused {
812                consecutive_timeouts: state.consecutive_timeouts,
813                retry_in: Duration::ZERO,
814                probe_in_flight: true,
815            };
816        }
817        let now = Instant::now();
818        if now < cooldown_until {
819            return RouteBindAdmission::Refused {
820                consecutive_timeouts: state.consecutive_timeouts,
821                retry_in: cooldown_until - now,
822                probe_in_flight: false,
823            };
824        }
825        let token = Arc::new(());
826        state.probe_in_flight = Some(Arc::clone(&token));
827        admitted(Some(token))
828    }
829
830    fn record_accepted(&self, module_id: &str) -> bool {
831        self.lock()
832            .remove(module_id)
833            .is_some_and(|state| state.cooldown_until.is_some())
834    }
835
836    fn record_timeout(
837        &self,
838        module_id: &str,
839        probe_token: Option<&Arc<()>>,
840        threshold: u32,
841        cooldown: Duration,
842    ) -> Option<BreakerOpened> {
843        let mut modules = self.lock();
844        let state = modules.entry(module_id.to_string()).or_default();
845        let was_open = state.cooldown_until.is_some();
846        let was_probe = Self::owns_probe(state, probe_token);
847        if was_probe {
848            state.probe_in_flight = None;
849        }
850        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
851        if state.consecutive_timeouts < threshold {
852            return None;
853        }
854        state.cooldown_until = Some(Instant::now() + cooldown);
855        Some(BreakerOpened {
856            consecutive_timeouts: state.consecutive_timeouts,
857            reopened_after_probe: was_open && was_probe,
858        })
859    }
860
861    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
862        // A relay can finish after the breaker was reset or after it opened
863        // again and started a new probe. Only the guard whose token matches the
864        // active probe may release it, so a late relay never frees a newer probe.
865        state
866            .probe_in_flight
867            .as_ref()
868            .zip(token)
869            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
870    }
871
872    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
873        if let Some(state) = self.lock().get_mut(module_id) {
874            if Self::owns_probe(state, probe_token) {
875                state.probe_in_flight = None;
876            }
877        }
878    }
879
880    /// Discard what was learned about a module, because the process it was
881    /// learned about is gone. Returns the discarded count when it was non-zero.
882    ///
883    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
884    /// `module_id` is a configuration identity that outlives any particular
885    /// child; what the breaker observed was the process behind the module
886    /// connection of the moment. When a new connection registers under that id
887    /// the verdict's subject no longer exists, so the verdict is stale by
888    /// construction rather than merely likely to be wrong. Keeping it would
889    /// apply a dead process's record to a live one, which is the same defect
890    /// class this breaker exists to stop the daemon committing.
891    ///
892    /// A half-open probe in flight is discarded with the rest: it was a
893    /// question about the old process.
894    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
895        self.lock()
896            .remove(module_id)
897            .map(|state| state.consecutive_timeouts)
898            .filter(|discarded| *discarded > 0)
899    }
900
901    /// Open breakers, for the `server.describe` counters object. `None` when
902    /// none is open, so the key stays absent rather than present-and-empty.
903    ///
904    /// This is the operator's answer to "is this module refusing instantly or
905    /// is it fine?", which look identical from a client that retries and then
906    /// succeeds.
907    fn open_snapshot(&self) -> Option<serde_json::Value> {
908        let now = Instant::now();
909        let modules = self.lock();
910        let open = modules
911            .iter()
912            .filter_map(|(module_id, state)| {
913                let cooldown_until = state.cooldown_until?;
914                Some((
915                    module_id.clone(),
916                    serde_json::json!({
917                        "consecutive_timeouts": state.consecutive_timeouts,
918                        "cooldown_remaining_ms":
919                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
920                        "probe_in_flight": state.probe_in_flight.is_some(),
921                    }),
922                ))
923            })
924            .collect::<serde_json::Map<String, serde_json::Value>>();
925        (!open.is_empty()).then_some(serde_json::Value::Object(open))
926    }
927}
928
929impl ControlHandler {
930    pub fn new(registry: Arc<Registry>) -> Self {
931        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
932    }
933
934    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
935        let counters = forwarding.counters();
936        // Taken from the forwarding table rather than created here, so that the
937        // breaker a `route.open` consults is the same one a module's
938        // registration resets, however many handlers are built over one table.
939        let route_bind_breakers = forwarding.route_bind_breakers();
940        let route_bind_concurrency = forwarding.route_bind_concurrency();
941        let route_outages = forwarding.route_outages();
942        Self {
943            registry,
944            activity: None,
945            activity_streams: Arc::new(Mutex::new(HashMap::new())),
946            forwarding,
947            process_liveness: None,
948            supervisor: SupervisorHandle::new(),
949            subc_capabilities: Arc::from([
950                CAP_MANIFEST_REGISTRATION.to_string(),
951                CAP_CHANNEL_LIFECYCLE.to_string(),
952                CAP_PING_PONG.to_string(),
953                CAP_SESSION_ATTACH.to_string(),
954                CAP_ADMISSION_FACTS_RELAY.to_string(),
955                CAP_SCOPES_V1.to_string(),
956                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
957                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
958                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
959                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
961                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
962                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
963                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
964            ]),
965            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
966            route_bind_relay_timeouts: BTreeMap::new(),
967            route_bind_breakers,
968            route_bind_concurrency,
969            route_outages,
970            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
971            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
972            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
973            storage_config: None,
974            machine_id: None,
975            admission_facts_carrier_module_id: None,
976            admission_facts_targets: None,
977            scopes: Arc::new(RwLock::new(ScopeTable::new(
978                crate::daemon_config::default_scope_authority_owners(),
979            ))),
980            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
981            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
982            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
983            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
984            #[cfg(test)]
985            caller_lookup: None,
986            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
987            rescan: None,
988            connected_clients: ConnectedClients::new(),
989            counters,
990            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
991            daemon_shutdown: DaemonShutdownFlag::default(),
992            daemon_provenance: DaemonProvenanceFacts::default(),
993            #[cfg(test)]
994            control_dispatch_delay: None,
995            #[cfg(test)]
996            provenance_probe_override: None,
997            #[cfg(test)]
998            route_bind_relay_sent_hold: None,
999            #[cfg(test)]
1000            hello_registry_to_forwarding_hold: None,
1001        }
1002    }
1003
1004    /// Set the central storage policy: registering modules then receive their
1005    /// resolved storage descriptor in HELLO_ACK.
1006    pub fn with_storage_config(
1007        mut self,
1008        storage_config: Option<crate::daemon_config::StorageConfig>,
1009    ) -> Self {
1010        self.storage_config = storage_config;
1011        self
1012    }
1013
1014    /// Open the ledger on its own writer thread; an unavailable sink does not
1015    /// prevent the daemon from serving routing or supervisor requests.
1016    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1017        self.activity = Some(crate::activity::ActivityLedger::open(
1018            run_dir.join("activity.jsonl"),
1019            incarnation,
1020        ));
1021        self.supervisor.set_activity(self.activity.clone());
1022        self
1023    }
1024
1025    /// Set the machine id served to every registering module (HELLO_ACK) and on
1026    /// `server.describe`.
1027    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1028        self.machine_id = machine_id;
1029        self
1030    }
1031
1032    /// Configure the exact reserved module and target ids permitted to relay
1033    /// opaque admission facts. Config-file loading validates this authority;
1034    /// this builder keeps the same policy available to embedded test daemons.
1035    pub fn with_admission_facts_config(
1036        mut self,
1037        carrier_module_id: Option<String>,
1038        targets: Option<Vec<String>>,
1039    ) -> Self {
1040        self.admission_facts_carrier_module_id = carrier_module_id;
1041        self.admission_facts_targets = targets;
1042        self
1043    }
1044
1045    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1046    /// Replaces the scope table with an empty one under the new list, so call it
1047    /// while building the handler, before any module can sync.
1048    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1049        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1050        self.scope_authority_owners = owners;
1051        self
1052    }
1053
1054    /// Set the startup policy before installing the rescan context or serving requests.
1055    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1056        self.privileged_operator_prompt = mode;
1057        self
1058    }
1059
1060    /// Override the route.bind relay timeout. Used by tests that assert the
1061    /// timeout path so they don't block on the production-safe default.
1062    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1063        self.route_bind_relay_timeout = timeout;
1064        self
1065    }
1066
1067    /// Install per-module route.bind relay budget overrides. A module id
1068    /// listed here wins over the daemon-wide default set via
1069    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1070    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1071    /// the same `Duration` the bind path will use.
1072    pub fn with_route_bind_relay_timeouts(
1073        mut self,
1074        timeouts: impl IntoIterator<Item = (String, Duration)>,
1075    ) -> Self {
1076        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1077        self
1078    }
1079
1080    /// Resolve the route.bind relay budget for a specific target module id.
1081    /// Per-module overrides win; the daemon-wide value (set via
1082    /// `with_route_bind_relay_timeout` or the built-in default) is the
1083    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1084    /// the same resolution `handle_route_open` will use.
1085    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1086        self.route_bind_relay_timeouts
1087            .get(module_id)
1088            .copied()
1089            .unwrap_or(self.route_bind_relay_timeout)
1090    }
1091
1092    /// Override the per-module bind-relay breaker policy.
1093    ///
1094    /// Used by tests, which cannot spend three production budgets opening a
1095    /// breaker or twenty seconds waiting for its cooldown. The production
1096    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1097    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1098    /// reasoning for the numbers.
1099    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1100        self.route_bind_breaker_threshold = threshold.max(1);
1101        self.route_bind_breaker_cooldown = cooldown;
1102        self
1103    }
1104
1105    #[cfg(test)]
1106    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1107        self.health_probe_timeout = timeout;
1108        self
1109    }
1110
1111    #[cfg(test)]
1112    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1113        self.control_dispatch_delay = Some(delay);
1114        self
1115    }
1116
1117    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1118    /// announce it is parked, once to be released. Build the barrier for two
1119    /// parties, the test being the other one.
1120    #[cfg(test)]
1121    pub(crate) fn with_route_bind_relay_sent_hold(
1122        mut self,
1123        hold: Arc<tokio::sync::Barrier>,
1124    ) -> Self {
1125        self.route_bind_relay_sent_hold = Some(hold);
1126        self
1127    }
1128
1129    /// A HELLO waits on `hold` twice between reserving its registry entry and
1130    /// installing its forwarding endpoint: once to announce it is parked, once
1131    /// to be released. Build the barrier for two parties, the test being the
1132    /// other one, and run the HELLO on a thread of its own.
1133    #[cfg(test)]
1134    pub(crate) fn with_hello_registry_to_forwarding_hold(
1135        mut self,
1136        hold: Arc<std::sync::Barrier>,
1137    ) -> Self {
1138        self.hello_registry_to_forwarding_hold = Some(hold);
1139        self
1140    }
1141
1142    #[cfg(test)]
1143    fn hold_between_registry_and_forwarding(&self) {
1144        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1145            hold.wait();
1146            hold.wait();
1147        }
1148    }
1149
1150    pub fn with_process_liveness(
1151        mut self,
1152        process_liveness: Arc<dyn ModuleProcessLiveness>,
1153    ) -> Self {
1154        self.process_liveness = Some(process_liveness);
1155        self
1156    }
1157
1158    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1159        self.supervisor = supervisor;
1160        self.supervisor.set_activity(self.activity.clone());
1161        self
1162    }
1163
1164    pub fn with_daemon_provenance(
1165        mut self,
1166        pid: u32,
1167        started_at_ms: u64,
1168        executable_path: Option<PathBuf>,
1169        build_git_sha: Option<String>,
1170        build_lock_digest: Option<String>,
1171    ) -> Self {
1172        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1173        let process_start_time = process_start_time(pid);
1174        self.daemon_provenance = DaemonProvenanceFacts {
1175            build: DaemonBuildProvenance {
1176                build_git_sha,
1177                build_lock_digest,
1178            },
1179            pid: Some(pid),
1180            started_at_ms: Some(started_at_ms),
1181            start_clock: None,
1182            executable_path,
1183            executable_identity,
1184            process_start_time,
1185            probe: ExecutableIdentityProbe::default(),
1186        };
1187        self
1188    }
1189
1190    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1191        self.daemon_provenance.start_clock = Some(clock);
1192        self
1193    }
1194
1195    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1196    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1197        self.scopes
1198            .write()
1199            .unwrap_or_else(|poisoned| poisoned.into_inner())
1200            .set_wall_clock(Arc::new(clock));
1201        self
1202    }
1203
1204    #[cfg(test)]
1205    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1206        self.provenance_probe_override = Some(result);
1207        self
1208    }
1209
1210    /// Install the configured module set and its reserved capability bindings.
1211    /// Bindings are configuration-scoped and may point at a provider that has not
1212    /// been installed yet, so this does not require the bound module to exist.
1213    pub fn with_capability_config(
1214        self,
1215        modules: impl IntoIterator<Item = (String, bool)>,
1216        reserved_capabilities: BTreeMap<String, String>,
1217    ) -> Self {
1218        self.capability_evaluator
1219            .configure(modules, reserved_capabilities);
1220        self
1221    }
1222
1223    /// Share the supervisor's daemon-shutdown flag with this handler, so
1224    /// requirement changes after the daemon has begun its announced shutdown
1225    /// are not reported as faults.
1226    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1227        self.daemon_shutdown = flag;
1228        self
1229    }
1230
1231    pub fn with_supervisor_rescan(
1232        mut self,
1233        supervisor: Supervisor,
1234        config_path: impl Into<PathBuf>,
1235        configured_port: Option<u16>,
1236    ) -> Self {
1237        self.rescan = Some(SupervisorRescanContext {
1238            supervisor,
1239            config_path: config_path.into(),
1240            configured_port,
1241            storage_config: self.storage_config.clone(),
1242            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1243            admission_facts_targets: self.admission_facts_targets.clone(),
1244            scope_authority_owners: self.scope_authority_owners.clone(),
1245            privileged_operator_prompt: self.privileged_operator_prompt,
1246            generation: Arc::default(),
1247        });
1248        self
1249    }
1250
1251    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1252        self.connected_clients = connected_clients;
1253        self
1254    }
1255
1256    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1257        Arc::clone(&self.forwarding)
1258    }
1259
1260    pub(crate) fn counters(&self) -> DaemonCounters {
1261        self.counters.clone()
1262    }
1263
1264    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1265    /// requirement event without depending on an operator polling a status command.
1266    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1267        tokio::spawn(async move {
1268            loop {
1269                self.capability_evaluator
1270                    .wait_for_change_or_deadline()
1271                    .await;
1272                self.refresh_capability_requirements();
1273            }
1274        });
1275    }
1276
1277    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1278    /// deadline passed during sleep is enforced on the first tick after wake.
1279    /// While no live scope carries a deadline the loop waits for an accepted
1280    /// scope change instead of ticking, so a daemon without deadlines never
1281    /// wakes for expiry.
1282    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1283        tokio::spawn(async move {
1284            loop {
1285                let has_deadlines = self
1286                    .scopes
1287                    .read()
1288                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1289                    .has_deadlines();
1290                if !has_deadlines {
1291                    self.scope_deadline_added.notified().await;
1292                    continue;
1293                }
1294                tokio::time::sleep(Duration::from_secs(1)).await;
1295                if let Err(error) = self.sweep_expired_scopes() {
1296                    warn!(%error, "scope expiry sweep failed");
1297                }
1298            }
1299        });
1300    }
1301
1302    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1303        let mut table = self
1304            .scopes
1305            .write()
1306            .unwrap_or_else(|poisoned| poisoned.into_inner());
1307        let swept = table.sweep_expired();
1308        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1309        drop(table);
1310        self.close_scope_drained_routes(drained);
1311        Ok(())
1312    }
1313
1314    fn publish_scope_changes(
1315        &self,
1316        changes: &[crate::scopes::ScopeTagChange],
1317        expired: &[crate::scopes::ScopeExpired],
1318    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1319        if changes.is_empty() {
1320            return Ok(Vec::new());
1321        }
1322        let drained = self
1323            .forwarding
1324            .publish_scope_changes(changes)
1325            .map_err(RouterError::Forwarding)?;
1326        let mut counts = HashMap::new();
1327        for route in &drained {
1328            *counts
1329                .entry((
1330                    route.scope.owner.as_str(),
1331                    route.scope.scope_ref.as_str(),
1332                    route.scope.tag.scope_epoch,
1333                ))
1334                .or_insert(0usize) += 1;
1335        }
1336        for scope in expired {
1337            let routes_closed = counts
1338                .get(&(
1339                    scope.owner.as_str(),
1340                    scope.scope_ref.as_str(),
1341                    scope.scope_epoch,
1342                ))
1343                .copied()
1344                .unwrap_or(0);
1345            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1346                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1347                routes_closed, "scope expired");
1348        }
1349        Ok(drained)
1350    }
1351
1352    fn runtime_capability_snapshot(
1353        &self,
1354    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1355        let runtime = self
1356            .supervisor
1357            .list()
1358            .into_iter()
1359            .map(|module| {
1360                let status = module.status().map_err(|err| {
1361                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1362                })?;
1363                Ok(RuntimeModule {
1364                    module_id: status.module_id,
1365                    state: status.state,
1366                    enabled: status.enabled,
1367                })
1368            })
1369            .collect::<Result<Vec<_>, RouterError>>()?;
1370        // Include every registration, published or not. The capability
1371        // evaluator counts a module as providing its capabilities as soon as it
1372        // has registered, even before `catalog.list` shows it, so a HELLO in
1373        // progress does not change other modules' required-capability
1374        // verdicts. See `Registry::list_registered_modules`.
1375        let registrations = self.registry.list_registered_modules().map_err(|err| {
1376            RouterError::backend(
1377                0,
1378                0,
1379                format!("failed to list capability registrations: {err}"),
1380            )
1381        })?;
1382        let registrations = registrations
1383            .into_iter()
1384            .map(|registration| RegisteredModule {
1385                module_id: registration.manifest.module_id,
1386                module_version: registration.manifest.module_version,
1387                capabilities: registration.manifest.capabilities,
1388            })
1389            .collect();
1390        Ok((runtime, registrations))
1391    }
1392
1393    /// The capability side effects of a module becoming the active registration
1394    /// for its id: cache its manifest (warning if its claims drifted), run the
1395    /// deny census when its declarations call for one, and recompute the
1396    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1397    /// candidate's does not, and the supervisor does it at promotion instead,
1398    /// through [`crate::supervise::SwapPromotionObserver`].
1399    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1400        let cached_registration = RegisteredModule {
1401            module_id: registration.manifest.module_id.clone(),
1402            module_version: registration.manifest.module_version.clone(),
1403            capabilities: registration.manifest.capabilities.clone(),
1404        };
1405        if self.capability_evaluator.record_hello(&cached_registration) {
1406            warn!(
1407                module_id = %cached_registration.module_id,
1408                "capability claims drifted from the cached manifest"
1409            );
1410        }
1411        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1412            self.enforce_capability_denies();
1413        }
1414        self.refresh_capability_requirements();
1415    }
1416
1417    /// Point the shared supervisor handle at this handler for swap promotions.
1418    /// Called wherever a handler is put behind the `Arc` the router serves, so
1419    /// it can be held weakly.
1420    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1421        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1422            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1423        self.supervisor.set_swap_promotion_observer(observer);
1424    }
1425
1426    pub fn refresh_capability_requirements(&self) {
1427        match self.runtime_capability_snapshot() {
1428            Ok((runtime, registrations)) => {
1429                log_requirement_events(
1430                    self.capability_evaluator
1431                        .evaluate_now(&runtime, &registrations),
1432                    &self.daemon_shutdown,
1433                );
1434            }
1435            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1436        }
1437    }
1438
1439    /// Reconcile only live, attested route bindings after a capability deny edge
1440    /// or target claim was added. This is deliberately a control-plane census:
1441    /// the opaque forwarding hot path must not grow a per-frame capability check.
1442    fn enforce_capability_denies(&self) {
1443        let registrations = match self.registry.list_registered_modules() {
1444            Ok(snapshot) => snapshot,
1445            Err(err) => {
1446                warn!(error = %err, "failed to read registrations for capability deny census");
1447                return;
1448            }
1449        };
1450        let manifests = registrations
1451            .into_iter()
1452            .map(|registration| {
1453                (
1454                    registration.manifest.module_id.clone(),
1455                    registration.manifest,
1456                )
1457            })
1458            .collect::<BTreeMap<_, _>>();
1459        let census = match self.forwarding.route_census(None) {
1460            Ok(census) => census,
1461            Err(err) => {
1462                warn!(error = %err, "failed to read route census for capability deny enforcement");
1463                return;
1464            }
1465        };
1466
1467        for (target_module_id, routes) in census {
1468            let Some(target_manifest) = manifests.get(&target_module_id) else {
1469                continue;
1470            };
1471            let mut closed_routes = Vec::new();
1472            let mut module_goodbyes = Vec::new();
1473            for route in routes {
1474                let Principal::Reserved {
1475                    module_id: opening_module_id,
1476                } = &route.principal
1477                else {
1478                    continue;
1479                };
1480                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1481                    continue;
1482                };
1483                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1484                    continue;
1485                };
1486
1487                match self.forwarding.release_client_route(
1488                    route.goodbye_target.connection_id,
1489                    route.goodbye_target.channel,
1490                    route.goodbye_target.epoch,
1491                ) {
1492                    Ok(RouteRelease::Removed(module_goodbye)) => {
1493                        warn!(
1494                            opening_module_id,
1495                            target_module_id,
1496                            capability,
1497                            "force-closing route because an attested capability deny edge now matches"
1498                        );
1499                        closed_routes.push(route);
1500                        module_goodbyes.push(module_goodbye);
1501                    }
1502                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1503                    Err(err) => warn!(
1504                        opening_module_id,
1505                        target_module_id,
1506                        capability,
1507                        error = %err,
1508                        "failed to force-close capability-denied route"
1509                    ),
1510                }
1511            }
1512
1513            if closed_routes.is_empty() {
1514                continue;
1515            }
1516            send_route_control_pushes(
1517                &self.forwarding,
1518                closed_routes,
1519                ClientControlPush::RouteClosed {
1520                    module_id: target_module_id,
1521                    channels: Vec::new(),
1522                    reason: RouteCloseReason::CapabilityDenied,
1523                    drained: false,
1524                    abandoned: 0,
1525                    excluded_subscriptions: 0,
1526                    terminal: Some(false),
1527                },
1528            );
1529            self.emit_route_goodbyes(module_goodbyes);
1530        }
1531    }
1532
1533    /// Why a registered module is not accepting new route binds, or `None` when
1534    /// it is. This is the module's effective readiness: its declared readiness
1535    /// first, then every `need: required` capability it declares evaluating to
1536    /// `provided`. `route.open` and `catalog.list` both read it here so the
1537    /// catalog never reports a module routable that `route.open` would refuse.
1538    fn not_ready_reason(
1539        &self,
1540        registration: &crate::registry::ModuleRegistration,
1541    ) -> Option<NotReadyReason> {
1542        if !registration.ready {
1543            return Some(NotReadyReason {
1544                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1545                capability: None,
1546            });
1547        }
1548        self.first_unprovided_required_capability(registration)
1549            .map(|capability| NotReadyReason {
1550                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1551                capability: Some(capability),
1552            })
1553    }
1554
1555    /// The lexicographically first capability this registration declares
1556    /// `need: required` whose evaluator verdict is not `provided`.
1557    ///
1558    /// The verdicts are the capability evaluator's own; nothing here decides
1559    /// what "provided" means. The evaluator counts a capability provided as
1560    /// soon as a module claiming it has REGISTERED, not once that module is
1561    /// ready. That distinction is what keeps two modules that require each
1562    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1563    /// is ready", each would wait for the other to become ready first and
1564    /// neither ever would. Do not tighten it to readiness.
1565    ///
1566    /// A required capability with no verdict at all means this registration's
1567    /// HELLO or catalog.update landed after the last recompute; recompute once
1568    /// rather than let a missing verdict read as either answer. If it is still
1569    /// missing (the recompute itself failed) the capability counts as
1570    /// unprovided: the refusal is retryable, and routing a module whose
1571    /// required provider is unknown is the outcome this check exists to stop.
1572    fn first_unprovided_required_capability(
1573        &self,
1574        registration: &crate::registry::ModuleRegistration,
1575    ) -> Option<String> {
1576        let required = registration
1577            .manifest
1578            .capabilities
1579            .iter()
1580            .flat_map(|declarations| declarations.requires.iter())
1581            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1582            .map(|requirement| requirement.capability.as_str())
1583            .collect::<BTreeSet<_>>();
1584        if required.is_empty() {
1585            return None;
1586        }
1587        let module_id = registration.manifest.module_id.as_str();
1588        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1589        if required
1590            .iter()
1591            .any(|capability| verdict(capability).is_none())
1592        {
1593            self.refresh_capability_requirements();
1594        }
1595        required
1596            .into_iter()
1597            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1598            .map(str::to_string)
1599    }
1600
1601    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1602        self.capability_evaluator
1603            .statuses()
1604            .into_iter()
1605            .map(capability_requirement_status)
1606            .collect()
1607    }
1608
1609    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1610    /// registration-release watch. The signal is what the supervisor waits on
1611    /// before spawning a replacement, so it must only fire once forwarding
1612    /// teardown is also done (see [`Self::cleanup_connection`] /
1613    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1614    /// admitted but its forwarding endpoint could not be installed.
1615    fn deregister_connection(
1616        &self,
1617        connection_id: ConnectionId,
1618        reason: RegistrationEndReason,
1619    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1620        self.registry
1621            .deregister_connection_with_reason(connection_id, reason)
1622    }
1623
1624    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1625        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1626            return None;
1627        }
1628        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1629            parse_client_control_request(&frame.body)
1630        else {
1631            return None;
1632        };
1633        Some(target_module_id(&target).to_string())
1634    }
1635
1636    pub(crate) fn route_open_capacity_refusal(
1637        &self,
1638        ctx: &RouteCtx,
1639        frame: &Frame,
1640        target_module_id: &str,
1641        in_flight: usize,
1642        limit: usize,
1643    ) -> Result<Frame, RouterError> {
1644        self.route_open_admission_refusal_frame(
1645            ctx,
1646            frame,
1647            target_module_id,
1648            "open_admission_full",
1649            (in_flight, limit),
1650            format!(
1651                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1652            ),
1653        )
1654    }
1655
1656    fn route_open_target_capacity_refusal(
1657        &self,
1658        ctx: &RouteCtx,
1659        frame: &Frame,
1660        target_module_id: &str,
1661        in_flight: usize,
1662    ) -> Result<Frame, RouterError> {
1663        self.route_open_admission_refusal_frame(
1664            ctx,
1665            frame,
1666            target_module_id,
1667            "target_binds_full",
1668            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1669            format!(
1670                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1671            ),
1672        )
1673    }
1674
1675    /// Admission pressure clears as existing binds settle, so its refusal must
1676    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1677    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1678    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1679    /// cannot currently reach its target; `module_timeout` would falsely claim
1680    /// that a wait expired. A new, cleaner code would be terminal to deployed
1681    /// clients, so it requires a client-tolerance rollout before daemon emission.
1682    fn route_open_admission_refusal_frame(
1683        &self,
1684        ctx: &RouteCtx,
1685        frame: &Frame,
1686        target_module_id: &str,
1687        reason: &'static str,
1688        (in_flight, limit): (usize, usize),
1689        message: impl Into<String>,
1690    ) -> Result<Frame, RouterError> {
1691        let code = error_codes::TARGET_UNAVAILABLE;
1692        self.counters.increment_route_open_refused(code);
1693        info!(
1694            target: "control",
1695            code,
1696            reason,
1697            module_id = ?target_module_id,
1698            connection_id = ctx.connection_id.get(),
1699            in_flight,
1700            limit,
1701            "route.open refused"
1702        );
1703        control_error_frame(frame, code, message.into())
1704    }
1705
1706    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1707    ///
1708    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1709    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1710    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1711    #[cfg(test)]
1712    pub fn handle_control(
1713        &self,
1714        connection_id: ConnectionId,
1715        frame: Frame,
1716    ) -> Result<Vec<Frame>, RouterError> {
1717        match frame.header.ty {
1718            FrameType::Ping => Ok(vec![pong(&frame)?]),
1719            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1720            FrameType::Goodbye => self.handle_goodbye(connection_id),
1721            ty => Ok(vec![control_error_frame(
1722                &frame,
1723                "unsupported_control_frame",
1724                format!("unsupported channel-0 frame {ty:?}"),
1725            )?]),
1726        }
1727    }
1728
1729    pub async fn handle_control_frame(
1730        &self,
1731        ctx: &RouteCtx,
1732        frame: Frame,
1733    ) -> Result<Vec<Frame>, RouterError> {
1734        self.handle_control_frame_timed(ctx, frame, None).await
1735    }
1736
1737    pub(crate) fn record_caller_endpoints(
1738        &self,
1739        connection: ConnectionId,
1740        endpoints: Option<(SocketAddr, SocketAddr)>,
1741    ) {
1742        let mut callers = self
1743            .caller_endpoints
1744            .lock()
1745            .unwrap_or_else(|p| p.into_inner());
1746        callers.remove(&connection);
1747        if let Some(endpoints) = endpoints {
1748            callers.insert(connection, endpoints);
1749        }
1750    }
1751
1752    #[cfg(test)]
1753    pub(crate) fn with_caller_lookup_for_test(
1754        mut self,
1755        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1756    ) -> Self {
1757        self.caller_lookup = Some(Arc::new(lookup));
1758        self
1759    }
1760
1761    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1762        let endpoints = self
1763            .caller_endpoints
1764            .lock()
1765            .unwrap_or_else(|p| p.into_inner())
1766            .get(&connection)
1767            .copied();
1768        #[cfg(test)]
1769        let lookup = self.caller_lookup.clone();
1770        tokio::task::spawn_blocking(move || {
1771            #[cfg(test)]
1772            if let Some(lookup) = lookup {
1773                return lookup(endpoints);
1774            }
1775            endpoints
1776                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1777                .unwrap_or_default()
1778        })
1779        .await
1780        .unwrap_or_default()
1781    }
1782
1783    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1784        frame.header.channel == 0
1785            && frame.header.ty == FrameType::Request
1786            && self
1787                .forwarding
1788                .module_endpoint_for_connection(ctx.connection_id)
1789                .is_ok_and(|endpoint| endpoint.is_none())
1790            && parse_client_control_request(&frame.body)
1791                .ok()
1792                .is_some_and(|request| privileged_operation(&request).is_some())
1793    }
1794
1795    pub(crate) async fn handle_control_frame_timed(
1796        &self,
1797        ctx: &RouteCtx,
1798        frame: Frame,
1799        dispatch_started_at: Option<StdInstant>,
1800    ) -> Result<Vec<Frame>, RouterError> {
1801        match frame.header.ty {
1802            FrameType::Ping => Ok(vec![pong(&frame)?]),
1803            FrameType::Hello => {
1804                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1805            }
1806            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1807            FrameType::Cancel => {
1808                // A Cancel on channel 0 names a waiting operator.confirm or a
1809                // subscription. Successful cancellation has no ordinary reply.
1810                if self
1811                    .forwarding
1812                    .operator_confirms()
1813                    .cancel(ctx.connection_id, frame.header.corr)
1814                    || self
1815                        .supervisor
1816                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1817                    || self.cancel_activity_subscription(ctx, &frame)?
1818                {
1819                    Ok(Vec::new())
1820                } else {
1821                    Ok(vec![control_error_frame(
1822                        &frame,
1823                        "unknown_subscription",
1824                        "no subscription has this correlation id",
1825                    )?])
1826                }
1827            }
1828            FrameType::Request => {
1829                // This additive operation is not part of the existing exhaustive
1830                // module-control enum. Probe the op before decoding that enum.
1831                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1832                if op
1833                    .as_ref()
1834                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1835                {
1836                    return self.handle_activity_record(ctx, frame).await;
1837                }
1838                if op.as_ref().is_some_and(|probe| {
1839                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1840                }) {
1841                    return self.handle_activity_read(ctx, frame).await;
1842                }
1843                if op
1844                    .as_ref()
1845                    .is_some_and(|probe| probe.op == "operator.confirm")
1846                {
1847                    return self.handle_operator_confirm(ctx, frame);
1848                }
1849                if op
1850                    .as_ref()
1851                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1852                {
1853                    return self.handle_scope_served_by(ctx.connection_id, frame);
1854                }
1855                if self
1856                    .forwarding
1857                    .module_endpoint_for_connection(ctx.connection_id)
1858                    .map_err(RouterError::Forwarding)?
1859                    .is_some()
1860                {
1861                    if !is_known_module_request_op(&frame.body) {
1862                        return Ok(vec![control_error_frame(
1863                            &frame,
1864                            "unsupported_control_frame",
1865                            "module-originated channel-0 REQUEST is not supported",
1866                        )?]);
1867                    }
1868                    let request = match parse_module_control_request_from_module(&frame.body) {
1869                        Ok(request) => request,
1870                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1871                            return Ok(vec![control_error_frame(
1872                                &frame,
1873                                "unsupported_control_frame",
1874                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1875                            )?])
1876                        }
1877                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1878                            return Ok(vec![control_error_frame(
1879                                &frame,
1880                                "invalid_control_body",
1881                                format!("malformed module control body: {err}"),
1882                            )?])
1883                        }
1884                    };
1885                    let op = module_control_request_op(&request);
1886                    let corr = frame.header.corr;
1887                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1888                    let result =
1889                        self.handle_module_control_request(ctx.connection_id, frame, request);
1890                    log_slow_control_dispatch(
1891                        dispatch_started_at,
1892                        op,
1893                        ctx.connection_id,
1894                        corr,
1895                        None,
1896                    );
1897                    return result;
1898                }
1899
1900                if is_known_module_request_op(&frame.body) {
1901                    return Ok(vec![control_error_frame(
1902                        &frame,
1903                        "not_registered",
1904                        "catalog.update requires an active module registration owned by this connection",
1905                    )?]);
1906                }
1907
1908                let request = match parse_client_control_request(&frame.body) {
1909                    Ok(request) => request,
1910                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1911                        return Ok(vec![control_error_frame(
1912                            &frame,
1913                            "unknown_control_op",
1914                            format!("unknown client control op: {err}"),
1915                        )?])
1916                    }
1917                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1918                        return Ok(vec![control_error_frame(
1919                            &frame,
1920                            "invalid_control_body",
1921                            format!("malformed client control body: {err}"),
1922                        )?])
1923                    }
1924                };
1925                let op = client_control_request_op(&request);
1926                let corr = frame.header.corr;
1927                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1928                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1929                #[cfg(test)]
1930                if let Some(delay) = self.control_dispatch_delay {
1931                    tokio::time::sleep(delay).await;
1932                }
1933                let result = self
1934                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1935                    .await;
1936                if let Some(timings) = route_open_timings.as_mut() {
1937                    timings.finish();
1938                }
1939                log_slow_control_dispatch(
1940                    dispatch_started_at,
1941                    op,
1942                    ctx.connection_id,
1943                    corr,
1944                    route_open_timings.as_ref(),
1945                );
1946                result
1947            }
1948            FrameType::Push => {
1949                let Some(endpoint) = self
1950                    .forwarding
1951                    .module_endpoint_for_connection(ctx.connection_id)
1952                    .map_err(RouterError::Forwarding)?
1953                else {
1954                    return Ok(vec![control_error_frame(
1955                        &frame,
1956                        "unsupported_control_frame",
1957                        "client-originated channel-0 PUSH is not supported",
1958                    )?]);
1959                };
1960                self.handle_status_update(endpoint, frame)
1961            }
1962            FrameType::Response | FrameType::Error
1963                if self
1964                    .forwarding
1965                    .module_endpoint_for_connection(ctx.connection_id)
1966                    .map_err(RouterError::Forwarding)?
1967                    .is_some() =>
1968            {
1969                self.handle_module_relay_response(ctx.connection_id, frame)
1970            }
1971            ty => Ok(vec![control_error_frame(
1972                &frame,
1973                "unsupported_control_frame",
1974                format!("unsupported channel-0 frame {ty:?}"),
1975            )?]),
1976        }
1977    }
1978
1979    pub fn cleanup_connection(
1980        &self,
1981        connection_id: ConnectionId,
1982    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1983        self.caller_endpoints
1984            .lock()
1985            .unwrap_or_else(|p| p.into_inner())
1986            .remove(&connection_id);
1987        self.forwarding
1988            .operator_confirms()
1989            .connection_closed(connection_id);
1990        self.cleanup_connection_with_end_reason(
1991            connection_id,
1992            RegistrationEndReason::ConnectionClosed,
1993        )
1994    }
1995
1996    fn cleanup_connection_with_end_reason(
1997        &self,
1998        connection_id: ConnectionId,
1999        requested_reason: RegistrationEndReason,
2000    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2001        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2002            self.registry
2003                .get_module_by_connection(connection_id)?
2004                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2005                .and_then(|module| module.registration_end_reason().ok().flatten())
2006                .unwrap_or(requested_reason)
2007        } else {
2008            requested_reason
2009        };
2010        let crash_closed = self
2011            .registry
2012            .get_module_by_connection(connection_id)?
2013            .and_then(|registration| {
2014                self.forwarding
2015                    .module_endpoint_for_connection(connection_id)
2016                    .ok()
2017                    .flatten()
2018                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2019                    .map(|routes| (registration.manifest.module_id, routes))
2020            });
2021        let crash_closed = crash_closed.map(|(module_id, routes)| {
2022            let terminal = match self.supervisor.get(&module_id) {
2023                None => false,
2024                Some(module) => match module.will_recover_after_connection_loss() {
2025                    Ok(will_recover) => !will_recover,
2026                    Err(err) => {
2027                        warn!(
2028                            %module_id,
2029                            error = %err,
2030                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2031                        );
2032                        false
2033                    }
2034                },
2035            };
2036            // The forwarding table gates all providers at the start of daemon
2037            // shutdown, before their connections are closed. An ordinary
2038            // module disconnect still reports crash if that gate is not set.
2039            let (reason, drained) = match self
2040                .forwarding
2041                .daemon_drain_result_for_connection(connection_id)
2042            {
2043                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2044                Ok(None) => (RouteCloseReason::Crash, false),
2045                Err(err) => {
2046                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2047                    (RouteCloseReason::Crash, false)
2048                }
2049            };
2050            (module_id, routes, reason, drained, terminal)
2051        });
2052        let registrations = self.deregister_connection(connection_id, end_reason);
2053        let cleanup = if crash_closed.is_some() {
2054            self.forwarding.cleanup_connection_counted(connection_id)
2055        } else {
2056            // A module connection's teardown needs the count of abandoned
2057            // route.bind relays for its route.closed notice below. Any other
2058            // connection, such as a client's, sends no such notice and needs
2059            // only its routes released, so it uses the route-only wrapper and
2060            // reports zero.
2061            self.forwarding
2062                .cleanup_connection(connection_id)
2063                .map(|released| crate::forwarding::ConnectionCleanup {
2064                    released,
2065                    abandoned_relays: 0,
2066                })
2067        };
2068        // The route.closed push waits for forwarding teardown because only
2069        // teardown knows how many pending route.bind relays it aborted. It still
2070        // goes out before the GOODBYEs for the released routes, and its targets
2071        // were captured above, before teardown removed those routes.
2072        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2073            let abandoned = cleanup
2074                .as_ref()
2075                .map_or(0, |cleanup| cleanup.abandoned_relays);
2076            send_route_control_pushes(
2077                &self.forwarding,
2078                routes,
2079                ClientControlPush::RouteClosed {
2080                    module_id,
2081                    channels: Vec::new(),
2082                    reason,
2083                    drained,
2084                    abandoned,
2085                    excluded_subscriptions: 0,
2086                    terminal: Some(terminal),
2087                },
2088            );
2089        }
2090        if let Ok(cleanup) = cleanup {
2091            self.emit_route_goodbyes(cleanup.released);
2092        }
2093        // Signal the registration-release watch only now that BOTH registry and
2094        // forwarding teardown are done, so a supervisor waiting to spawn a
2095        // replacement never observes release while old routes still exist.
2096        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2097            crate::supervise::notify_registration_release();
2098            self.capability_evaluator.wake_deadline_loop();
2099            self.refresh_capability_requirements();
2100        }
2101        self.supervisor.remove_spawn_subscribers(connection_id);
2102        self.activity_streams
2103            .lock()
2104            .unwrap_or_else(|p| p.into_inner())
2105            .retain(|(id, _), task| {
2106                if *id == connection_id {
2107                    task.abort();
2108                    false
2109                } else {
2110                    true
2111                }
2112            });
2113        // Sync authority dies with its connection, so the owner's next
2114        // connection can take it; the owner's scopes stay as they are.
2115        self.hello_launch_nonces
2116            .lock()
2117            .unwrap_or_else(|poisoned| poisoned.into_inner())
2118            .forget(connection_id);
2119        self.scopes
2120            .write()
2121            .unwrap_or_else(|poisoned| poisoned.into_inner())
2122            .release_connection(connection_id);
2123        registrations
2124    }
2125
2126    pub(crate) fn handle_route_goodbye(
2127        &self,
2128        connection_id: ConnectionId,
2129        route_channel: u16,
2130        route_epoch: u32,
2131    ) -> Result<bool, RouterError> {
2132        debug!(
2133            connection_id = connection_id.get(),
2134            route_channel, route_epoch, "handling route GOODBYE"
2135        );
2136        let RouteRelease::Removed(released_route) = self
2137            .forwarding
2138            .release_client_route(connection_id, route_channel, route_epoch)
2139            .map_err(RouterError::Forwarding)?
2140        else {
2141            return Ok(false);
2142        };
2143        self.emit_route_goodbyes(vec![released_route]);
2144        Ok(true)
2145    }
2146
2147    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2148        for released in released_routes {
2149            let frame = match Frame::build_with_version(
2150                released.negotiated_ver,
2151                FrameType::Goodbye,
2152                control_flags(),
2153                released.channel,
2154                released.epoch,
2155                0,
2156                Vec::new(),
2157            ) {
2158                Ok(frame) => frame,
2159                Err(err) => {
2160                    warn!(
2161                        route_channel = released.channel,
2162                        error = %err,
2163                        "failed to build route GOODBYE frame"
2164                    );
2165                    continue;
2166                }
2167            };
2168            if !released.close_on_delivery_failure() {
2169                crate::forwarding::send_module_route_goodbye(
2170                    &self.counters,
2171                    &released.sink,
2172                    frame,
2173                    released.module_id.as_deref(),
2174                    "client route released",
2175                );
2176                continue;
2177            }
2178            if let Err(err) = released.sink.try_send(frame) {
2179                warn!(
2180                    target_connection_id = released.connection_id.get(),
2181                    route_channel = released.channel,
2182                    error = %err,
2183                    "route GOODBYE was not delivered to client; closing target connection"
2184                );
2185                if self
2186                    .forwarding
2187                    .escalate_client_delivery_failure(
2188                        released.connection_id,
2189                        released.channel,
2190                        released.epoch,
2191                        CloseReason::new(
2192                            "route_goodbye_delivery_failed",
2193                            format!(
2194                                "failed to enqueue route GOODBYE for channel {}: {err}",
2195                                released.channel
2196                            ),
2197                        ),
2198                        crate::forwarding::UndeliveredFrame {
2199                            module_id: released.module_id.as_deref(),
2200                            sink: &released.sink,
2201                        },
2202                    )
2203                    .unwrap_or(false)
2204                {
2205                    self.counters.increment_goodbye_relay_client_failed();
2206                }
2207            }
2208        }
2209    }
2210
2211    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2212    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2213    /// own commit failed after the module had already accepted). Without this, a
2214    /// module that accepts late keeps a binding subc has torn down, so a later
2215    /// frame on that module channel could misdeliver if the channel is reused.
2216    ///
2217    /// Never closes the shared module connection on failure: a dropped notification
2218    /// only wastes a bounded amount of warm module-side state, which the module's
2219    /// own idle reaper reclaims. Only call this once the route.bind relay was
2220    /// actually enqueued to the module — if the relay send itself failed, the
2221    /// module never created a binding and there is nothing to tear down.
2222    fn send_abandoned_route_bind_goodbye(
2223        &self,
2224        module_sink: &crate::FrameSink,
2225        negotiated_ver: u8,
2226        module_channel: u16,
2227        module_epoch: u32,
2228    ) {
2229        let frame = match Frame::build_with_version(
2230            negotiated_ver,
2231            FrameType::Goodbye,
2232            control_flags(),
2233            module_channel,
2234            module_epoch,
2235            0,
2236            Vec::new(),
2237        ) {
2238            Ok(frame) => frame,
2239            Err(err) => {
2240                warn!(
2241                    route_channel = module_channel,
2242                    error = %err,
2243                    "failed to build GOODBYE for abandoned route.bind"
2244                );
2245                return;
2246            }
2247        };
2248        crate::forwarding::send_module_route_goodbye(
2249            &self.counters,
2250            module_sink,
2251            frame,
2252            None,
2253            "abandoned route.bind",
2254        );
2255    }
2256
2257    fn handle_hello(
2258        &self,
2259        connection_id: ConnectionId,
2260        sink: Option<crate::FrameSink>,
2261        frame: Frame,
2262    ) -> Result<Vec<Frame>, RouterError> {
2263        debug!(
2264            connection_id = connection_id.get(),
2265            corr = frame.header.corr,
2266            "handling HELLO"
2267        );
2268        // A module connection has one identity for its entire lifetime. A second
2269        // registration would leave the old registry owner behind while replacing
2270        // its forwarding endpoint and launch nonce.
2271        if self
2272            .registry
2273            .get_module_by_connection(connection_id)
2274            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2275            .is_some()
2276        {
2277            return Ok(vec![control_error_frame(
2278                &frame,
2279                "invalid_hello",
2280                "connection is already registered as a module",
2281            )?]);
2282        }
2283        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2284            Ok(value) => value,
2285            Err(err) => {
2286                return Ok(vec![control_error_frame(
2287                    &frame,
2288                    "invalid_hello",
2289                    format!("malformed HELLO body: {err}"),
2290                )?])
2291            }
2292        };
2293        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2294            return Ok(vec![control_error_frame(
2295                &frame,
2296                "invalid_capability_grammar",
2297                err.to_string(),
2298            )?]);
2299        }
2300        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2301            return Ok(vec![control_error_frame(
2302                &frame,
2303                "invalid_manifest",
2304                err.to_string(),
2305            )?]);
2306        }
2307        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2308            return Ok(vec![control_error_frame(
2309                &frame,
2310                "invalid_event_declaration",
2311                err.to_string(),
2312            )?]);
2313        }
2314        if let Some(provenance) = hello_value
2315            .get("manifest")
2316            .and_then(|manifest| manifest.get("provenance"))
2317        {
2318            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2319                return Ok(vec![control_error_frame(
2320                    &frame,
2321                    "invalid_manifest",
2322                    format!("malformed manifest provenance: {err}"),
2323                )?]);
2324            }
2325        }
2326        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2327            Ok(hello) => hello,
2328            Err(err) => {
2329                return Ok(vec![control_error_frame(
2330                    &frame,
2331                    "invalid_hello",
2332                    format!("malformed HELLO body: {err}"),
2333                )?])
2334            }
2335        };
2336
2337        if hello.protocol_ver != hello.manifest.protocol_ver {
2338            return Ok(vec![control_error_frame(
2339                &frame,
2340                "invalid_manifest",
2341                format!(
2342                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2343                    hello.protocol_ver, hello.manifest.protocol_ver
2344                ),
2345            )?]);
2346        }
2347
2348        if hello.manifest.module_id.trim().is_empty() {
2349            return Ok(vec![control_error_frame(
2350                &frame,
2351                "invalid_manifest",
2352                "manifest module_id must not be empty",
2353            )?]);
2354        }
2355
2356        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2357            Ok(negotiated_ver) => negotiated_ver,
2358            Err(message) => {
2359                return Ok(vec![control_error_frame(
2360                    &frame,
2361                    "version_unsupported",
2362                    message,
2363                )?])
2364            }
2365        };
2366
2367        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2368        // swap is open for this id, the only HELLO admitted as a second process
2369        // is the one carrying the candidate's launch nonce (the swap token), and
2370        // it registers into the candidate slot rather than being refused as a
2371        // duplicate. Run after the reserved gate, a reserved module's candidate
2372        // would be refused `reserved_module` for presenting a nonce that gate
2373        // does not know. See `SupervisorHandle::swap_hello_admission`.
2374        let swap_admission = self
2375            .supervisor
2376            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2377        if swap_admission == SwapHelloAdmission::Refused {
2378            warn!(
2379                module_id = %hello.manifest.module_id,
2380                connection_id = connection_id.get(),
2381                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2382            );
2383            return Ok(vec![control_error_frame(
2384                &frame,
2385                "swap_token_invalid",
2386                format!(
2387                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2388                    hello.manifest.module_id
2389                ),
2390            )?]);
2391        }
2392        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2393
2394        // Reserved-module identity gate: a module_id configured `reserved` may be
2395        // registered ONLY by the process subc spawned for it, proven by echoing the
2396        // one-time launch nonce subc injected. A non-reserved id has no recorded
2397        // nonce and always passes. This blocks a key-holder from impersonating a
2398        // security-boundary module (e.g. the credential vault) while the real one is
2399        // down/restarting and its registration slot is momentarily free. A swap
2400        // candidate has already proven the same thing with its own nonce above.
2401        if let Some(rejection) = (!swap_candidate)
2402            .then(|| {
2403                self.supervisor.reserved_hello_rejection(
2404                    &hello.manifest.module_id,
2405                    hello.launch_nonce.as_deref(),
2406                )
2407            })
2408            .flatten()
2409        {
2410            let message = match rejection {
2411                ReservedHelloRejection::Exact { module_id } => format!(
2412                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2413                ),
2414                ReservedHelloRejection::Prefix {
2415                    prefix,
2416                    owner_module_id,
2417                } => format!(
2418                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2419                    hello.manifest.module_id
2420                ),
2421            };
2422            return Ok(vec![control_error_frame(
2423                &frame,
2424                "reserved_module",
2425                message,
2426            )?]);
2427        }
2428
2429        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2430            &hello.manifest.module_id,
2431            hello.manifest.capabilities.as_ref(),
2432        );
2433        if let Some(refusal) = reserved_capability_refusals.first() {
2434            let capability = refusal.capability.clone();
2435            let bound_module = refusal.claimants[0].clone();
2436            log_duplicate_claim_events(reserved_capability_refusals);
2437            return Ok(vec![control_error_frame(
2438                &frame,
2439                "reserved_capability",
2440                format!(
2441                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2442                    capability, bound_module, hello.manifest.module_id
2443                ),
2444            )?]);
2445        }
2446
2447        // A connection that already opened client routes must not also register as
2448        // a module: cleanup would then release only one side and leak the other.
2449        if self
2450            .forwarding
2451            .connection_has_client_routes(connection_id)
2452            .map_err(RouterError::Forwarding)?
2453        {
2454            return Ok(vec![control_error_frame(
2455                &frame,
2456                "invalid_hello",
2457                "connection has open client routes and cannot also register as a module",
2458            )?]);
2459        }
2460
2461        // Kept for scope sync authority, which goes only to the connection that
2462        // presented the module's current launch nonce. Recorded before the
2463        // registration is attempted: a connection whose registration then fails
2464        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2465        self.hello_launch_nonces
2466            .lock()
2467            .unwrap_or_else(|poisoned| poisoned.into_inner())
2468            .record(connection_id, hello.launch_nonce.as_deref());
2469        let control_ops = effective_module_control_ops(hello.control_ops);
2470        // Built before anything is registered so an encoding failure leaves no
2471        // registry or forwarding state behind.
2472        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2473        if swap_candidate {
2474            return self.register_swap_candidate(
2475                connection_id,
2476                sink,
2477                &frame,
2478                hello.manifest,
2479                negotiated_ver,
2480                control_ops,
2481                hello_ack,
2482            );
2483        }
2484        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2485        // is refused, but `catalog.list` does not show the module and the
2486        // catalog generation does not move until it is published below, once
2487        // its forwarding endpoint exists. Listing it now would advertise a
2488        // module that `route.open` still refuses for having no forwarding
2489        // connection.
2490        let registration = match self.registry.register_unpublished_with_control_ops(
2491            hello.manifest,
2492            negotiated_ver,
2493            connection_id,
2494            control_ops,
2495        ) {
2496            Ok(registration) => registration,
2497            Err(RegistryError::DuplicateModuleId { module_id }) => {
2498                return Ok(vec![control_error_frame(
2499                    &frame,
2500                    "duplicate_module_id",
2501                    format!(
2502                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2503                    ),
2504                )?])
2505            }
2506            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2507                return Ok(vec![control_error_frame(
2508                    &frame,
2509                    "invalid_module_id",
2510                    err.to_string(),
2511                )?])
2512            }
2513            Err(err) => {
2514                return Ok(vec![control_error_frame(
2515                    &frame,
2516                    "registry_error",
2517                    err.to_string(),
2518                )?])
2519            }
2520        };
2521        #[cfg(test)]
2522        self.hold_between_registry_and_forwarding();
2523
2524        let reply = if let Some(sink) = sink {
2525            // The forwarding table's module store is also the daemon-to-module
2526            // control-RPC lane, so every HELLO gets a live endpoint even when the
2527            // manifest has no routable provider role. Non-routable modules still
2528            // cannot receive route.bind in production: `handle_route_open` checks
2529            // the registry manifest with `target_has_required_role` before the
2530            // only production call to `begin_route_bind_relay_for` below that
2531            // route.open path. The remaining direct relay callers are unit tests
2532            // and benchmark harnesses that construct forwarding state explicitly.
2533            //
2534            // The HELLO_ACK is queued by the forwarding table itself, before the
2535            // endpoint becomes visible, and is NOT returned as a reply. A module
2536            // reads HELLO_ACK first and exits on anything else; a reply is only
2537            // written after this handler returns, by which time a route.open on
2538            // another connection could already have queued a route.bind request
2539            // for this module ahead of it.
2540            let concurrency = manifest_concurrency(&registration.manifest);
2541            if let Err(err) = self.forwarding.register_module_connection_acked(
2542                connection_id,
2543                registration.manifest.module_id.clone(),
2544                negotiated_ver,
2545                concurrency,
2546                sink,
2547                hello_ack,
2548            ) {
2549                // Forwarding registration failed, so there is no forwarding
2550                // state to tear down. Remove the registry entry, which was never
2551                // published and so never listed or counted in the catalog
2552                // generation, and signal the release watch directly.
2553                if matches!(
2554                    self.deregister_connection(
2555                        connection_id,
2556                        RegistrationEndReason::RegistrationFailed,
2557                    ),
2558                    Ok(r) if !r.is_empty()
2559                ) {
2560                    crate::supervise::notify_registration_release();
2561                }
2562                return Ok(vec![control_error_frame(
2563                    &frame,
2564                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2565                        "invalid_hello"
2566                    } else {
2567                        forwarding_error_code(&err)
2568                    },
2569                    err.to_string(),
2570                )?]);
2571            }
2572            Vec::new()
2573        } else {
2574            // No sink means no forwarding endpoint, so nothing can be routed
2575            // ahead of the ack; it goes out as the reply.
2576            vec![hello_ack]
2577        };
2578
2579        // The module is routable now (or, without a sink, as routable as it
2580        // will ever be), so the catalog may list it. Publishing bumps the
2581        // catalog generation in the same registry critical section that makes
2582        // the module listed.
2583        if !self
2584            .registry
2585            .publish_registration(connection_id)
2586            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2587        {
2588            debug!(
2589                module_id = %registration.manifest.module_id,
2590                connection_id = connection_id.get(),
2591                "registration was removed before it could be published"
2592            );
2593        }
2594
2595        // Exposure over assumption: Concurrency's serde default is pinned to the
2596        // pre-field behavior (ModuleManaged), so a management surface that is
2597        // genuinely Serial and just never declared it inherits concurrent
2598        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2599        // turns "no module has been bitten yet" into the checkable claim "no
2600        // module is exposed" -- one read of the boot log instead of a fleet
2601        // audit. Detected from the raw HELLO bytes because the serde default
2602        // deliberately erases the absent/declared distinction from the type.
2603        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2604            info!(
2605                module_id = %registration.manifest.module_id,
2606                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2607            );
2608        }
2609
2610        self.apply_registration_capabilities(&registration);
2611
2612        self.supervisor
2613            .record_activity_registration(&registration.manifest.module_id);
2614
2615        info!(
2616            module_id = %registration.manifest.module_id,
2617            module_version = %registration.manifest.module_version,
2618            negotiated_ver,
2619            routable_provider = manifest_provides_routable_role(&registration.manifest),
2620            connection_id = connection_id.get(),
2621            "module registered"
2622        );
2623
2624        Ok(reply)
2625    }
2626
2627    /// Register a HELLO the swap gate admitted into the candidate slot of the
2628    /// registry and of forwarding, where it is reachable over its own
2629    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2630    /// nothing routes to it until the supervisor cuts over.
2631    ///
2632    /// Registry first (unpublished), then forwarding, then publication, the
2633    /// same order as an ordinary HELLO; a forwarding failure removes the
2634    /// registry entry again. The swap supervisor treats a published, ready
2635    /// candidate as registered and promotes it, and promotion needs the
2636    /// candidate's forwarding endpoint, so the candidate is published only
2637    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2638    /// recompute capability verdicts or run the capability deny census
2639    /// (`enforce_capability_denies`): both are about the module in the active
2640    /// slot, which `route.open` can target. A candidate sits in the candidate
2641    /// slot, so it neither provides capabilities to other modules nor can be
2642    /// routed to until promotion moves it into the active slot.
2643    #[allow(clippy::too_many_arguments)]
2644    fn register_swap_candidate(
2645        &self,
2646        connection_id: ConnectionId,
2647        sink: Option<crate::FrameSink>,
2648        frame: &Frame,
2649        manifest: ModuleManifest,
2650        negotiated_ver: u8,
2651        control_ops: Vec<String>,
2652        hello_ack: Frame,
2653    ) -> Result<Vec<Frame>, RouterError> {
2654        let module_id = manifest.module_id.clone();
2655        let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2656            manifest,
2657            negotiated_ver,
2658            connection_id,
2659            control_ops,
2660        ) {
2661            Ok(registration) => registration,
2662            Err(RegistryError::DuplicateModuleId { module_id }) => {
2663                return Ok(vec![control_error_frame(
2664                    frame,
2665                    "duplicate_module_id",
2666                    format!(
2667                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2668                    ),
2669                )?])
2670            }
2671            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2672                return Ok(vec![control_error_frame(
2673                    frame,
2674                    "invalid_module_id",
2675                    err.to_string(),
2676                )?])
2677            }
2678            Err(err) => {
2679                return Ok(vec![control_error_frame(
2680                    frame,
2681                    "registry_error",
2682                    err.to_string(),
2683                )?])
2684            }
2685        };
2686        #[cfg(test)]
2687        self.hold_between_registry_and_forwarding();
2688        let reply = if let Some(sink) = sink {
2689            // Same ordering as an ordinary HELLO: the forwarding table queues
2690            // the HELLO_ACK before the candidate endpoint is inserted, because
2691            // a module exits if its first frame after HELLO is anything else.
2692            let concurrency = manifest_concurrency(&registration.manifest);
2693            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2694                connection_id,
2695                module_id.clone(),
2696                negotiated_ver,
2697                concurrency,
2698                sink,
2699                hello_ack,
2700            ) {
2701                if matches!(
2702                    self.deregister_connection(
2703                        connection_id,
2704                        RegistrationEndReason::RegistrationFailed,
2705                    ),
2706                    Ok(r) if !r.is_empty()
2707                ) {
2708                    crate::supervise::notify_registration_release();
2709                }
2710                return Ok(vec![control_error_frame(
2711                    frame,
2712                    forwarding_error_code(&err),
2713                    err.to_string(),
2714                )?]);
2715            }
2716            Vec::new()
2717        } else {
2718            vec![hello_ack]
2719        };
2720        if !self
2721            .registry
2722            .publish_registration(connection_id)
2723            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2724        {
2725            debug!(
2726                module_id = %module_id,
2727                connection_id = connection_id.get(),
2728                "swap candidate registration was removed before it could be published"
2729            );
2730        }
2731        self.supervisor.mark_swap_candidate_admitted(&module_id);
2732        info!(
2733            module_id = %module_id,
2734            module_version = %registration.manifest.module_version,
2735            negotiated_ver,
2736            ready = registration.ready,
2737            connection_id = connection_id.get(),
2738            "swap candidate registered; not routable until cutover"
2739        );
2740        Ok(reply)
2741    }
2742
2743    fn build_hello_ack(
2744        &self,
2745        frame: &Frame,
2746        negotiated_ver: u8,
2747        module_id: &str,
2748    ) -> Result<Frame, RouterError> {
2749        let ack = ModuleHelloAckBody {
2750            negotiated_ver,
2751            subc_ops: module_subc_ops(),
2752            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2753            storage: self
2754                .storage_config
2755                .as_ref()
2756                .map(|cfg| cfg.descriptor_for(module_id)),
2757            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2758        };
2759        let body = serde_json::to_vec(&ack).map_err(|err| {
2760            RouterError::backend(
2761                0,
2762                frame.header.corr,
2763                format!("failed to encode HELLO_ACK: {err}"),
2764            )
2765        })?;
2766
2767        Frame::build_with_version(
2768            negotiated_ver,
2769            FrameType::HelloAck,
2770            control_flags(),
2771            0,
2772            0,
2773            frame.header.corr,
2774            body,
2775        )
2776        .map_err(RouterError::FrameBuild)
2777    }
2778
2779    async fn handle_client_control_request(
2780        &self,
2781        ctx: &RouteCtx,
2782        frame: Frame,
2783        request: ClientControlRequest,
2784        route_open_timings: &mut Option<RouteOpenTimings>,
2785    ) -> Result<Vec<Frame>, RouterError> {
2786        if let Some(operation) = privileged_operation(&request) {
2787            let caller = self.resolve_caller(ctx.connection_id).await;
2788            let op = client_control_request_op(&request);
2789            let text = prompt_text(operation, &caller);
2790            match self.privileged_operator_prompt {
2791                PrivilegedOperatorPrompt::Log => {
2792                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2793                        target = ?operation, caller = %caller.description(), summary = %text,
2794                        "privileged request would prompt the operator");
2795                }
2796                PrivilegedOperatorPrompt::Enforce => {
2797                    let key =
2798                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2799                    let outcome = self
2800                        .forwarding
2801                        .operator_confirms()
2802                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2803                        .await;
2804                    if outcome.code != "confirmed" {
2805                        return Ok(vec![outcome.frame(&frame)]);
2806                    }
2807                }
2808            }
2809        }
2810        match request {
2811            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2812            ClientControlRequest::CatalogList { module_id } => {
2813                self.handle_catalog_list(frame, module_id)
2814            }
2815            ClientControlRequest::RouteOpen {
2816                target,
2817                identity,
2818                consumer_identity,
2819                consumer_capabilities,
2820                role_versions,
2821                admission_facts,
2822                scope,
2823            } => {
2824                self.handle_route_open(
2825                    ctx,
2826                    frame,
2827                    RouteOpenRequest {
2828                        target,
2829                        identity,
2830                        consumer_identity,
2831                        consumer_capabilities,
2832                        role_versions,
2833                        admission_facts,
2834                        scope,
2835                    },
2836                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2837                )
2838                .await
2839            }
2840            ClientControlRequest::RoutePoll {
2841                route_channel,
2842                route_epoch,
2843                kind,
2844            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2845            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2846            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2847                self.handle_supervisor_spawn_snapshot(frame)
2848            }
2849            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2850                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2851            }
2852            ClientControlRequest::SupervisorRestart {
2853                module_id,
2854                drain_timeout_ms,
2855            } => {
2856                self.log_supervisor_request_received(
2857                    ctx,
2858                    frame.header.corr,
2859                    ops::SUPERVISOR_RESTART,
2860                    Some(&module_id),
2861                    None,
2862                )?;
2863                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2864                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2865                    .await
2866            }
2867            ClientControlRequest::SupervisorSwap {
2868                module_id,
2869                ready_timeout_ms,
2870            } => {
2871                self.log_supervisor_request_received(
2872                    ctx,
2873                    frame.header.corr,
2874                    ops::SUPERVISOR_SWAP,
2875                    Some(&module_id),
2876                    None,
2877                )?;
2878                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2879                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2880                    .await
2881            }
2882            ClientControlRequest::SupervisorReload { module_id } => {
2883                self.log_supervisor_request_received(
2884                    ctx,
2885                    frame.header.corr,
2886                    ops::SUPERVISOR_RELOAD,
2887                    Some(&module_id),
2888                    None,
2889                )?;
2890                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2891                self.handle_supervisor_reload(frame, module_id, requester)
2892                    .await
2893            }
2894            ClientControlRequest::SupervisorRescan { preview } => {
2895                if !preview {
2896                    self.log_supervisor_request_received(
2897                        ctx,
2898                        frame.header.corr,
2899                        ops::SUPERVISOR_RESCAN,
2900                        None,
2901                        None,
2902                    )?;
2903                }
2904                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2905                self.handle_supervisor_rescan(
2906                    frame,
2907                    if preview {
2908                        RescanMode::Preview
2909                    } else {
2910                        RescanMode::Execute
2911                    },
2912                    requester,
2913                )
2914                .await
2915            }
2916            ClientControlRequest::SupervisorRescanChecked {
2917                preview,
2918                candidate,
2919                expect,
2920            } => {
2921                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2922                    .await
2923            }
2924            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2925                self.log_supervisor_request_received(
2926                    ctx,
2927                    frame.header.corr,
2928                    ops::SUPERVISOR_RELEASE_RESERVED,
2929                    Some(&module_id),
2930                    None,
2931                )?;
2932                self.handle_supervisor_release_reserved(frame, module_id)
2933                    .await
2934            }
2935            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2936                self.log_supervisor_request_received(
2937                    ctx,
2938                    frame.header.corr,
2939                    ops::SUPERVISOR_SET_ENABLED,
2940                    Some(&module_id),
2941                    Some(enabled),
2942                )?;
2943                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2944                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2945                    .await
2946            }
2947            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2948                self.handle_supervisor_health_probe(frame, module_id).await
2949            }
2950            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2951            ClientControlRequest::SupervisorRoutes { module_id } => {
2952                self.handle_supervisor_routes(frame, module_id)
2953            }
2954            ClientControlRequest::SupervisorProvenance { module_id } => {
2955                self.handle_supervisor_provenance(frame, module_id).await
2956            }
2957            ClientControlRequest::SupervisorStderrTail {
2958                module_id,
2959                max_lines,
2960                max_bytes,
2961            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2962            ClientControlRequest::SupervisorTerminals { module_id } => {
2963                self.handle_supervisor_terminals(frame, module_id).await
2964            }
2965        }
2966    }
2967
2968    fn log_supervisor_request_received(
2969        &self,
2970        ctx: &RouteCtx,
2971        corr: u64,
2972        op: &'static str,
2973        module_id: Option<&str>,
2974        enabled: Option<bool>,
2975    ) -> Result<(), RouterError> {
2976        let caller = self.supervisor_requester(ctx, corr)?;
2977        let caller = principal_label(&caller);
2978        // `Option` fields are recorded only when present, so a request with no
2979        // module id (rescan) or no enabled flag simply omits that field.
2980        info!(
2981            target: "control",
2982            op,
2983            module_id,
2984            enabled,
2985            connection_id = ctx.connection_id.get(),
2986            caller = %caller,
2987            "supervisor request received"
2988        );
2989        Ok(())
2990    }
2991
2992    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2993        Ok(self
2994            .registry
2995            .get_module_by_connection(ctx.connection_id)
2996            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2997            .map(|registration| Principal::Reserved {
2998                module_id: registration.manifest.module_id,
2999            })
3000            .unwrap_or(Principal::Direct))
3001    }
3002
3003    fn handle_module_control_request(
3004        &self,
3005        connection_id: ConnectionId,
3006        frame: Frame,
3007        request: ModuleControlRequestFromModule,
3008    ) -> Result<Vec<Frame>, RouterError> {
3009        match request {
3010            ModuleControlRequestFromModule::CatalogUpdate {
3011                provides,
3012                capabilities,
3013                ready,
3014            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3015            ModuleControlRequestFromModule::LiveRoots {} => {
3016                let registered = self
3017                    .registry
3018                    .get_module_by_connection(connection_id)
3019                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3020                let Some(registration) = registered else {
3021                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3022                };
3023                let response = self
3024                    .forwarding
3025                    .live_roots(&registration.manifest.module_id)
3026                    .map_err(RouterError::Forwarding)?;
3027                Ok(vec![control_response_body_frame(
3028                    &frame,
3029                    &response,
3030                    "ModuleControlResponseToModule::LiveRoots",
3031                )?])
3032            }
3033            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3034                self.handle_scope_sync(connection_id, frame, generation, scopes)
3035            }
3036            ModuleControlRequestFromModule::ScopeApply {
3037                generation,
3038                upsert,
3039                end,
3040            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3041            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3042                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3043            }
3044        }
3045    }
3046
3047    fn handle_scope_served_by(
3048        &self,
3049        connection_id: ConnectionId,
3050        frame: Frame,
3051    ) -> Result<Vec<Frame>, RouterError> {
3052        let registered = self
3053            .registry
3054            .get_module_by_connection(connection_id)
3055            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3056        let Some(registered) = registered else {
3057            return Ok(vec![control_error_frame(
3058                &frame,
3059                "not_registered",
3060                "scope.served_by requires an active module registration owned by this connection",
3061            )?]);
3062        };
3063        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3064        {
3065            Ok(request) => request,
3066            Err(err) => {
3067                return Ok(vec![control_error_frame(
3068                    &frame,
3069                    "invalid_control_body",
3070                    format!("malformed scope.served_by body: {err}"),
3071                )?])
3072            }
3073        };
3074        let answer = self
3075            .scopes
3076            .read()
3077            .unwrap_or_else(|poisoned| poisoned.into_inner())
3078            .served_by(
3079                &registered.manifest.module_id,
3080                &request.scope_ref,
3081                request.scope_epoch,
3082                &request.module_id,
3083            );
3084        let reply = subc_control::ScopeServedByReply::new(
3085            answer,
3086            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3087        );
3088        Ok(vec![control_response_body_frame(
3089            &frame,
3090            &reply,
3091            "ScopeServedByReply",
3092        )?])
3093    }
3094
3095    fn handle_operator_confirm(
3096        &self,
3097        ctx: &RouteCtx,
3098        frame: Frame,
3099    ) -> Result<Vec<Frame>, RouterError> {
3100        use crate::operator_confirm::{audit, Outcome};
3101        let registration = self
3102            .registry
3103            .get_module_by_connection(ctx.connection_id)
3104            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3105        let Some(registration) = registration else {
3106            let outcome = Outcome::refusal("not_registered");
3107            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3108            return Ok(vec![outcome.frame(&frame)]);
3109        };
3110        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3111            Ok(request) => request,
3112            Err(_) => {
3113                let outcome = Outcome::refusal("invalid_control_body");
3114                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3115                return Ok(vec![outcome.frame(&frame)]);
3116            }
3117        };
3118        let module_id = registration.manifest.module_id;
3119        // Read the launch nonce (under its own lock) before taking the forwarding
3120        // table's lock below: holding forwarding while waiting on another daemon
3121        // lock risks a lock-order deadlock with paths that take them the other way.
3122        let nonce = self
3123            .hello_launch_nonces
3124            .lock()
3125            .unwrap_or_else(|p| p.into_inner())
3126            .nonce(ctx.connection_id)
3127            .map(str::to_owned);
3128        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3129            self.supervisor
3130                .spawned_consumer_authorized(&module_id, nonce)
3131        });
3132        let confirms = self.forwarding.operator_confirms();
3133        self.forwarding
3134            .with_operator_route(
3135                ctx.connection_id,
3136                request.route_channel,
3137                request.route_epoch,
3138                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3139            )
3140            .map_err(RouterError::Forwarding)
3141    }
3142
3143    /// `scope.sync`: the owner is the module registered on this connection.
3144    /// A connection with no registration (every client connection, `direct`
3145    /// included) is refused `not_registered` before the table is consulted.
3146    fn handle_scope_sync(
3147        &self,
3148        connection_id: ConnectionId,
3149        frame: Frame,
3150        generation: u64,
3151        scopes: Vec<ScopeRecord>,
3152    ) -> Result<Vec<Frame>, RouterError> {
3153        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3154    }
3155
3156    fn handle_scope_change(
3157        &self,
3158        connection_id: ConnectionId,
3159        frame: Frame,
3160        generation: u64,
3161        scopes: Vec<ScopeRecord>,
3162        end: Option<Vec<ScopeEnd>>,
3163    ) -> Result<Vec<Frame>, RouterError> {
3164        let is_apply = end.is_some();
3165        let Some(registration) = self
3166            .registry
3167            .get_module_by_connection(connection_id)
3168            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3169        else {
3170            return Ok(vec![control_error_frame(
3171                &frame,
3172                "not_registered",
3173                "scope.sync and scope.apply require an active module registration owned by this connection",
3174            )?]);
3175        };
3176        let owner = registration.manifest.module_id;
3177        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3178        let is_current_launch = |connection: ConnectionId| {
3179            self.hello_launch_nonces
3180                .lock()
3181                .unwrap_or_else(|poisoned| poisoned.into_inner())
3182                .presented(connection, current_nonce.as_deref())
3183        };
3184        // Lock order is the scope table, then the forwarding table: the new
3185        // tags are published, and the routes the change closes are selected,
3186        // while the scope table is still write-locked, so no admission can read
3187        // a record whose tag is not yet published.
3188        let mut table = self
3189            .scopes
3190            .write()
3191            .unwrap_or_else(|poisoned| poisoned.into_inner());
3192        let outcome = match end {
3193            Some(end) => table.apply(
3194                &owner,
3195                connection_id,
3196                is_current_launch,
3197                generation,
3198                scopes,
3199                end,
3200            ),
3201            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3202        };
3203        let drained = match &outcome {
3204            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3205            Err(_) => Vec::new(),
3206        };
3207        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3208        drop(table);
3209        if has_deadlines {
3210            self.scope_deadline_added.notify_one();
3211        }
3212        match outcome {
3213            Ok(applied) => {
3214                let counts = ScopeOutcomeCounts::of(&applied.results);
3215                info!(
3216                    owner = %owner,
3217                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3218                    generation,
3219                    records = applied.results.len(),
3220                    created = counts.created,
3221                    replaced = counts.replaced,
3222                    updated = counts.updated,
3223                    unchanged = counts.unchanged,
3224                    refused = counts.refused,
3225                    ended = applied.ended.len(),
3226                    tag_changes = applied.tag_changes.len(),
3227                    routes_closed = drained.len(),
3228                    "scope change accepted"
3229                );
3230                // An accepted scope change can still refuse individual records, and the
3231                // owner is the only party that sees the reply. Name them here so
3232                // an operator can tell a refused session from a missing one
3233                // without the owner's logs. Capped so a call that refuses
3234                // thousands cannot flood the log; the count above is complete.
3235                for refused in applied
3236                    .results
3237                    .iter()
3238                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3239                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3240                {
3241                    warn!(
3242                        owner = %owner,
3243                        generation,
3244                        scope_ref = %refused.scope_ref,
3245                        scope_epoch = refused.scope_epoch,
3246                        code = refused.code.as_deref().unwrap_or(""),
3247                        "scope record refused"
3248                    );
3249                }
3250                self.close_scope_drained_routes(drained);
3251                let response = if is_apply {
3252                    ModuleControlResponseToModule::ScopeApply {
3253                        generation,
3254                        results: applied.results,
3255                        end_results: applied.end_results,
3256                        ended: applied.ended,
3257                    }
3258                } else {
3259                    ModuleControlResponseToModule::ScopeSync {
3260                        generation,
3261                        results: applied.results,
3262                        ended: applied.ended,
3263                    }
3264                };
3265                Ok(vec![control_response_body_frame(
3266                    &frame,
3267                    &response,
3268                    "scope change response",
3269                )?])
3270            }
3271            Err(refusal) => {
3272                info!(
3273                    owner = %owner,
3274                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3275                    generation,
3276                    code = refusal.code,
3277                    "scope change refused"
3278                );
3279                Ok(vec![control_error_frame(
3280                    &frame,
3281                    refusal.code,
3282                    refusal.message,
3283                )?])
3284            }
3285        }
3286    }
3287
3288    /// Tell both ends of each route a scope change closed. The module gets a
3289    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3290    /// the client's route handle. The client also gets `route.closed` with the
3291    /// scope reason, one push per module and reason, so it can tell a revoked
3292    /// route from an ordinary close and not reopen it.
3293    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3294        if drained.is_empty() {
3295            return;
3296        }
3297        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3298            BTreeMap::new();
3299        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3300        for route in drained {
3301            warn!(
3302                module_id = %route.module_id,
3303                reason = ?route.reason,
3304                client_connection_id = route.client.connection_id.get(),
3305                route_channel = route.client.channel,
3306                "closing route because its scope changed"
3307            );
3308            pushes
3309                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3310                .or_insert_with(|| (route.reason, Vec::new()))
3311                .1
3312                .push(EndpointRoute {
3313                    goodbye_target: route.client.clone(),
3314                    principal: Principal::Unverified,
3315                    bound_at: Instant::now(),
3316                    draining: false,
3317                    drain_reason: None,
3318                });
3319            goodbyes.push(route.module);
3320            goodbyes.push(route.client);
3321        }
3322        for ((module_id, _), (reason, routes)) in pushes {
3323            send_route_control_pushes(
3324                &self.forwarding,
3325                routes,
3326                ClientControlPush::RouteClosed {
3327                    module_id,
3328                    channels: Vec::new(),
3329                    reason,
3330                    drained: false,
3331                    abandoned: 0,
3332                    excluded_subscriptions: 0,
3333                    terminal: Some(false),
3334                },
3335            );
3336        }
3337        self.emit_route_goodbyes(goodbyes);
3338    }
3339
3340    /// `scope.describe`: any registered module may read any scope, because a
3341    /// provider must read the scope a route it serves is stamped with.
3342    fn handle_scope_describe(
3343        &self,
3344        connection_id: ConnectionId,
3345        frame: Frame,
3346        owner: Principal,
3347        scope_ref: String,
3348    ) -> Result<Vec<Frame>, RouterError> {
3349        let registered = self
3350            .registry
3351            .get_module_by_connection(connection_id)
3352            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3353        let Some(registered) = registered else {
3354            return Ok(vec![control_error_frame(
3355                &frame,
3356                "not_registered",
3357                "scope.describe requires an active module registration owned by this connection",
3358            )?]);
3359        };
3360        let mut description = self
3361            .scopes
3362            .read()
3363            .unwrap_or_else(|poisoned| poisoned.into_inner())
3364            .describe(&owner, &scope_ref);
3365        // Describe readers also decode strict ScopeAttributes. Attribution is
3366        // safe to omit unless the requesting module declares it understands it.
3367        if !registered
3368            .manifest
3369            .capabilities
3370            .as_ref()
3371            .is_some_and(|caps| {
3372                caps.provides
3373                    .iter()
3374                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3375            })
3376        {
3377            if let Some(stamp) = description.stamp.as_mut() {
3378                stamp.attributes.on_behalf_of_agent = None;
3379            }
3380        }
3381        let owner_configured = match &owner {
3382            // Ask whether the owner is configured (`is_configured`), not
3383            // whether it is on the roster (`get(..).is_some()`): a supervised
3384            // module's process can register and describe a scope before the
3385            // supervisor has put it on the roster.
3386            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3387            _ => false,
3388        };
3389        let response = ModuleControlResponseToModule::ScopeDescribe {
3390            status: description.status,
3391            scope_epoch: description.scope_epoch,
3392            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3393            owner_synced: description.owner_synced,
3394            owner_configured,
3395            scope: description.stamp,
3396        };
3397        Ok(vec![control_response_body_frame(
3398            &frame,
3399            &response,
3400            "ModuleControlResponseToModule::ScopeDescribe",
3401        )?])
3402    }
3403
3404    fn handle_catalog_update(
3405        &self,
3406        connection_id: ConnectionId,
3407        frame: Frame,
3408        provides: Vec<ProviderRole>,
3409        capabilities: Option<CapabilityDeclarations>,
3410        ready: Option<bool>,
3411    ) -> Result<Vec<Frame>, RouterError> {
3412        self.refresh_capability_requirements();
3413        let Some(registration) = self
3414            .registry
3415            .get_module_by_connection(connection_id)
3416            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3417        else {
3418            return Ok(vec![control_error_frame(
3419                &frame,
3420                "not_registered",
3421                "catalog.update requires an active module registration owned by this connection",
3422            )?]);
3423        };
3424
3425        if let Some(message) =
3426            catalog_update_frozen_field_message(&registration.manifest, &provides)
3427        {
3428            return Ok(vec![control_error_frame(
3429                &frame,
3430                "catalog_update_frozen_field",
3431                message,
3432            )?]);
3433        }
3434
3435        let mut candidate = registration.manifest.clone();
3436        candidate.provides = provides.clone();
3437        candidate.capabilities = capabilities
3438            .clone()
3439            .or_else(|| registration.manifest.capabilities.clone());
3440        if let Err(err) = candidate.validate_capability_grammar() {
3441            return Ok(vec![control_error_frame(
3442                &frame,
3443                "invalid_capability_grammar",
3444                err.to_string(),
3445            )?]);
3446        }
3447
3448        // Updates must honor the same reserved owner as initial registration;
3449        // otherwise an empty HELLO could acquire the claim after admission.
3450        let mut conflicts = self
3451            .capability_evaluator
3452            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3453        if let Some(conflict) = conflicts.first() {
3454            let message = format!(
3455                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3456                conflict.capability, conflict.claimants[0], candidate.module_id
3457            );
3458            for conflict in &mut conflicts {
3459                conflict.source = DuplicateClaimSource::CatalogUpdate;
3460            }
3461            log_duplicate_claim_events(conflicts);
3462            return Ok(vec![control_error_frame(
3463                &frame,
3464                "reserved_capability",
3465                message,
3466            )?]);
3467        }
3468
3469        let updated = self
3470            .registry
3471            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3472            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3473        if updated.is_none() {
3474            return Ok(vec![control_error_frame(
3475                &frame,
3476                "not_registered",
3477                "catalog.update requires an active module registration owned by this connection",
3478            )?]);
3479        }
3480        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3481            log_duplicate_claim_events(
3482                self.capability_evaluator
3483                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3484            );
3485        }
3486        if capability_census_trigger(
3487            registration.manifest.capabilities.as_ref(),
3488            updated
3489                .as_ref()
3490                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3491        ) {
3492            self.enforce_capability_denies();
3493        }
3494        self.refresh_capability_requirements();
3495
3496        let response = ModuleControlResponseToModule::CatalogUpdate {};
3497        control_response_body_frame(
3498            &frame,
3499            &response,
3500            "ModuleControlResponseToModule::CatalogUpdate",
3501        )
3502        .map(|frame| vec![frame])
3503    }
3504
3505    pub(crate) fn with_activity_ledger_config(
3506        mut self,
3507        run_dir: &Path,
3508        incarnation: String,
3509        config: crate::activity::ActivityConfig,
3510    ) -> Self {
3511        self.activity = Some(crate::activity::ActivityLedger::open_config(
3512            run_dir.join("activity.jsonl"),
3513            incarnation,
3514            config,
3515        ));
3516        self.supervisor.set_activity(self.activity.clone());
3517        self
3518    }
3519
3520    pub(crate) async fn configure_activity(
3521        &self,
3522        config: crate::activity::ActivityConfig,
3523    ) -> Result<(), &'static str> {
3524        match &self.activity {
3525            Some(ledger) => ledger.configure(config).await,
3526            None => Ok(()),
3527        }
3528    }
3529
3530    fn cancel_activity_subscription(
3531        &self,
3532        ctx: &RouteCtx,
3533        frame: &Frame,
3534    ) -> Result<bool, RouterError> {
3535        let task = self
3536            .activity_streams
3537            .lock()
3538            .unwrap_or_else(|p| p.into_inner())
3539            .remove(&(ctx.connection_id, frame.header.corr));
3540        let Some(task) = task else {
3541            return Ok(false);
3542        };
3543        task.abort();
3544        let end = Frame::build_with_version(
3545            response_version(frame),
3546            FrameType::StreamEnd,
3547            control_flags(),
3548            0,
3549            0,
3550            frame.header.corr,
3551            Vec::new(),
3552        )
3553        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3554        let sink = ctx.egress.clone();
3555        tokio::spawn(async move {
3556            let _ = sink.send(end).await;
3557        });
3558        Ok(true)
3559    }
3560
3561    async fn handle_activity_read(
3562        &self,
3563        ctx: &RouteCtx,
3564        frame: Frame,
3565    ) -> Result<Vec<Frame>, RouterError> {
3566        use subc_control::{
3567            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3568            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3569        };
3570        #[derive(Deserialize)]
3571        struct Identity {
3572            #[serde(default)]
3573            consumer_identity: Option<ConsumerIdentity>,
3574        }
3575        // A registered module connection already identifies its caller. Ignore
3576        // consumer_identity there so a module cannot borrow a mirror's identity.
3577        // Only separate consumer connections use the Direct/nonce trust boundary.
3578        let principal = match self
3579            .forwarding
3580            .module_id_for_connection(ctx.connection_id)
3581            .map_err(RouterError::Forwarding)?
3582        {
3583            Some(module_id) => Ok(Principal::Reserved { module_id }),
3584            None => {
3585                let identity: Identity = match serde_json::from_slice(&frame.body) {
3586                    Ok(identity) => identity,
3587                    Err(_) => {
3588                        return Ok(vec![control_error_frame(
3589                            &frame,
3590                            ACTIVITY_MALFORMED,
3591                            "malformed activity read",
3592                        )?])
3593                    }
3594                };
3595                self.route_open_principal(&frame, identity.consumer_identity)?
3596            }
3597        };
3598        let admitted = matches!(principal, Ok(Principal::Direct))
3599            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3600        if !admitted {
3601            return Ok(vec![control_error_frame(
3602                &frame,
3603                ACTIVITY_READ_FORBIDDEN,
3604                "activity reads require an operator or trusted mirror principal",
3605            )?]);
3606        }
3607        let Some(ledger) = &self.activity else {
3608            return Ok(vec![control_error_frame(
3609                &frame,
3610                ACTIVITY_UNAVAILABLE,
3611                "activity ledger unavailable",
3612            )?]);
3613        };
3614        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3615            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3616        {
3617            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3618                Ok(request) => ledger.query(request).await,
3619                Err(_) => Err(ACTIVITY_MALFORMED),
3620            };
3621            return match result {
3622                Ok(reply) => Ok(vec![control_response_body_frame(
3623                    &frame,
3624                    &reply,
3625                    ops::ACTIVITY_QUERY,
3626                )?]),
3627                Err(code) => Ok(vec![control_error_frame(
3628                    &frame,
3629                    code,
3630                    "activity query refused",
3631                )?]),
3632            };
3633        }
3634        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3635            Ok(request) => request,
3636            Err(_) => {
3637                return Ok(vec![control_error_frame(
3638                    &frame,
3639                    ACTIVITY_MALFORMED,
3640                    "malformed activity subscription",
3641                )?])
3642            }
3643        };
3644        let mut subscription = match ledger.subscribe(request.since).await {
3645            Ok(subscription) => subscription,
3646            Err(code) => {
3647                return Ok(vec![control_error_frame(
3648                    &frame,
3649                    code,
3650                    "activity subscription refused",
3651                )?])
3652            }
3653        };
3654        let key = (ctx.connection_id, frame.header.corr);
3655        let sink = ctx.egress.clone();
3656        let mut streams = self
3657            .activity_streams
3658            .lock()
3659            .unwrap_or_else(|p| p.into_inner());
3660        streams.retain(|_, task| !task.is_finished());
3661        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3662            return Ok(vec![control_error_frame(
3663                &frame,
3664                "duplicate_subscription",
3665                "activity subscription correlation id already in use",
3666            )?]);
3667        }
3668        let task = tokio::spawn(async move {
3669            loop {
3670                let entry = if let Some(entry) = subscription.replay.pop_front() {
3671                    Some(entry)
3672                } else {
3673                    subscription.receiver.recv().await
3674                };
3675                let Some(entry) = entry else {
3676                    break;
3677                };
3678                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3679                    return;
3680                };
3681                let Ok(next) = Frame::build_with_version(
3682                    response_version(&frame),
3683                    FrameType::StreamData,
3684                    control_flags(),
3685                    0,
3686                    0,
3687                    frame.header.corr,
3688                    body,
3689                ) else {
3690                    return;
3691                };
3692                if sink.send(next).await.is_err() {
3693                    return;
3694                }
3695            }
3696            // Dropping a full subscriber's sender drains its accepted entries
3697            // before the terminal error, preserving their order on the wire.
3698            if let Some(code) = subscription.terminal_code() {
3699                if let Ok(end) =
3700                    control_error_frame(&frame, code, "activity subscriber fell behind")
3701                {
3702                    let _ = sink.send(end).await;
3703                }
3704            }
3705        });
3706        streams.insert(key, task.abort_handle());
3707        Ok(Vec::new())
3708    }
3709
3710    async fn handle_activity_record(
3711        &self,
3712        ctx: &RouteCtx,
3713        frame: Frame,
3714    ) -> Result<Vec<Frame>, RouterError> {
3715        use subc_control::{
3716            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3717            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3718        };
3719        let result = async {
3720            let module = self
3721                .forwarding
3722                .module_id_for_connection(ctx.connection_id)
3723                .ok()
3724                .flatten()
3725                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3726            let request: ActivityRecordRequest =
3727                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3728            crate::activity::validate(&module, &request)?;
3729            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3730            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3731            let at_ms = std::time::SystemTime::now()
3732                .duration_since(std::time::UNIX_EPOCH)
3733                .unwrap_or_default()
3734                .as_millis() as u64;
3735            ledger.record(module, request, actor, at_ms).await
3736        }
3737        .await;
3738        match result {
3739            Ok(entry) => Ok(vec![control_response_body_frame(
3740                &frame,
3741                &ActivityRecordResponse { seq: entry.seq },
3742                "activity.record",
3743            )?]),
3744            Err(code) => Ok(vec![control_error_frame(
3745                &frame,
3746                code,
3747                "activity recording refused",
3748            )?]),
3749        }
3750    }
3751
3752    fn activity_actor(
3753        &self,
3754        connection: ConnectionId,
3755        module: &str,
3756        on: Option<&subc_control::ActivityOn>,
3757    ) -> Result<subc_control::ActivityActor, &'static str> {
3758        use crate::forwarding::{DataRoute, DataRouteState};
3759        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3760        let principal = |id: &str| Principal::Reserved {
3761            module_id: id.to_owned(),
3762        };
3763        let mut actor = ActivityActor {
3764            principal: principal(module),
3765            scope_owner: None,
3766            agent_id: None,
3767            run_id: None,
3768            flow_id: None,
3769            project_root: None,
3770        };
3771        let scope = match on {
3772            None => return Ok(actor),
3773            Some(ActivityOn::Scope(scope)) => {
3774                let selector = subc_protocol::scope::ScopeSelector::new(
3775                    principal(module),
3776                    scope.scope_ref.clone(),
3777                    Some(scope.epoch),
3778                );
3779                self.scopes
3780                    .read()
3781                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3782                    .attribute(&principal(module), module, &selector, true)
3783                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3784            }
3785            Some(ActivityOn::Route(route)) => {
3786                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3787                    .forwarding
3788                    .lookup_data_route(connection, route.channel, route.epoch)
3789                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3790                else {
3791                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3792                };
3793                if binding.module_id != module {
3794                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3795                }
3796                actor.principal = binding.principal.clone();
3797                actor.project_root = binding
3798                    .project_root
3799                    .as_ref()
3800                    .map(|root| root.as_path().to_path_buf());
3801                let Some(bound) = &binding.scope else {
3802                    return Ok(actor);
3803                };
3804                let owner = principal(&bound.owner);
3805                let selector = subc_protocol::scope::ScopeSelector::new(
3806                    owner.clone(),
3807                    bound.scope_ref.clone(),
3808                    Some(bound.tag.scope_epoch),
3809                );
3810                let admitted = self
3811                    .scopes
3812                    .read()
3813                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3814                    .attribute(&owner, module, &selector, true)
3815                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3816                if admitted.tag != bound.tag {
3817                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3818                }
3819                admitted
3820            }
3821        };
3822        actor.scope_owner = Some(scope.stamp.owner);
3823        actor.agent_id = scope.stamp.attributes.agent_id;
3824        actor.run_id = scope.stamp.attributes.run_id;
3825        actor.flow_id = scope.stamp.attributes.flow_id;
3826        Ok(actor)
3827    }
3828
3829    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3830        self.refresh_capability_requirements();
3831        // A bare connection count is ambiguous between many clients holding a
3832        // route each and one client accumulating hundreds, so publish the
3833        // concentration alongside it. Route state is best-effort here: a
3834        // diagnostic endpoint must still answer if the forwarding lock is
3835        // contended.
3836        let mut counters = self.counters.snapshot();
3837        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3838            obj.insert(
3839                "activity_unparseable_lines".into(),
3840                ledger.read_errors().into(),
3841            );
3842            obj.insert(
3843                "activity_lifecycle_append_failures".into(),
3844                ledger.lifecycle_failures().into(),
3845            );
3846            obj.insert(
3847                "activity_started_digest_skips".into(),
3848                ledger.lifecycle_digest_skips().into(),
3849            );
3850        }
3851        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3852            self.forwarding.client_route_concentration(),
3853            counters.as_object_mut(),
3854        ) {
3855            obj.insert(
3856                "client_connections_with_routes".into(),
3857                connections_with_routes.into(),
3858            );
3859            obj.insert("max_routes_on_one_connection".into(), max.into());
3860        }
3861        // A module that is being fast-refused and a module that is fine look
3862        // identical from a client that retries and succeeds, so name the open
3863        // breakers here. This rides the existing free-form counters object
3864        // rather than a new wire field, so no sibling that deserializes
3865        // `ServerDescribe` has to be rebuilt to keep reading it.
3866        if let (Some(open_breakers), Some(obj)) = (
3867            self.route_bind_breakers.open_snapshot(),
3868            counters.as_object_mut(),
3869        ) {
3870            obj.insert("route_bind_breakers_open".into(), open_breakers);
3871        }
3872        let response = ClientControlResponse::ServerDescribe {
3873            protocol_ver: PROTOCOL_VERSION,
3874            subc_ops: subc_ops(),
3875            capabilities: self.describe_capabilities(),
3876            connected_clients: self.connected_clients.count(),
3877            counters: Some(counters),
3878            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3879            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3880            capability_requirements: self.capability_requirement_statuses(),
3881            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3882            config_path: self
3883                .rescan
3884                .as_ref()
3885                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3886        };
3887        Ok(vec![control_response_body_frame(
3888            &frame,
3889            &response,
3890            "ClientControlResponse::ServerDescribe",
3891        )?])
3892    }
3893
3894    fn handle_catalog_list(
3895        &self,
3896        frame: Frame,
3897        module_id: Option<String>,
3898    ) -> Result<Vec<Frame>, RouterError> {
3899        let (generation, modules) = self.registry.list_modules().map_err(|err| {
3900            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3901        })?;
3902        let entries = modules
3903            .into_iter()
3904            .filter(|registration| {
3905                module_id
3906                    .as_deref()
3907                    .map(|wanted| registration.manifest.module_id == wanted)
3908                    .unwrap_or(true)
3909            })
3910            .map(|registration| {
3911                let not_ready = self.not_ready_reason(&registration);
3912                let roles = registration.manifest.provides;
3913                CatalogEntry::new(
3914                    registration.manifest.module_id,
3915                    roles,
3916                    registration.control_ops,
3917                )
3918                .with_ready(not_ready.is_none())
3919                .with_not_ready(not_ready)
3920                .with_module_version(Some(registration.manifest.module_version))
3921                .with_capabilities(registration.manifest.capabilities)
3922                .with_self_signals(registration.manifest.self_signals)
3923                .with_events(registration.manifest.events)
3924                .with_operation_delivery(registration.manifest.operation_delivery)
3925            })
3926            .collect();
3927        let response = ClientControlResponse::CatalogList {
3928            generation,
3929            modules: entries,
3930            subc_ops: subc_ops(),
3931        };
3932        Ok(vec![control_response_body_frame(
3933            &frame,
3934            &response,
3935            "ClientControlResponse::CatalogList",
3936        )?])
3937    }
3938
3939    fn route_open_principal(
3940        &self,
3941        frame: &Frame,
3942        consumer_identity: Option<ConsumerIdentity>,
3943    ) -> Result<Result<Principal, Frame>, RouterError> {
3944        let Some(consumer_identity) = consumer_identity else {
3945            return Ok(Ok(Principal::Direct));
3946        };
3947
3948        if self.supervisor.spawned_consumer_authorized(
3949            &consumer_identity.module_id,
3950            &consumer_identity.launch_nonce,
3951        ) {
3952            return Ok(Ok(Principal::Reserved {
3953                module_id: consumer_identity.module_id,
3954            }));
3955        }
3956
3957        Ok(Err(control_error_frame(
3958            frame,
3959            "bad_consumer_identity",
3960            format!(
3961                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3962                consumer_identity.module_id
3963            ),
3964        )?))
3965    }
3966
3967    /// Ordinary `route.open` refusals go through here; admission and breaker
3968    /// refusals log separately with their capacity or breaker state. The daemon can
3969    /// attest which code it sent: without the event, a client's "the daemon
3970    /// refused me" and the daemon's own view could only be reconciled by
3971    /// argument. Malformed input (`invalid_project_root`) does not come here;
3972    /// rejecting a request that was never a valid open is not a refusal of one.
3973    fn route_open_refusal_frame(
3974        &self,
3975        ctx: &RouteCtx,
3976        frame: &Frame,
3977        module_id: &str,
3978        reason: &'static str,
3979        code: &'static str,
3980        message: impl Into<String>,
3981    ) -> Result<Frame, RouterError> {
3982        self.observe_route_open_refusal(ctx, module_id, reason, code);
3983        control_error_frame(frame, code, message.into())
3984    }
3985
3986    /// Refuse a `route.open` because the target module's bind-relay breaker is
3987    /// open, without attempting the relay.
3988    ///
3989    /// The wire code is `module_timeout`, which is the truth (the module has
3990    /// not been answering binds) and which both SDKs already classify as
3991    /// retryable with capped backoff. Reusing it is what keeps this change out
3992    /// of both SDKs; the daemon-side distinction lives in the counter key
3993    /// instead.
3994    ///
3995    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
3996    /// While a breaker is open this fires on every open to that module, and the
3997    /// stall written up in `docs/designs/route-open-head-of-line.md` already
3998    /// produced 261 lines about a single module inside 3000 lines of daemon
3999    /// log. The rare transitions are logged at warn/info instead and the volume
4000    /// is carried by the counter, so the evidence survives without the flood.
4001    /// The debug line keeps a per-refusal record reachable for whoever turns
4002    /// the level up.
4003    fn route_open_breaker_refusal_frame(
4004        &self,
4005        ctx: &RouteCtx,
4006        frame: &Frame,
4007        module_id: &str,
4008        consecutive_timeouts: u32,
4009        retry_in: Duration,
4010        probe_in_flight: bool,
4011    ) -> Result<Frame, RouterError> {
4012        self.counters
4013            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4014        debug!(
4015            target: "control",
4016            code = "module_timeout",
4017            module_id = ?module_id,
4018            connection_id = ctx.connection_id.get(),
4019            consecutive_timeouts,
4020            retry_in_ms = retry_in.as_millis() as u64,
4021            probe_in_flight,
4022            "route.open refused by open bind-relay breaker"
4023        );
4024        // Say what a caller can act on. An open bind-relay breaker means the
4025        // module timed out accepting several new routes in a row. The module
4026        // is still running and its established routes keep working; only new
4027        // route.open requests are refused until the cooldown ends and one
4028        // test route (the probe) gets through. A message that only counts
4029        // failed relays reads as "the module is down" to a worker that sees it.
4030        let detail = if probe_in_flight {
4031            "one test route is already being tried; retry once it settles".to_string()
4032        } else {
4033            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4034        };
4035        control_error_frame(
4036            frame,
4037            "module_timeout",
4038            format!(
4039                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4040                 timed out in a row); {detail}; its established routes are unaffected"
4041            ),
4042        )
4043    }
4044
4045    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4046    /// requester's bytes (an unknown target is whatever the client sent) and
4047    /// is Debug-formatted so control characters land in the log escaped
4048    /// rather than as terminal sequences for whoever tails it.
4049    ///
4050    /// `reason` names the check that refused, because one wire code has
4051    /// several senders: after a module registers, `target_unavailable` can
4052    /// come from a missing role, an inactive registration, a supervisor that
4053    /// has not marked the process live, a missing forwarding connection, or a
4054    /// failed relay, and a log that records only the code cannot say which of
4055    /// them fired. It is a static, daemon-chosen label per branch, so it is
4056    /// safe to print plainly and stays a closed set.
4057    fn observe_route_open_refusal(
4058        &self,
4059        ctx: &RouteCtx,
4060        module_id: &str,
4061        reason: &'static str,
4062        code: &'static str,
4063    ) {
4064        self.counters.increment_route_open_refused(code);
4065        info!(
4066            target: "control",
4067            code,
4068            reason,
4069            module_id = ?module_id,
4070            connection_id = ctx.connection_id.get(),
4071            "route.open refused"
4072        );
4073        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4074            self.route_outages.record_not_serving(module_id, reason);
4075        }
4076    }
4077
4078    /// Record an ACCEPTED route.open.
4079    ///
4080    /// Refusals have been logged and counted since the attestation work; accepts
4081    /// were invisible, so the daemon knew every principal it stamped and wrote
4082    /// none of them down. The party that attests the identity was the only party
4083    /// not recording it, which left a credential vault unable to name the sender
4084    /// of a call that reached it (claustrum #43) and left the launch-nonce
4085    /// concurrency question unanswerable from the outside.
4086    ///
4087    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4088    /// `code`/`module_id`/`connection_id` returns both directions of the same
4089    /// decision rather than two shapes a reader has to join by hand.
4090    ///
4091    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4092    /// and the difference carries information rather than being an
4093    /// inconsistency. This line is only reachable after a successful bind to a
4094    /// REGISTERED module, so the value has already passed HELLO validation
4095    /// including the path-hazard refusal and cannot contain control bytes. A
4096    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4097    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4098    ///
4099    /// Bare is also what every other daemon line already emits (`module
4100    /// registered`, `configured module supervised`). Shipping `?module_id` here
4101    /// made this instrument the only one in the file whose ids did not answer
4102    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4103    /// line whose whole purpose is being grepped beside its sibling.
4104    ///
4105    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4106    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4107    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4108    /// one are recorded identically. A test written against that harness passes
4109    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4110    /// green assertion that cannot fail. The same limit applies to the escaping
4111    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4112    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4113    /// Fencing either needs the real formatter, not the capture layer.
4114    ///
4115    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4116    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4117    /// to record. The ephemeral port would decay within minutes and answer only a
4118    /// live question. The identity question is instead answered by counting
4119    /// distinct live connections presenting one module's `consumer_identity` --
4120    /// "is anyone else holding this secret" rather than "is this the right
4121    /// process".
4122    fn observe_route_open_accept(
4123        &self,
4124        ctx: &RouteCtx,
4125        module_id: &str,
4126        principal: &str,
4127        scope: &str,
4128    ) {
4129        self.route_outages.record_accepted(module_id);
4130        self.counters.increment_route_open_accepted(principal);
4131        info!(
4132            target: "control",
4133            principal,
4134            module_id,
4135            connection_id = ctx.connection_id.get(),
4136            scope = %scope,
4137            "route.open accepted"
4138        );
4139    }
4140
4141    fn supervised_absent_route_open_refusal_frame(
4142        &self,
4143        ctx: &RouteCtx,
4144        frame: &Frame,
4145        module_id: &str,
4146        code: &'static str,
4147        status: &crate::supervise::ModuleStatus,
4148    ) -> Result<Frame, RouterError> {
4149        self.counters.increment_route_open_refused(code);
4150        info!(
4151            target: "control",
4152            code,
4153            reason = "supervised_not_registered",
4154            module_id = ?module_id,
4155            connection_id = ctx.connection_id.get(),
4156            state = %status.state,
4157            enabled = status.enabled,
4158            live = status.live,
4159            "route.open refused"
4160        );
4161        // A supervised module whose process has not registered is not
4162        // serving, whatever the reason; the supervisor knows this id, so it is
4163        // safe to track.
4164        self.route_outages
4165            .record_not_serving(module_id, "supervised_not_registered");
4166        let message = if let Some(pid) = status.waiting_for_previous_pid {
4167            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4168                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4169            })
4170        } else {
4171            format!(
4172                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4173                status.state, status.enabled, status.live
4174            )
4175        };
4176        control_error_frame(frame, code, message)
4177    }
4178
4179    async fn handle_route_open(
4180        &self,
4181        ctx: &RouteCtx,
4182        frame: Frame,
4183        request: RouteOpenRequest,
4184        timings: &mut RouteOpenTimings,
4185    ) -> Result<Vec<Frame>, RouterError> {
4186        let RouteOpenRequest {
4187            target,
4188            mut identity,
4189            consumer_identity,
4190            consumer_capabilities,
4191            role_versions,
4192            admission_facts,
4193            scope,
4194        } = request;
4195        let target_module_id = target_module_id(&target).to_string();
4196        timings.module_id.clone_from(&target_module_id);
4197        if self
4198            .registry
4199            .get_module_by_connection(ctx.connection_id)
4200            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4201            .is_some()
4202        {
4203            return Ok(vec![control_error_frame(
4204                &frame,
4205                "invalid_request",
4206                "module connections cannot open client routes",
4207            )?]);
4208        }
4209        debug!(
4210            connection_id = ctx.connection_id.get(),
4211            corr = frame.header.corr,
4212            module_id = %target_module_id,
4213            "handling route.open"
4214        );
4215
4216        // A malformed declaration is refused first, before anything about the
4217        // target is looked up: the same body would be refused against any
4218        // module, so the caller learns nothing by retrying or waiting. An empty
4219        // map declares nothing and travels as no field at all, so a provider
4220        // only ever sees a missing field or a non-empty one.
4221        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4222        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4223            self.observe_route_open_refusal(
4224                ctx,
4225                &target_module_id,
4226                "invalid_role_versions",
4227                error_codes::INVALID_REQUEST,
4228            );
4229            return Ok(vec![control_error_body_frame(
4230                &frame,
4231                ErrorBody {
4232                    code: error_codes::INVALID_REQUEST.to_string(),
4233                    message: error.to_string(),
4234                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4235                },
4236            )?]);
4237        }
4238
4239        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4240            if status.waiting_for_previous_pid.is_some() {
4241                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4242                    ctx,
4243                    &frame,
4244                    &target_module_id,
4245                    "module_warming",
4246                    &status,
4247                )?]);
4248            }
4249        }
4250
4251        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4252        // opposite. Below, a caller learns whether a module is unregistered,
4253        // supervised-but-down (with state/enabled/live), or registered without the
4254        // requested role. Elsewhere that is an enumeration leak: a probe learning
4255        // the shape of a fleet it cannot otherwise see.
4256        //
4257        // It is not one here, and the reason is the ACCESS MODEL rather than
4258        // anything about these errors. Reaching route.open requires the
4259        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4260        // connection file, so any caller who completes it already runs as this
4261        // user -- and can read subc.jsonc for the module list and `ck module
4262        // status` for live state. The reply discloses nothing the caller cannot
4263        // read more easily from disk, while the precision is load-bearing:
4264        // `unknown_module` is retryable and a missing role is not.
4265        //
4266        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4267        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4268        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4269        let Some(registration) = self
4270            .registry
4271            .get_module(&target_module_id)
4272            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4273        else {
4274            if let Some((status, warming)) =
4275                self.supervisor_status(&target_module_id, frame.header.corr)?
4276            {
4277                // BEFORE the two availability codes below, because for a module
4278                // that speaks no subc wire both of them are false comfort: they
4279                // say "not right now" and are retried, and this module will
4280                // never register no matter how long the caller waits. The
4281                // absence here is the declaration being honoured, not a module
4282                // that is late.
4283                if status.protocol == ModuleProtocol::None {
4284                    return Ok(vec![self.route_open_refusal_frame(
4285                        ctx,
4286                        &frame,
4287                        &target_module_id,
4288                        "protocol_none",
4289                        error_codes::MODULE_NO_PROTOCOL,
4290                        format!(
4291                            "module_id '{target_module_id}' is declared protocol: none; \
4292                             it speaks no subc wire and serves no routes"
4293                        ),
4294                    )?]);
4295                }
4296                let code = if warming {
4297                    "module_warming"
4298                } else {
4299                    "target_unavailable"
4300                };
4301                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4302                    ctx,
4303                    &frame,
4304                    &target_module_id,
4305                    code,
4306                    &status,
4307                )?]);
4308            }
4309            if let Some(removed_ago_ms) =
4310                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4311            {
4312                return Ok(vec![self.route_open_refusal_frame(
4313                    ctx,
4314                    &frame,
4315                    &target_module_id,
4316                    "removed",
4317                    error_codes::MODULE_REMOVED,
4318                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4319                )?]);
4320            }
4321            return Ok(vec![self.route_open_refusal_frame(
4322                ctx,
4323                &frame,
4324                &target_module_id,
4325                "not_registered",
4326                error_codes::UNKNOWN_MODULE,
4327                format!("module_id '{target_module_id}' is not registered"),
4328            )?]);
4329        };
4330
4331        // Best-effort only: registry readiness and forwarding reservation use
4332        // different locks, so a module can flip readiness between this read and
4333        // the relay. Modules must still tolerate an `on_bind` while not ready.
4334        if !registration.ready {
4335            self.counters
4336                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4337            info!(
4338                target: "control",
4339                code = error_codes::MODULE_WARMING,
4340                module_id = ?target_module_id,
4341                connection_id = ctx.connection_id.get(),
4342                reason = "declared_not_ready",
4343                "route.open refused"
4344            );
4345            // The module is registered but says it cannot take work, which is
4346            // an outage from the caller's side even though its process is up.
4347            self.route_outages
4348                .record_not_serving(&target_module_id, "declared_not_ready");
4349            return Ok(vec![control_error_body_frame(
4350                &frame,
4351                ErrorBody {
4352                    code: error_codes::MODULE_WARMING.to_string(),
4353                    message: format!(
4354                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4355                    ),
4356                    detail: Some(serde_json::json!({
4357                        "reason": "declared_not_ready"
4358                    })),
4359                },
4360            )?]);
4361        }
4362
4363        // Effective readiness, second half: a module that declares a capability
4364        // `need: required` is not routable while that capability has no
4365        // registered provider. It is enforced HERE, as a retryable routing
4366        // refusal, and deliberately not as spawn ordering or a boot block. The
4367        // module is still started and registered and can make its own calls;
4368        // spawn ordering is a promise that cannot be kept once a provider
4369        // crashes at runtime, and refusing to boot would stop the whole
4370        // machine, including the tools needed to fix its configuration.
4371        //
4372        // "Provided" is the evaluator's verdict, which counts a provider as
4373        // soon as it has REGISTERED, not once it is ready. Two modules that
4374        // require each other's capabilities are therefore both routable once
4375        // both register; counting readiness instead would deadlock them.
4376        //
4377        // Only new opens are refused. Routes already bound when a provider
4378        // goes away stay bound: nothing here tears them down, and the module
4379        // answers them as it can. Like the readiness read above this is
4380        // best-effort against a provider registering or leaving concurrently.
4381        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4382            self.counters
4383                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4384            info!(
4385                target: "control",
4386                code = error_codes::MODULE_WARMING,
4387                module_id = ?target_module_id,
4388                connection_id = ctx.connection_id.get(),
4389                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4390                capability = %capability,
4391                "route.open refused"
4392            );
4393            return Ok(vec![control_error_body_frame(
4394                &frame,
4395                ErrorBody {
4396                    code: error_codes::MODULE_WARMING.to_string(),
4397                    message: format!(
4398                        "module_id '{target_module_id}' requires capability '{capability}', \
4399                         which no registered module provides; retry"
4400                    ),
4401                    detail: Some(serde_json::json!({
4402                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4403                        "capability": capability,
4404                    })),
4405                },
4406            )?]);
4407        }
4408
4409        if !target_has_required_role(&target, &registration.manifest.provides) {
4410            return Ok(vec![self.route_open_refusal_frame(
4411                ctx,
4412                &frame,
4413                &target_module_id,
4414                "role_not_provided",
4415                "target_unavailable",
4416                format!("module_id '{target_module_id}' does not provide the requested target"),
4417            )?]);
4418        }
4419
4420        if registration.state != ChannelState::Active {
4421            return Ok(vec![self.route_open_refusal_frame(
4422                ctx,
4423                &frame,
4424                &target_module_id,
4425                "registration_not_active",
4426                "target_unavailable",
4427                format!("module_id '{target_module_id}' is not active"),
4428            )?]);
4429        }
4430
4431        if self
4432            .forwarding
4433            .module_is_draining(&target_module_id)
4434            .map_err(RouterError::Forwarding)?
4435        {
4436            return Ok(vec![self.route_open_refusal_frame(
4437                ctx,
4438                &frame,
4439                &target_module_id,
4440                "reloading",
4441                "module_reloading",
4442                format!("module_id '{target_module_id}' is reloading"),
4443            )?]);
4444        }
4445
4446        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4447            process_liveness.process_live(&target_module_id) == Some(false)
4448        }) {
4449            // A module the supervisor is restarting or reloading can still hold
4450            // a registration: the old process before its connection closes, or
4451            // a new one that registered while the supervisor was draining. The
4452            // forwarding table does not see that as draining, but the consumer
4453            // should still be told to retry soon, exactly as for the drain
4454            // above, rather than that the target is unavailable.
4455            if process_liveness.process_replacing(&target_module_id) {
4456                return Ok(vec![self.route_open_refusal_frame(
4457                    ctx,
4458                    &frame,
4459                    &target_module_id,
4460                    "reloading",
4461                    "module_reloading",
4462                    format!("module_id '{target_module_id}' is reloading"),
4463                )?]);
4464            }
4465            return Ok(vec![self.route_open_refusal_frame(
4466                ctx,
4467                &frame,
4468                &target_module_id,
4469                "supervisor_not_live",
4470                "target_unavailable",
4471                format!("module_id '{target_module_id}' is not live"),
4472            )?]);
4473        }
4474
4475        if !self
4476            .forwarding
4477            .has_live_module_connection(&target_module_id)
4478            .map_err(RouterError::Forwarding)?
4479        {
4480            return Ok(vec![self.route_open_refusal_frame(
4481                ctx,
4482                &frame,
4483                &target_module_id,
4484                "no_forwarding_connection",
4485                "target_unavailable",
4486                format!("module_id '{target_module_id}' has no live forwarding connection"),
4487            )?]);
4488        }
4489
4490        if let Some(error) =
4491            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4492        {
4493            self.observe_route_open_refusal(
4494                ctx,
4495                &target_module_id,
4496                "op_not_allowed",
4497                "op_not_allowed",
4498            );
4499            return Ok(vec![error]);
4500        }
4501
4502        let principal = match self.route_open_principal(&frame, consumer_identity)? {
4503            Ok(principal) => principal,
4504            Err(error) => {
4505                self.observe_route_open_refusal(
4506                    ctx,
4507                    &target_module_id,
4508                    "bad_consumer_identity",
4509                    "bad_consumer_identity",
4510                );
4511                return Ok(vec![error]);
4512            }
4513        };
4514        timings.principal = principal_label(&principal);
4515
4516        // This is attested, control-plane policy for supervised module origins.
4517        // Keep it before route reservation and out of the opaque forwarding hot
4518        // path: data frames must never acquire a per-frame capability check.
4519        if let Principal::Reserved {
4520            module_id: opening_module_id,
4521        } = &principal
4522        {
4523            if let Some(opening_registration) = self
4524                .registry
4525                .get_module(opening_module_id)
4526                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4527            {
4528                if let Some(capability) =
4529                    denied_capability(&opening_registration.manifest, &registration.manifest)
4530                {
4531                    warn!(
4532                        opening_module_id,
4533                        target_module_id,
4534                        capability,
4535                        "refusing route.open because an attested capability deny edge matches"
4536                    );
4537                    return Ok(vec![self.route_open_refusal_frame(
4538                        ctx,
4539                        &frame,
4540                        &target_module_id,
4541                        "capability_deny_edge",
4542                        "capability_forbidden",
4543                        format!(
4544                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4545                        ),
4546                    )?]);
4547                }
4548            }
4549        }
4550
4551        if admission_facts.is_some() {
4552            let carrier_matches = matches!(
4553                &principal,
4554                Principal::Reserved { module_id }
4555                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4556            );
4557            if !carrier_matches {
4558                return Ok(vec![self.route_open_refusal_frame(
4559                    ctx,
4560                    &frame,
4561                    &target_module_id,
4562                    "admission_facts_carrier_not_permitted",
4563                    "admission_facts_not_permitted",
4564                    "admission facts may only be carried by the configured reserved module",
4565                )?]);
4566            }
4567
4568            let target_allowed = self
4569                .admission_facts_targets
4570                .as_ref()
4571                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4572            if !target_allowed {
4573                return Ok(vec![self.route_open_refusal_frame(
4574                    ctx,
4575                    &frame,
4576                    &target_module_id,
4577                    "admission_facts_target_not_listed",
4578                    "admission_facts_target_not_allowed",
4579                    format!(
4580                        "admission facts are not permitted for target module_id '{target_module_id}'"
4581                    ),
4582                )?]);
4583            }
4584
4585            // Keep the value opaque to subc. The downstream admission validator owns
4586            // schema and semantic checks; this daemon only enforces carrier authority
4587            // and the configured destination allowlist.
4588        }
4589
4590        // Scope admission, on the attested principal above and never on the
4591        // request body. The tag read here travels with the pending bind and is
4592        // compared with the published one at commit, so a sync between here
4593        // and the module's ack refuses the open instead of binding a stamp
4594        // that is no longer true.
4595        let (bound_scope, mut scope_stamp) = match &scope {
4596            None => (None, None),
4597            Some(selector) => {
4598                let owner_configured = match &selector.owner {
4599                    // A reserved owner counts as configured from before its
4600                    // process is spawned (see `SupervisorHandle::is_configured`).
4601                    // So an owner that has not synced its scopes yet is refused
4602                    // as retryable (`scope_not_synced`), not as one that will
4603                    // never sync.
4604                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4605                    _ => false,
4606                };
4607                let mut scopes = self
4608                    .scopes
4609                    .write()
4610                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4611                let admitted = scopes
4612                    .admit(&principal, &target_module_id, selector, owner_configured)
4613                    .and_then(|admission| {
4614                        crate::scopes::check_target_flow_support(
4615                            &admission.stamp,
4616                            &target_module_id,
4617                            registration.manifest.capabilities.as_ref(),
4618                        )?;
4619                        crate::scopes::check_target_agent_run_support(
4620                            &admission.stamp,
4621                            &target_module_id,
4622                            registration.manifest.capabilities.as_ref(),
4623                        )?;
4624                        let bound = scopes.reserve_bind(
4625                            &admission,
4626                            &principal,
4627                            ctx.connection_id,
4628                            selector.hold_presence,
4629                        );
4630                        Ok((admission, bound))
4631                    });
4632                match admitted {
4633                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4634                    Err(refusal) => {
4635                        return Ok(vec![self.route_open_refusal_frame(
4636                            ctx,
4637                            &frame,
4638                            &target_module_id,
4639                            refusal.code,
4640                            refusal.code,
4641                            refusal.message,
4642                        )?]);
4643                    }
4644                }
4645            }
4646        };
4647
4648        // Bind admits a root that no longer exists on disk, because refusing here
4649        // closes the only exit from a paused run: cancel needs a bound route, and a
4650        // renamed or reclaimed directory makes that route unopenable forever. The
4651        // run itself is intact and still addressable by its recorded identity.
4652        //
4653        // This does NOT relax the rule the strict constructor protects. That rule is
4654        // that no root is ever aliased into NEW durable state -- a missing component
4655        // can reappear as a symlink elsewhere, which would move the identity and
4656        // split a session's history across two of them. The engine now refuses the
4657        // two operations that create such state (send and import) at admission,
4658        // which is a narrower way to hold the same invariant: reads and terminations
4659        // are admitted, writes are not. That refusal had to ship before this line
4660        // changed, or there is an interval where a send commits under a provisional
4661        // identity -- the exact failure the original policy existed to prevent.
4662        //
4663        // Resolution follows realpath rather than lexical cleanup: the longest
4664        // existing ancestor is canonicalized and the missing tail re-appended, so a
4665        // live root is unchanged and a vanished leaf keeps the identity it was
4666        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4667        // same caller the moment the directory vanished, which strands the run more
4668        // quietly than refusing it.
4669        timings.enter(RouteOpenPhase::ProjectRoot);
4670        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4671            Ok(project_root) => project_root,
4672            Err(err) => {
4673                return Ok(vec![control_error_frame(
4674                    &frame,
4675                    "invalid_project_root",
4676                    err.to_string(),
4677                )?])
4678            }
4679        };
4680        timings.enter(RouteOpenPhase::Admission);
4681        identity.project_root = project_root.as_path().to_path_buf();
4682
4683        // Last gate before any relay work, and deliberately after the cheap
4684        // registry and availability checks above: those name a more precise
4685        // condition (unknown, removed, reloading) and a caller is better served
4686        // by the precise code than by this one.
4687        //
4688        // Everything below this point costs an egress permit, a reserved handle
4689        // pair and, if the module does not answer, the whole relay budget. The
4690        // reader no longer waits for that budget, so cap each target explicitly;
4691        // serial dispatch used to provide the accidental cap of one relay per
4692        // connection. Admission is a mutex-protected count and never waits.
4693        let _concurrency_guard = match self
4694            .route_bind_concurrency
4695            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4696        {
4697            Ok(guard) => guard,
4698            Err(in_flight) => {
4699                return Ok(vec![self.route_open_target_capacity_refusal(
4700                    ctx,
4701                    &frame,
4702                    &target_module_id,
4703                    in_flight,
4704                )?]);
4705            }
4706        };
4707
4708        // A module that has already burned the whole budget `threshold` times
4709        // in a row does not get to charge it again until a probe says it recovered.
4710        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4711            RouteBindAdmission::Admitted { guard, probe } => {
4712                if probe {
4713                    info!(
4714                        module_id = %target_module_id,
4715                        connection_id = ctx.connection_id.get(),
4716                        "route.bind breaker half-open: admitting one probe"
4717                    );
4718                }
4719                guard
4720            }
4721            RouteBindAdmission::Refused {
4722                consecutive_timeouts,
4723                retry_in,
4724                probe_in_flight,
4725            } => {
4726                return Ok(vec![self.route_open_breaker_refusal_frame(
4727                    ctx,
4728                    &frame,
4729                    &target_module_id,
4730                    consecutive_timeouts,
4731                    retry_in,
4732                    probe_in_flight,
4733                )?]);
4734            }
4735        };
4736
4737        // Resolve the per-module budget here so the wait matches the operator's
4738        // intent for this specific target. A per-module override in
4739        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4740        // daemons) wins over the daemon-wide default.
4741        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4742        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4743        timings.enter(RouteOpenPhase::EgressReservation);
4744        let pending = match self
4745            .forwarding
4746            .begin_route_bind_relay_for(
4747                ctx.connection_id,
4748                ctx.egress.clone(),
4749                response_version(&frame),
4750                frame.header.corr,
4751                &target_module_id,
4752                principal.clone(),
4753                bound_scope,
4754                Some(project_root),
4755                relay_deadline,
4756            )
4757            .await
4758        {
4759            Ok(pending) => pending,
4760            Err(err) => {
4761                return Ok(vec![self.route_open_refusal_frame(
4762                    ctx,
4763                    &frame,
4764                    &target_module_id,
4765                    "relay_reservation_failed",
4766                    forwarding_error_code(&err),
4767                    err.to_string(),
4768                )?])
4769            }
4770        };
4771        timings.enter(RouteOpenPhase::Admission);
4772        let crate::forwarding::PendingRouteBindRelay {
4773            endpoint,
4774            module_sink,
4775            negotiated_ver,
4776            client_channel,
4777            client_epoch,
4778            module_channel,
4779            module_epoch,
4780            corr: relay_corr,
4781            receiver,
4782        } = pending;
4783        let mut reservation =
4784            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4785
4786        // Reserving egress can wait while a module reconnects or a swap cuts
4787        // over. Check the relay's captured connection, not the earlier by-id
4788        // lookup: the original module's flow/run capabilities cannot authorize
4789        // its replacement. The captured sink stays bound to that connection.
4790        let relay_registration = self
4791            .registry
4792            .get_module_by_connection(endpoint.connection_id)
4793            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4794        let relay_capabilities = relay_registration
4795            .as_ref()
4796            .and_then(|registration| registration.manifest.capabilities.as_ref());
4797        if let Some(selector) = &scope {
4798            // Egress reservation may wait across a scope update. Recheck the
4799            // proof against the current record before sending the captured bind.
4800            if let Err(refusal) = self
4801                .scopes
4802                .read()
4803                .unwrap_or_else(|poisoned| poisoned.into_inner())
4804                .check_bind_proof(&principal, selector)
4805            {
4806                reservation.release_and_disarm();
4807                return Ok(vec![self.route_open_refusal_frame(
4808                    ctx,
4809                    &frame,
4810                    &target_module_id,
4811                    refusal.code,
4812                    refusal.code,
4813                    refusal.message,
4814                )?]);
4815            }
4816        }
4817        if let Some(stamp) = scope_stamp
4818            .as_ref()
4819            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4820        {
4821            if let Err(refusal) = crate::scopes::check_target_flow_support(
4822                stamp,
4823                &target_module_id,
4824                relay_capabilities,
4825            )
4826            .and_then(|()| {
4827                crate::scopes::check_target_agent_run_support(
4828                    stamp,
4829                    &target_module_id,
4830                    relay_capabilities,
4831                )
4832            }) {
4833                reservation.release_and_disarm();
4834                return Ok(vec![self.route_open_refusal_frame(
4835                    ctx,
4836                    &frame,
4837                    &target_module_id,
4838                    refusal.code,
4839                    refusal.code,
4840                    refusal.message,
4841                )?]);
4842            }
4843        }
4844
4845        // Older readers may reject unknown fields. Select optional fields from
4846        // the captured endpoint's declaration, including on unscoped binds.
4847        if !relay_capabilities.is_some_and(|caps| {
4848            caps.provides
4849                .iter()
4850                .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4851        }) {
4852            if let Some(stamp) = scope_stamp.as_mut() {
4853                stamp.attributes.on_behalf_of_agent = None;
4854            }
4855        }
4856        let call_mode = if relay_capabilities
4857            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4858        {
4859            Some(derive_call_mode(scope_stamp.as_ref()))
4860        } else {
4861            if let Some(stamp) = scope_stamp.as_mut() {
4862                stamp.expires_at_ms = None;
4863            }
4864            None
4865        };
4866
4867        debug!(
4868            connection_id = ctx.connection_id.get(),
4869            client_channel,
4870            client_epoch,
4871            module_channel,
4872            module_epoch,
4873            "reserved route handle pair"
4874        );
4875        // Rendered BEFORE the move into the relay, because the accept arm below
4876        // is where it is logged and the principal is gone by then.
4877        let route_principal_label = principal_label(&principal);
4878        let scope_label = scope_stamp
4879            .as_ref()
4880            .map_or_else(|| "none".to_string(), route_open_scope_label);
4881        let relay = ModuleControlRequest::RouteBind {
4882            route_channel: module_channel,
4883            epoch: module_epoch,
4884            target,
4885            identity,
4886            principal: Some(principal),
4887            consumer_capabilities,
4888            role_versions,
4889            admission_facts,
4890            scope: scope_stamp,
4891            call_mode,
4892        };
4893        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4894            RouterError::backend(
4895                0,
4896                frame.header.corr,
4897                format!("failed to encode route.bind request: {err}"),
4898            )
4899        })?;
4900        let relay_frame = Frame::build_with_version(
4901            negotiated_ver,
4902            FrameType::Request,
4903            control_flags(),
4904            0,
4905            0,
4906            relay_corr,
4907            relay_body,
4908        )
4909        .map_err(RouterError::FrameBuild)?;
4910
4911        timings.enter(RouteOpenPhase::BindSend);
4912        let sent = module_sink.send(relay_frame).await;
4913        timings.enter(RouteOpenPhase::Admission);
4914        if let Err(err) = sent {
4915            reservation.release_and_disarm();
4916            return Ok(vec![self.route_open_refusal_frame(
4917                ctx,
4918                &frame,
4919                &target_module_id,
4920                "relay_send_failed",
4921                "target_unavailable",
4922                err.to_string(),
4923            )?]);
4924        }
4925
4926        #[cfg(test)]
4927        if let Some(hold) = &self.route_bind_relay_sent_hold {
4928            // First wait: tell the test the bind is queued. Second wait: stay
4929            // here until the test has driven whatever it wants to happen
4930            // between queueing the bind and marking it enqueued.
4931            hold.wait().await;
4932            hold.wait().await;
4933        }
4934
4935        // A missing relay is not by itself an abandoned bind: the module may
4936        // already have acked it and the route may be committed. Only a relay
4937        // dropped without any module reply needs this GOODBYE; one a reply
4938        // settled has already had its GOODBYE decided by the reply path.
4939        if self
4940            .forwarding
4941            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4942            .map_err(RouterError::Forwarding)?
4943            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4944        {
4945            self.send_abandoned_route_bind_goodbye(
4946                &module_sink,
4947                negotiated_ver,
4948                module_channel,
4949                module_epoch,
4950            );
4951        }
4952
4953        timings.enter(RouteOpenPhase::BindAckWait);
4954        let outcome = timeout_at(relay_deadline, receiver).await;
4955        timings.enter(RouteOpenPhase::Admission);
4956        match outcome {
4957            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4958                reservation.disarm();
4959                if breaker.record_accepted() {
4960                    info!(
4961                        module_id = %target_module_id,
4962                        "route.bind breaker closed: the probe was accepted"
4963                    );
4964                }
4965                self.observe_route_open_accept(
4966                    ctx,
4967                    &target_module_id,
4968                    &route_principal_label,
4969                    &scope_label,
4970                );
4971                Ok(Vec::new())
4972            }
4973            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4974                reservation.release_and_disarm();
4975                // A module that says no in microseconds is healthy. Rejection
4976                // is a different condition with its own refusal and must not
4977                // move the breaker.
4978                breaker.record_inconclusive();
4979                // The daemon's own commit re-check refused the bind because the
4980                // scope ended or changed after admission. The module accepted;
4981                // counting it as a module rejection would blame the module.
4982                let scope_code = match body.code.as_str() {
4983                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4984                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4985                    _ => None,
4986                };
4987                if let Some(code) = scope_code {
4988                    self.observe_route_open_refusal(
4989                        ctx,
4990                        &target_module_id,
4991                        "scope_changed_before_commit",
4992                        code,
4993                    );
4994                    return Ok(vec![control_error_body_frame(&frame, body)?]);
4995                }
4996                self.counters
4997                    .increment_route_open_refused("module_rejected");
4998                info!(
4999                    target: "control",
5000                    code = "module_rejected",
5001                    module_code = ?body.code,
5002                    module_id = ?target_module_id,
5003                    connection_id = ctx.connection_id.get(),
5004                    "route.open refused"
5005                );
5006                Ok(vec![control_error_body_frame(&frame, body)?])
5007            }
5008            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5009                reservation.release_and_disarm();
5010                breaker.record_inconclusive();
5011                // Fires when the module's connection closes while a relayed
5012                // bind is pending -- typically a caller racing a module restart
5013                // whose bind was relayed BEFORE the drain mark went up. Logged
5014                // because the caller sees only its own error and the fleet has
5015                // already spent one diagnosis round unable to tell this arm
5016                // from a relay timeout without daemon-side evidence.
5017                tracing::warn!(
5018                    module_id = %target_module_id,
5019                    "route.bind relay abandoned: {message}"
5020                );
5021                Ok(vec![self.route_open_refusal_frame(
5022                    ctx,
5023                    &frame,
5024                    &target_module_id,
5025                    "relay_abandoned",
5026                    "target_unavailable",
5027                    message,
5028                )?])
5029            }
5030            Ok(Err(_)) => {
5031                reservation.release_and_disarm();
5032                breaker.record_inconclusive();
5033                Ok(vec![self.route_open_refusal_frame(
5034                    ctx,
5035                    &frame,
5036                    &target_module_id,
5037                    "relay_waiter_canceled",
5038                    "target_unavailable",
5039                    "route.bind relay waiter was canceled before the module responded",
5040                )?])
5041            }
5042            Err(_) => {
5043                reservation.release_and_disarm();
5044                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5045                // answer at all is the one condition a fast refusal can
5046                // usefully stand in for; every other arm already answered.
5047                if let Some(opened) = breaker.record_timeout(
5048                    self.route_bind_breaker_threshold,
5049                    self.route_bind_breaker_cooldown,
5050                ) {
5051                    warn!(
5052                        module_id = %target_module_id,
5053                        consecutive_timeouts = opened.consecutive_timeouts,
5054                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5055                        reopened_after_probe = opened.reopened_after_probe,
5056                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5057                    );
5058                }
5059                // The generous budget just burned to no answer: the module is
5060                // registered and its connection is up, but its bind handler sat
5061                // on the ack for the full budget (warm-on-bind, cold configure,
5062                // or a wedged handler). Every earlier unavailability shape
5063                // fast-refuses BEFORE the relay, so this arm firing means the
5064                // slowness is module-side -- log it so the per-module timeline
5065                // is reconstructable without client audit rows.
5066                tracing::warn!(
5067                    module_id = %target_module_id,
5068                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5069                    "route.bind relay timed out: module did not ack within budget"
5070                );
5071                Ok(vec![self.route_open_refusal_frame(
5072                    ctx,
5073                    &frame,
5074                    &target_module_id,
5075                    "relay_timed_out",
5076                    "module_timeout",
5077                    format!(
5078                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5079                        route_bind_relay_timeout
5080                    ),
5081                )?])
5082            }
5083        }
5084    }
5085
5086    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5087        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5088            snapshot: self.supervisor.spawn_snapshot(),
5089        };
5090        Ok(vec![control_response_body_frame(
5091            &frame,
5092            &response,
5093            "ClientControlResponse::SupervisorSpawnSnapshot",
5094        )?])
5095    }
5096
5097    fn handle_supervisor_spawn_subscribe(
5098        &self,
5099        ctx: &RouteCtx,
5100        frame: Frame,
5101        since: Option<SpawnCursor>,
5102    ) -> Result<Vec<Frame>, RouterError> {
5103        match self.supervisor.subscribe_spawns(
5104            ctx.connection_id,
5105            frame.header.corr,
5106            response_version(&frame),
5107            since,
5108            ctx.egress.clone(),
5109        ) {
5110            Ok(()) => Ok(Vec::new()),
5111            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5112                Ok(vec![control_error_body_frame(
5113                    &frame,
5114                    ErrorBody {
5115                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5116                        message: "spawn cursor belongs to a different daemon incarnation"
5117                            .to_string(),
5118                        detail: Some(serde_json::json!({
5119                            "current_daemon_incarnation": current
5120                        })),
5121                    },
5122                )?])
5123            }
5124            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5125                &frame,
5126                ErrorBody {
5127                    code: "spawn_cursor_too_old".to_string(),
5128                    message: "spawn cursor predates the retained event ring".to_string(),
5129                    detail: Some(serde_json::json!({
5130                        "oldest_retained_cursor": oldest
5131                    })),
5132                },
5133            )?]),
5134            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5135                0,
5136                frame.header.corr,
5137                format!("failed to open supervisor spawn subscription: {error}"),
5138            )),
5139        }
5140    }
5141
5142    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5143        let generation = self
5144            .registry
5145            .generation()
5146            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5147        let mut modules = Vec::new();
5148        for module in self.supervisor.list() {
5149            let status = module.status_for_control("list").map_err(|err| {
5150                RouterError::backend(
5151                    0,
5152                    frame.header.corr,
5153                    format!("failed to read supervisor status: {err}"),
5154                )
5155            })?;
5156            let (configured, _) = module.configuration().map_err(|err| {
5157                RouterError::backend(
5158                    0,
5159                    frame.header.corr,
5160                    format!("failed to read module configuration: {err}"),
5161                )
5162            })?;
5163            // Status and configuration snapshots release their locks before the image probe awaits.
5164            let image = module.running_image_agreement().await;
5165            // Read per request so the figure is current when the operator asks;
5166            // the daemon samples nothing in between.
5167            let resources = Some(module.child_resource_usage());
5168            let mut verdict =
5169                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5170            verdict.spec = Some(launch_spec_agreement(
5171                &configured,
5172                status.process_alive,
5173                status.launch_spec_digest.as_ref(),
5174            ));
5175            let pending_reload = Some(verdict);
5176            // Keep the retired policy field on the wire for one release so
5177            // existing status consumers still receive the platform policy.
5178            modules.push(
5179                SupervisorEntry::new(
5180                    status.module_id,
5181                    status.state.to_string(),
5182                    status.enabled,
5183                    status.live,
5184                    status.health.status,
5185                )
5186                .with_launch_nonce_env(Some(!cfg!(unix)))
5187                .with_launch_nonce_source(status.launch_nonce_source)
5188                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5189                .with_protocol(status.protocol)
5190                .with_pending_reload(pending_reload)
5191                .with_last_probe_ms(status.health.last_probe_ms)
5192                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5193                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5194                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5195                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5196                .with_restart_count(Some(status.restart_count))
5197                .with_max_restarts(Some(status.max_restarts))
5198                .with_lifetime_restarts(Some(status.lifetime_restarts))
5199                .with_spawn_generation(Some(status.spawn_generation))
5200                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5201                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5202                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5203                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5204                .with_resources(resources),
5205            );
5206        }
5207        let response = ClientControlResponse::SupervisorList {
5208            generation,
5209            modules,
5210        };
5211        Ok(vec![control_response_body_frame(
5212            &frame,
5213            &response,
5214            "ClientControlResponse::SupervisorList",
5215        )?])
5216    }
5217
5218    fn handle_supervisor_stderr_tail(
5219        &self,
5220        frame: Frame,
5221        module_id: String,
5222        max_lines: Option<u32>,
5223        max_bytes: Option<u32>,
5224    ) -> Result<Vec<Frame>, RouterError> {
5225        let Some(module) = self.supervisor.get(&module_id) else {
5226            return Ok(vec![control_error_frame(
5227                &frame,
5228                "unknown_module",
5229                format!("module_id '{module_id}' is not supervised"),
5230            )?]);
5231        };
5232
5233        let snapshot = module.stderr_tail(
5234            max_lines.map(|value| value as usize),
5235            max_bytes.map(|value| value as usize),
5236        );
5237
5238        let response = ClientControlResponse::SupervisorStderrTail {
5239            module_id,
5240            tail: StderrTail {
5241                capture: match snapshot.capture {
5242                    CaptureState::Captured => StderrCaptureState::Captured,
5243                    CaptureState::Incomplete { reason } => {
5244                        StderrCaptureState::Incomplete { reason }
5245                    }
5246                    CaptureState::NotCaptured { reason } => {
5247                        StderrCaptureState::NotCaptured { reason }
5248                    }
5249                },
5250                entries: snapshot
5251                    .entries
5252                    .into_iter()
5253                    .map(|entry| match entry {
5254                        TailEntry::Line {
5255                            text,
5256                            truncated,
5257                            at_ms,
5258                        } => StderrTailEntry::Line {
5259                            text,
5260                            truncated,
5261                            at_ms,
5262                        },
5263                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5264                    })
5265                    .collect(),
5266                dropped_lines: snapshot.dropped_lines,
5267            },
5268        };
5269        Ok(vec![control_response_body_frame(
5270            &frame,
5271            &response,
5272            "ClientControlResponse::SupervisorStderrTail",
5273        )?])
5274    }
5275
5276    async fn handle_supervisor_terminals(
5277        &self,
5278        frame: Frame,
5279        module_id: String,
5280    ) -> Result<Vec<Frame>, RouterError> {
5281        let Some(module) = self.supervisor.get(&module_id) else {
5282            return Ok(vec![control_error_frame(
5283                &frame,
5284                "unknown_module",
5285                format!("module_id '{module_id}' is not supervised"),
5286            )?]);
5287        };
5288
5289        // The journal read runs on a blocking thread: it can be megabytes of
5290        // file I/O and must not occupy a runtime worker.
5291        let terminals = module
5292            .read_durable_terminal_history()
5293            .await
5294            .map_err(|error| {
5295                RouterError::backend(
5296                    0,
5297                    frame.header.corr,
5298                    format!("failed to read terminal history: {error}"),
5299                )
5300            })?;
5301        let response = ClientControlResponse::SupervisorTerminals {
5302            module_id,
5303            terminals,
5304        };
5305        Ok(vec![control_response_body_frame(
5306            &frame,
5307            &response,
5308            "ClientControlResponse::SupervisorTerminals",
5309        )?])
5310    }
5311
5312    fn handle_supervisor_routes(
5313        &self,
5314        frame: Frame,
5315        module_id: Option<String>,
5316    ) -> Result<Vec<Frame>, RouterError> {
5317        let modules = self
5318            .forwarding
5319            .route_census(module_id.as_deref())
5320            .map_err(RouterError::Forwarding)?
5321            .into_iter()
5322            .map(|(module_id, routes)| SupervisorRouteModule {
5323                module_id,
5324                routes: routes
5325                    .into_iter()
5326                    .map(|route| SupervisorRoute {
5327                        consumer: match route.principal {
5328                            Principal::Reserved { module_id } => {
5329                                SupervisorRouteConsumer::Reserved { module_id }
5330                            }
5331                            Principal::Direct | Principal::Unverified => {
5332                                SupervisorRouteConsumer::Direct {
5333                                    connection_id: route.goodbye_target.connection_id.get(),
5334                                }
5335                            }
5336                        },
5337                        age_ms: Instant::now()
5338                            .saturating_duration_since(route.bound_at)
5339                            .as_millis()
5340                            .try_into()
5341                            .unwrap_or(u64::MAX),
5342                        draining: route.draining,
5343                        drain_reason: route.drain_reason,
5344                    })
5345                    .collect(),
5346            })
5347            .collect();
5348        let response = ClientControlResponse::SupervisorRoutes { modules };
5349        Ok(vec![control_response_body_frame(
5350            &frame,
5351            &response,
5352            "ClientControlResponse::SupervisorRoutes",
5353        )?])
5354    }
5355
5356    async fn handle_supervisor_provenance(
5357        &self,
5358        frame: Frame,
5359        module_id: Option<String>,
5360    ) -> Result<Vec<Frame>, RouterError> {
5361        let mut selected = if let Some(module_id) = module_id {
5362            let Some(module) = self.supervisor.get(&module_id) else {
5363                return Ok(vec![control_error_frame(
5364                    &frame,
5365                    "unknown_module",
5366                    format!("module_id '{module_id}' is not supervised"),
5367                )?]);
5368            };
5369            vec![module]
5370        } else {
5371            self.supervisor.list()
5372        };
5373
5374        let mut modules = Vec::with_capacity(selected.len());
5375        for module in selected.drain(..) {
5376            let (status, observed_image) = module
5377                .status_and_running_image_agreement()
5378                .await
5379                .map_err(|err| {
5380                    RouterError::backend(
5381                        0,
5382                        frame.header.corr,
5383                        format!("failed to read supervisor status: {err}"),
5384                    )
5385                })?;
5386            let module_declared = self
5387                .registry
5388                .get_module(&status.module_id)
5389                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5390                .and_then(|registration| registration.manifest.provenance)
5391                .map(|build| ModuleDeclaredProvenance::Reported { build })
5392                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5393            #[cfg(test)]
5394            let running_image = match &self.provenance_probe_override {
5395                Some(result) => result.clone(),
5396                None => observed_image,
5397            };
5398            #[cfg(not(test))]
5399            let running_image = observed_image;
5400            modules.push(SupervisorModuleProvenance {
5401                module_id: status.module_id,
5402                module_declared,
5403                daemon_observed: SupervisorObservedProcess {
5404                    launch_nonce_source: status.launch_nonce_source,
5405                    pid: status.pid,
5406                    spawned_at_ms: status.spawned_at_ms,
5407                    spawned_from: status.spawned_from,
5408                    running_image,
5409                },
5410            });
5411        }
5412        let daemon = SupervisorDaemonProvenance {
5413            daemon_build: self.daemon_provenance.build.clone(),
5414            daemon_observed: DaemonObservedProcess {
5415                pid: self.daemon_provenance.pid,
5416                started_at_ms: self
5417                    .daemon_provenance
5418                    .start_clock
5419                    .map(|clock| clock.started_at_ms())
5420                    .or(self.daemon_provenance.started_at_ms),
5421                running_image: self
5422                    .daemon_provenance
5423                    .probe
5424                    .observe(
5425                        self.daemon_provenance.pid,
5426                        self.daemon_provenance.executable_path.as_deref(),
5427                        self.daemon_provenance.executable_identity.clone(),
5428                        self.daemon_provenance.process_start_time,
5429                    )
5430                    .await,
5431            },
5432        };
5433        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5434        Ok(vec![control_response_body_frame(
5435            &frame,
5436            &response,
5437            "ClientControlResponse::SupervisorProvenance",
5438        )?])
5439    }
5440
5441    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5442        self.refresh_capability_requirements();
5443        let generation = self
5444            .registry
5445            .generation()
5446            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5447        let modules = self
5448            .supervisor
5449            .list()
5450            .into_iter()
5451            .map(|module| {
5452                let status = module.status_for_control("health").map_err(|err| {
5453                    RouterError::backend(
5454                        0,
5455                        frame.header.corr,
5456                        format!("failed to read supervisor health: {err}"),
5457                    )
5458                })?;
5459                let module_id = status.module_id;
5460                let capability_detail = self
5461                    .capability_evaluator
5462                    .required_problem_detail(&module_id);
5463                Ok(SupervisorHealthEntry {
5464                    module_id,
5465                    status: status.health.status,
5466                    detail: append_capability_problem_detail(
5467                        status.health.detail,
5468                        capability_detail,
5469                    ),
5470                    metrics: status.health.metrics,
5471                    consecutive_failures: status.health.consecutive_failures,
5472                    late_answer_count: status.health.late_answer_count,
5473                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5474                    last_action: status.health.last_action,
5475                    last_action_ms: status.health.last_action_ms,
5476                    last_probe_ms: status.health.last_probe_ms,
5477                })
5478            })
5479            .collect::<Result<Vec<_>, RouterError>>()?;
5480        let response = ClientControlResponse::SupervisorHealth {
5481            generation,
5482            modules,
5483        };
5484        Ok(vec![control_response_body_frame(
5485            &frame,
5486            &response,
5487            "ClientControlResponse::SupervisorHealth",
5488        )?])
5489    }
5490
5491    async fn handle_supervisor_restart(
5492        &self,
5493        frame: Frame,
5494        module_id: String,
5495        drain_timeout_ms: Option<u64>,
5496        requester: Principal,
5497    ) -> Result<Vec<Frame>, RouterError> {
5498        let operation_lock = self.supervisor.operation_lock();
5499        let _operation_guard = operation_lock.lock().await;
5500        let Some(module) = self.supervisor.get(&module_id) else {
5501            return Ok(vec![control_error_frame(
5502                &frame,
5503                "unknown_module",
5504                format!("module_id '{module_id}' is not supervised"),
5505            )?]);
5506        };
5507
5508        self.route_outages.mark_operator_action(&module_id);
5509        if let Err(err) = module
5510            .restart_requested(drain_timeout_ms, Some(requester))
5511            .await
5512        {
5513            self.route_outages
5514                .operator_action_ended_unrefused(&module_id);
5515            let (code, message) = match err {
5516                crate::supervise::SuperviseError::Disabled { .. } => {
5517                    ("module_disabled", err.to_string())
5518                }
5519                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5520                    ("swap_in_progress", err.to_string())
5521                }
5522                _ => (
5523                    "target_unavailable",
5524                    format!("failed to restart module_id '{module_id}': {err}"),
5525                ),
5526            };
5527            return Ok(vec![control_error_frame(&frame, code, message)?]);
5528        }
5529
5530        let response = ClientControlResponse::SupervisorAck {
5531            module_id,
5532            applied: true,
5533        };
5534        Ok(vec![control_response_body_frame(
5535            &frame,
5536            &response,
5537            "ClientControlResponse::SupervisorAck",
5538        )?])
5539    }
5540
5541    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5542    /// when the old process has finished draining: a caller whose own lane
5543    /// rides the old process must get its reply before that drain waits on it.
5544    async fn handle_supervisor_swap(
5545        &self,
5546        frame: Frame,
5547        module_id: String,
5548        ready_timeout_ms: Option<u64>,
5549        requester: Principal,
5550    ) -> Result<Vec<Frame>, RouterError> {
5551        // The daemon-wide operation lock is held only to resolve the handle,
5552        // not across the swap. The swap can take its whole readiness budget,
5553        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5554        // holding it here would park an operator's stop behind the swap it is
5555        // meant to abort. A rescan or stop that reaches the module during the
5556        // swap is served by the swap itself (see `supervise_swap`).
5557        let module = {
5558            let operation_lock = self.supervisor.operation_lock();
5559            let _operation_guard = operation_lock.lock().await;
5560            self.supervisor.get(&module_id)
5561        };
5562        let Some(module) = module else {
5563            return Ok(vec![control_error_frame(
5564                &frame,
5565                "unknown_module",
5566                format!("module_id '{module_id}' is not supervised"),
5567            )?]);
5568        };
5569
5570        self.route_outages.mark_operator_action(&module_id);
5571        if let Err(err) = module
5572            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5573            .await
5574        {
5575            self.route_outages
5576                .operator_action_ended_unrefused(&module_id);
5577            use crate::supervise::SuperviseError;
5578            let message = err.to_string();
5579            let error = match err {
5580                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5581                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5582                    code: "swap_refused".to_string(),
5583                    message,
5584                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5585                },
5586                SuperviseError::SwapFailed {
5587                    arm,
5588                    candidate_exit,
5589                    ..
5590                } => ErrorBody {
5591                    code: "swap_failed".to_string(),
5592                    message,
5593                    detail: Some(serde_json::json!({
5594                        "arm": arm.as_str(),
5595                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5596                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5597                    })),
5598                },
5599                _ => ErrorBody::new(
5600                    "target_unavailable",
5601                    format!("failed to swap module_id '{module_id}': {message}"),
5602                ),
5603            };
5604            return Ok(vec![control_error_body_frame(&frame, error)?]);
5605        }
5606        // A completed swap kept the incumbent serving until cutover, so it
5607        // usually opened no outage; a mark left behind would make the next,
5608        // unrelated outage read as requested.
5609        self.route_outages
5610            .operator_action_ended_unrefused(&module_id);
5611
5612        let response = ClientControlResponse::SupervisorAck {
5613            module_id,
5614            applied: true,
5615        };
5616        Ok(vec![control_response_body_frame(
5617            &frame,
5618            &response,
5619            "ClientControlResponse::SupervisorAck",
5620        )?])
5621    }
5622
5623    async fn handle_supervisor_reload(
5624        &self,
5625        frame: Frame,
5626        module_id: String,
5627        requester: Principal,
5628    ) -> Result<Vec<Frame>, RouterError> {
5629        let operation_lock = self.supervisor.operation_lock();
5630        let _operation_guard = operation_lock.lock().await;
5631        let Some(module) = self.supervisor.get(&module_id) else {
5632            return Ok(vec![control_error_frame(
5633                &frame,
5634                "unknown_module",
5635                format!("module_id '{module_id}' is not supervised"),
5636            )?]);
5637        };
5638
5639        self.route_outages.mark_operator_action(&module_id);
5640        if let Err(err) = module.reload_requested(Some(requester)).await {
5641            self.route_outages
5642                .operator_action_ended_unrefused(&module_id);
5643            let (code, message) = match err {
5644                crate::supervise::SuperviseError::Disabled { .. } => {
5645                    ("module_disabled", err.to_string())
5646                }
5647                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5648                    ("swap_in_progress", err.to_string())
5649                }
5650                _ => (
5651                    "reload_failed",
5652                    format!("failed to reload module_id '{module_id}': {err}"),
5653                ),
5654            };
5655            return Ok(vec![control_error_frame(&frame, code, message)?]);
5656        }
5657
5658        let response = ClientControlResponse::SupervisorAck {
5659            module_id,
5660            applied: true,
5661        };
5662        Ok(vec![control_response_body_frame(
5663            &frame,
5664            &response,
5665            "ClientControlResponse::SupervisorAck",
5666        )?])
5667    }
5668
5669    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5670    /// the configuration comes from and whether to apply it; every mode then
5671    /// runs the same comparison, so a preview describes the apply it previews.
5672    async fn handle_supervisor_rescan(
5673        &self,
5674        frame: Frame,
5675        mode: RescanMode,
5676        requester: Principal,
5677    ) -> Result<Vec<Frame>, RouterError> {
5678        let preview = mode.previews();
5679        let Some(context) = self.rescan.clone() else {
5680            return Ok(vec![control_error_frame(
5681                &frame,
5682                "rescan_unavailable",
5683                "the daemon was not started with a reloadable config path".to_string(),
5684            )?]);
5685        };
5686
5687        let operation_lock = self.supervisor.operation_lock();
5688        let _operation_guard = operation_lock.lock().await;
5689        // Loaded under the operation lock, so no other rescan can change the
5690        // staged configuration between a checked apply's comparison and the
5691        // apply itself.
5692        let loaded = match self.load_rescan_config(&mode, &context) {
5693            Ok(config) => config,
5694            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5695        };
5696        // `load` reports a missing file as Ok(None), which is correct at boot
5697        // (no config, nothing to supervise) and catastrophic here: rescan treats
5698        // "not in the config" as "remove it", so an absent file would read as an
5699        // empty module list and retire the entire running fleet. An editor
5700        // writing via write-new-then-rename, or a half-finished edit, is enough
5701        // to open that window. Refuse instead: a config that cannot be read
5702        // carries no instruction to remove anything.
5703        let Some((config, activity_config)) = loaded else {
5704            return Ok(vec![control_error_frame(
5705                &frame,
5706                "invalid_daemon_config",
5707                format!(
5708                    "daemon config not found at {}; refusing to rescan (an absent config would \
5709                     retire every supervised module)",
5710                    context.config_path.display()
5711                ),
5712            )?]);
5713        };
5714        let (
5715            configured_port,
5716            storage_config,
5717            admission_facts_carrier_module_id,
5718            admission_facts_targets,
5719            scope_authority_owners,
5720            privileged_operator_prompt,
5721            modules,
5722            reserved_capabilities,
5723        ) = (
5724            config.port,
5725            config.storage,
5726            config.admission_facts_carrier_module_id,
5727            config.admission_facts_targets,
5728            config.scope_authority_owners,
5729            config.privileged_operator_prompt,
5730            config.modules,
5731            config.reserved_capabilities,
5732        );
5733
5734        // Collect the sections rescan cannot apply, so the REPLY carries them.
5735        //
5736        // The warning below has always been correct and has always gone only to
5737        // the journal -- addressed to whoever reads logs, while the person who
5738        // just edited the config is looking at the CLI. Naming each section
5739        // individually rather than setting a flag: "something outside modules
5740        // changed" sends the operator back to diffing their own file, which is
5741        // the work this is meant to save.
5742        let mut restart_required = Vec::new();
5743        for section in RestartRequiredSection::ALL {
5744            let changed = match section {
5745                RestartRequiredSection::Port => configured_port != context.configured_port,
5746                RestartRequiredSection::Storage => storage_config != context.storage_config,
5747                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5748                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5749                }
5750                RestartRequiredSection::AdmissionFactsTargets => {
5751                    admission_facts_targets != context.admission_facts_targets
5752                }
5753                RestartRequiredSection::ScopeAuthorityOwners => {
5754                    scope_authority_owners != context.scope_authority_owners
5755                }
5756                RestartRequiredSection::PrivilegedOperatorPrompt => {
5757                    privileged_operator_prompt != context.privileged_operator_prompt
5758                }
5759            };
5760            if changed {
5761                restart_required.push(section.label().to_string());
5762            }
5763        }
5764        if !restart_required.is_empty() {
5765            warn!(
5766                config_path = %context.config_path.display(),
5767                sections = %restart_required.join(", "),
5768                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5769            );
5770        }
5771
5772        for configured in &modules {
5773            if let Err(err) = validate_spec(&configured.module_spec()) {
5774                return Ok(vec![control_error_frame(
5775                    &frame,
5776                    "invalid_daemon_config",
5777                    format!("supervisor rescan rejected daemon config: {err}"),
5778                )?]);
5779            }
5780        }
5781
5782        let configured_capabilities = modules
5783            .iter()
5784            .map(|module| (module.module_id.clone(), module.enabled))
5785            .collect::<Vec<_>>();
5786        let preview_capability_warnings = if preview {
5787            let (_, registrations) = self.runtime_capability_snapshot()?;
5788            let current_modules = self
5789                .supervisor
5790                .list()
5791                .into_iter()
5792                .map(|module| module.module_id().to_string())
5793                .collect::<BTreeSet<_>>();
5794            let resulting_modules = configured_capabilities.clone();
5795            let removed = current_modules
5796                .into_iter()
5797                .filter(|module_id| {
5798                    !resulting_modules
5799                        .iter()
5800                        .any(|(configured_id, _)| configured_id == module_id)
5801                })
5802                .collect::<Vec<_>>();
5803            // A candidate preview also names providers the candidate disables.
5804            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
5805            let disabled = match &mode {
5806                RescanMode::Candidate(_) => resulting_modules
5807                    .iter()
5808                    .filter(|(_, enabled)| !enabled)
5809                    .map(|(module_id, _)| module_id.clone())
5810                    .collect::<Vec<_>>(),
5811                _ => Vec::new(),
5812            };
5813            self.capability_evaluator.preview_warnings(
5814                resulting_modules,
5815                &removed,
5816                &disabled,
5817                &registrations,
5818            )
5819        } else {
5820            Vec::new()
5821        };
5822        let reconciled = match &mode {
5823            RescanMode::Candidate(candidate) => self
5824                .preview_candidate(&context, modules, candidate)
5825                .map(|result| (result, Vec::new())),
5826            _ => {
5827                self.reconcile_supervised_modules(
5828                    &context.supervisor,
5829                    modules,
5830                    preview,
5831                    requester.clone(),
5832                )
5833                .await
5834            }
5835        };
5836        let (result, changed) = match reconciled {
5837            Ok(result) => result,
5838            Err(message) => {
5839                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5840            }
5841        };
5842        if !preview {
5843            if let Some(ledger) = &self.activity {
5844                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
5845                ledger.config_changed(&result.added, &result.removed, &changed, requester);
5846            }
5847            if let Err(code) = self.configure_activity(activity_config).await {
5848                warn!(
5849                    code,
5850                    "activity retention could not be applied during rescan"
5851                );
5852            }
5853            self.capability_evaluator
5854                .configure(configured_capabilities, reserved_capabilities);
5855            self.capability_evaluator.wake_deadline_loop();
5856            self.refresh_capability_requirements();
5857        }
5858        let mut result = result;
5859        result.restart_required = restart_required;
5860        result.capability_warnings = preview_capability_warnings;
5861        let response = ClientControlResponse::SupervisorRescan { result };
5862        Ok(vec![control_response_body_frame(
5863            &frame,
5864            &response,
5865            "ClientControlResponse::SupervisorRescan",
5866        )?])
5867    }
5868
5869    async fn handle_supervisor_release_reserved(
5870        &self,
5871        frame: Frame,
5872        module_id: String,
5873    ) -> Result<Vec<Frame>, RouterError> {
5874        let Some(context) = self.rescan.clone() else {
5875            return Ok(vec![control_error_frame(
5876                &frame,
5877                "release_unavailable",
5878                "reserved-id release requires a daemon started with a reloadable config path",
5879            )?]);
5880        };
5881        let operation_lock = self.supervisor.operation_lock();
5882        let _operation_guard = operation_lock.lock().await;
5883        let loaded = match crate::daemon_config::load(&context.config_path) {
5884            Ok(Some(config)) => config,
5885            Ok(None) => {
5886                return Ok(vec![control_error_frame(
5887                    &frame,
5888                    "invalid_daemon_config",
5889                    format!(
5890                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5891                        context.config_path.display()
5892                    ),
5893                )?])
5894            }
5895            Err(err) => {
5896                return Ok(vec![control_error_frame(
5897                    &frame,
5898                    "invalid_daemon_config",
5899                    format!("unable to verify reserved-id release against daemon config: {err}"),
5900                )?])
5901            }
5902        };
5903        if loaded
5904            .modules
5905            .iter()
5906            .any(|configured| configured.module_id == module_id)
5907        {
5908            return Ok(vec![control_error_frame(
5909                &frame,
5910                "reserved_module_configured",
5911                format!(
5912                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5913                ),
5914            )?]);
5915        }
5916        if !self.supervisor.release_retained_reserved_gate(&module_id) {
5917            return Ok(vec![control_error_frame(
5918                &frame,
5919                "reserved_gate_not_retained",
5920                format!(
5921                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5922                ),
5923            )?]);
5924        }
5925
5926        let response = ClientControlResponse::SupervisorAck {
5927            module_id,
5928            applied: true,
5929        };
5930        Ok(vec![control_response_body_frame(
5931            &frame,
5932            &response,
5933            "ClientControlResponse::SupervisorAck",
5934        )?])
5935    }
5936
5937    /// Compare the configured module set with the supervised one, changing
5938    /// nothing. The staged spec each supervised module holds is what the
5939    /// configured entry is compared with.
5940    fn plan_rescan(
5941        &self,
5942        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5943    ) -> Result<RescanPlan, String> {
5944        let mut current = BTreeMap::new();
5945        for module in self.supervisor.list() {
5946            let (spec, health) = module.configuration().map_err(|err| {
5947                format!(
5948                    "failed to read configuration for module_id '{}': {err}",
5949                    module.module_id()
5950                )
5951            })?;
5952            let enabled = module
5953                .status()
5954                .map_err(|err| {
5955                    format!(
5956                        "failed to read status for module_id '{}': {err}",
5957                        module.module_id()
5958                    )
5959                })?
5960                .enabled;
5961            current.insert(
5962                module.module_id().to_string(),
5963                (module, spec, health, enabled),
5964            );
5965        }
5966        let configured = configured_modules
5967            .into_iter()
5968            .map(|module| (module.module_id.clone(), module))
5969            .collect::<BTreeMap<_, _>>();
5970
5971        let added = configured
5972            .keys()
5973            .filter(|module_id| !current.contains_key(*module_id))
5974            .cloned()
5975            .collect::<Vec<_>>();
5976        let removed = current
5977            .keys()
5978            .filter(|module_id| !configured.contains_key(*module_id))
5979            .cloned()
5980            .collect::<Vec<_>>();
5981        let mut changed_pending_reload = Vec::new();
5982        let mut configuration_changes = BTreeSet::new();
5983        let mut enabled_changes = BTreeSet::new();
5984        let mut unchanged = 0_u32;
5985
5986        for (module_id, configured_module) in &configured {
5987            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5988            else {
5989                continue;
5990            };
5991            // Compare the whole launch spec so a future launch field cannot
5992            // accidentally become a live-only policy change. Health is stored
5993            // separately and applies live without replacing the process.
5994            let launch_changed = *current_spec != configured_module.module_spec();
5995            let configuration_changed =
5996                launch_changed || *current_health != configured_module.health;
5997            let enabled_changed = *current_enabled != configured_module.enabled;
5998            if configuration_changed {
5999                configuration_changes.insert(module_id.clone());
6000            }
6001            if launch_changed {
6002                changed_pending_reload.push(module_id.clone());
6003            }
6004            if enabled_changed {
6005                enabled_changes.insert(module_id.clone());
6006            }
6007            if !configuration_changed && !enabled_changed {
6008                unchanged = unchanged.saturating_add(1);
6009            }
6010        }
6011        Ok(RescanPlan {
6012            current,
6013            configured,
6014            added,
6015            removed,
6016            changed_pending_reload,
6017            configuration_changes,
6018            enabled_changes,
6019            unchanged,
6020        })
6021    }
6022
6023    /// Reconcile the running module set against the configured one.
6024    ///
6025    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6026    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6027    /// deliberately shares this function with the executing path rather than
6028    /// computing the same diff somewhere else -- two implementations of one
6029    /// decision agree until they do not, and the whole value of a preview is that
6030    /// it describes the operation that will actually run.
6031    async fn reconcile_supervised_modules(
6032        &self,
6033        supervisor: &Supervisor,
6034        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6035        preview: bool,
6036        requester: Principal,
6037    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6038        let plan = self.plan_rescan(configured_modules)?;
6039
6040        // Everything above this point is pure computation over two snapshots.
6041        // Everything below MUTATES. The preview returns here so the boundary is a
6042        // single early return rather than a condition repeated at each mutation
6043        // site, where one missed guard would apply part of a change the caller was
6044        // told would not happen.
6045        if preview {
6046            return Ok((plan.preview_result(), Vec::new()));
6047        }
6048        let changed = plan
6049            .configuration_changes
6050            .union(&plan.enabled_changes)
6051            .cloned()
6052            .collect::<Vec<_>>();
6053        let RescanPlan {
6054            current,
6055            configured,
6056            added,
6057            removed,
6058            changed_pending_reload,
6059            configuration_changes,
6060            enabled_changes,
6061            unchanged,
6062        } = plan;
6063        // The staged generation moves before the first change rather than
6064        // after the last, so a rescan that fails partway still invalidates
6065        // every earlier `supervisor.rescan_checked` preview: part of its diff
6066        // may have run.
6067        if !(added.is_empty()
6068            && removed.is_empty()
6069            && configuration_changes.is_empty()
6070            && enabled_changes.is_empty())
6071        {
6072            if let Some(context) = &self.rescan {
6073                context
6074                    .generation
6075                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6076            }
6077        }
6078
6079        for module_id in &removed {
6080            let module = &current
6081                .get(module_id)
6082                .expect("removed module came from current supervisor state")
6083                .0;
6084            module
6085                .retire_requested(Some(requester.clone()))
6086                .await
6087                .map_err(|err| {
6088                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6089                })?;
6090            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6091            //
6092            // `handle_route_open` resolves an absent module in three steps:
6093            // registry, then supervisor status, then tombstone. Retiring first
6094            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6095            // went with the teardown above, the supervisor entry went with
6096            // `retire`, and the tombstone does not exist yet -- so a route.open
6097            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6098            // it") for a module that was deliberately removed and whose caller
6099            // should get `module_removed` (TERMINAL, carrying a removal age).
6100            //
6101            // Writing the tombstone first closes it: during the window the
6102            // supervisor entry still answers, so the caller gets
6103            // `target_unavailable` -- retryable, and TRUE, because the module
6104            // is mid-teardown. After both statements it is `module_removed`.
6105            // No instant remains where a removed module reads as one that
6106            // never existed.
6107            //
6108            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6109            // the absence of a test beside a fix invites deletion: these are
6110            // two sync statements with no await between them, so reaching the
6111            // window needs a second worker thread to land exactly between them
6112            // and there is no hook to force it. MEASURED: the 25 daemon_config
6113            // tests pass identically with the old order and the new one, so
6114            // the existing suite cannot see this and a green run is not
6115            // evidence either way. What the suite does hold is the
6116            // post-condition -- a removed module answers `module_removed` --
6117            // which this preserves.
6118            //
6119            // Found by an Athena panel reading the shipped tree against a
6120            // design note (2026-09-19), as the one concrete instance of that
6121            // note's class that survived contact with source. Direction is
6122            // benign: retryable where terminal was intended, never the reverse.
6123            self.supervisor.record_rescan_removal(module_id);
6124            self.supervisor.retire(module_id);
6125            self.route_outages.forget(module_id);
6126        }
6127
6128        for module_id in configured.keys() {
6129            let Some((module, _, _, _)) = current.get(module_id) else {
6130                continue;
6131            };
6132            let configured_module = configured
6133                .get(module_id)
6134                .expect("configured module id came from configured map");
6135            if configuration_changes.contains(module_id) {
6136                module
6137                    .update_configuration(
6138                        configured_module.module_spec(),
6139                        configured_module.health.clone(),
6140                        configured_module.drain_timeout_ms,
6141                    )
6142                    .await
6143                    .map_err(|err| {
6144                        format!(
6145                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6146                        )
6147                    })?;
6148            }
6149            if enabled_changes.contains(module_id) {
6150                // A rescan that starts or stops a module applies an operator's
6151                // edit to the config, so the resulting outage was asked for.
6152                self.route_outages.mark_operator_action(module_id);
6153                module
6154                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6155                    .await
6156                    .map_err(|err| {
6157                        self.route_outages.operator_action_ended_unrefused(module_id);
6158                        format!(
6159                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6160                            configured_module.enabled
6161                        )
6162                    })?;
6163            }
6164        }
6165
6166        for module_id in &added {
6167            let configured_module = configured
6168                .get(module_id)
6169                .expect("added module id came from configured map");
6170            supervisor
6171                .supervise_configured_requested(
6172                    configured_module.module_spec(),
6173                    configured_module.enabled,
6174                    configured_module.health.clone(),
6175                    configured_module.drain_timeout_ms,
6176                    configured_module.restart,
6177                    Some(requester.clone()),
6178                )
6179                .map_err(|err| {
6180                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6181                })?;
6182        }
6183
6184        Ok((
6185            SupervisorRescanResult {
6186                added,
6187                removed,
6188                changed_pending_reload,
6189                enabled_changes: enabled_changes.iter().cloned().collect(),
6190                unchanged,
6191                preview: false,
6192                // Filled by the caller, which is the only layer that can see the
6193                // previous config to diff against.
6194                restart_required: Vec::new(),
6195                capability_warnings: Vec::new(),
6196                module_actions: Vec::new(),
6197                earlier_pending: Vec::new(),
6198                expect: None,
6199            },
6200            changed,
6201        ))
6202    }
6203
6204    async fn handle_supervisor_set_enabled(
6205        &self,
6206        frame: Frame,
6207        module_id: String,
6208        enabled: bool,
6209        requester: Principal,
6210    ) -> Result<Vec<Frame>, RouterError> {
6211        let operation_lock = self.supervisor.operation_lock();
6212        let _operation_guard = operation_lock.lock().await;
6213        let Some(module) = self.supervisor.get(&module_id) else {
6214            return Ok(vec![control_error_frame(
6215                &frame,
6216                "unknown_module",
6217                format!("module_id '{module_id}' is not supervised"),
6218            )?]);
6219        };
6220
6221        // Enabling counts as well as disabling: a module an operator starts
6222        // is refused until it registers, and that wait was asked for.
6223        self.route_outages.mark_operator_action(&module_id);
6224        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6225            Ok(applied) => applied,
6226            Err(err) => {
6227                self.route_outages
6228                    .operator_action_ended_unrefused(&module_id);
6229                return Ok(vec![control_error_frame(
6230                    &frame,
6231                    "target_unavailable",
6232                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6233                )?]);
6234            }
6235        };
6236        if !applied {
6237            // Already in the requested state: nothing was made unavailable,
6238            // so the mark must not outlive this request.
6239            self.route_outages
6240                .operator_action_ended_unrefused(&module_id);
6241        }
6242
6243        self.capability_evaluator.wake_deadline_loop();
6244        self.refresh_capability_requirements();
6245        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6246        Ok(vec![control_response_body_frame(
6247            &frame,
6248            &response,
6249            "ClientControlResponse::SupervisorAck",
6250        )?])
6251    }
6252
6253    async fn handle_supervisor_health_probe(
6254        &self,
6255        frame: Frame,
6256        module_id: String,
6257    ) -> Result<Vec<Frame>, RouterError> {
6258        self.refresh_capability_requirements();
6259        let Some(registration) = self
6260            .registry
6261            .get_module(&module_id)
6262            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6263        else {
6264            return Ok(vec![control_error_frame(
6265                &frame,
6266                "unknown_module",
6267                format!("module_id '{module_id}' is not registered"),
6268            )?]);
6269        };
6270
6271        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6272        // named for it. Making `module_registration_grants_op` return false
6273        // unconditionally reddens five tests, and every one is named for something
6274        // else -- capability relay, probe/bind demultiplexing, supervision-only
6275        // probing. They exercise a successful advertisement check on the way to their
6276        // own subject.
6277        //
6278        // Real protection, fragile in a specific way: narrowing any of those tests to
6279        // focus on its stated subject would silently remove coverage nobody knows
6280        // they are carrying. Recorded here rather than as a sixth test, because the
6281        // useful fact is WHICH tests hold the guard up -- a new test would add
6282        // coverage without telling the next person what the existing ones quietly do.
6283        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6284        {
6285            return Ok(vec![control_error_frame(
6286                &frame,
6287                "health_not_advertised",
6288                format!("module_id '{module_id}' did not advertise health.check"),
6289            )?]);
6290        }
6291
6292        let deadline = Instant::now() + self.health_probe_timeout;
6293        let pending = match self.forwarding.begin_module_control_rpc_for(
6294            &module_id,
6295            MODULE_CONTROL_OP_HEALTH_CHECK,
6296            deadline,
6297        ) {
6298            Ok(pending) => pending,
6299            Err(err) => {
6300                return Ok(vec![control_error_frame(
6301                    &frame,
6302                    forwarding_error_code(&err),
6303                    err.to_string(),
6304                )?])
6305            }
6306        };
6307
6308        let PendingModuleControlRpc {
6309            endpoint,
6310            module_sink,
6311            negotiated_ver,
6312            corr: probe_corr,
6313            receiver,
6314        } = pending;
6315        let mut guard =
6316            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6317        let probe_body =
6318            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6319                RouterError::backend(
6320                    0,
6321                    frame.header.corr,
6322                    format!("failed to encode health.check request: {err}"),
6323                )
6324            })?;
6325        let probe_frame = Frame::build_with_version(
6326            negotiated_ver,
6327            FrameType::Request,
6328            control_flags(),
6329            0,
6330            0,
6331            probe_corr,
6332            probe_body,
6333        )
6334        .map_err(RouterError::FrameBuild)?;
6335
6336        // Queue admission and the reply share one deadline, so a module that
6337        // stops reading cannot park the requesting connection's reader forever.
6338        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6339            Ok(Ok(())) => {}
6340            Ok(Err(err)) => {
6341                return Ok(vec![control_error_frame(
6342                    &frame,
6343                    "target_unavailable",
6344                    err.to_string(),
6345                )?]);
6346            }
6347            Err(_) => {
6348                return Ok(vec![control_error_frame(
6349                    &frame,
6350                    "module_timeout",
6351                    format!(
6352                        "module_id '{module_id}' did not answer health.check within {:?}",
6353                        self.health_probe_timeout
6354                    ),
6355                )?]);
6356            }
6357        }
6358
6359        match timeout_at(deadline, receiver).await {
6360            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6361                guard.disarm();
6362                let Some(report) = response.health_report() else {
6363                    return Ok(vec![control_error_frame(
6364                        &frame,
6365                        "invalid_control_body",
6366                        "health.check RPC returned a non-health response",
6367                    )?]);
6368                };
6369                // Metrics go out whole here. The supervisor's cached snapshot
6370                // caps this blob (see truncate_health_metrics), and this path
6371                // exists precisely to answer without that cap -- so applying it
6372                // here would leave no way to see what the cached view drops.
6373                let HealthReport {
6374                    status,
6375                    detail,
6376                    metrics,
6377                } = report;
6378                let capability_detail = self
6379                    .capability_evaluator
6380                    .required_problem_detail(&module_id);
6381                let response = ClientControlResponse::SupervisorHealthProbe {
6382                    module_id,
6383                    status,
6384                    detail: append_capability_problem_detail(detail, capability_detail),
6385                    metrics,
6386                };
6387                Ok(vec![control_response_body_frame(
6388                    &frame,
6389                    &response,
6390                    "ClientControlResponse::SupervisorHealthProbe",
6391                )?])
6392            }
6393            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6394                guard.disarm();
6395                Ok(vec![control_error_body_frame(&frame, body)?])
6396            }
6397            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6398                guard.disarm();
6399                Ok(vec![control_error_frame(
6400                    &frame,
6401                    "target_unavailable",
6402                    message,
6403                )?])
6404            }
6405            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6406                guard.disarm();
6407                Ok(vec![control_error_frame(
6408                    &frame,
6409                    "invalid_control_body",
6410                    message,
6411                )?])
6412            }
6413            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6414                guard.disarm();
6415                Ok(vec![control_error_frame(
6416                    &frame,
6417                    "invalid_control_body",
6418                    format!("expected module-control op '{expected}', got '{actual}'"),
6419                )?])
6420            }
6421            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6422                guard.disarm();
6423                Ok(vec![control_error_frame(
6424                    &frame,
6425                    "module_timeout",
6426                    format!(
6427                        "module_id '{module_id}' answered health.check after {:?}",
6428                        self.health_probe_timeout
6429                    ),
6430                )?])
6431            }
6432            Ok(Err(_)) => Ok(vec![control_error_frame(
6433                &frame,
6434                "target_unavailable",
6435                "health.check waiter was canceled before the module responded",
6436            )?]),
6437            Err(_) => Ok(vec![control_error_frame(
6438                &frame,
6439                "module_timeout",
6440                format!(
6441                    "module_id '{module_id}' did not answer health.check within {:?}",
6442                    self.health_probe_timeout
6443                ),
6444            )?]),
6445        }
6446    }
6447
6448    fn supervisor_status(
6449        &self,
6450        module_id: &str,
6451        corr: u64,
6452    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6453        self.supervisor
6454            .get(module_id)
6455            .map(|module| {
6456                let warming = module.is_warming_for_control("status").map_err(|err| {
6457                    RouterError::backend(
6458                        0,
6459                        corr,
6460                        format!(
6461                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6462                        ),
6463                    )
6464                })?;
6465                module.status_for_control("status").map_err(|err| {
6466                    RouterError::backend(
6467                        0,
6468                        corr,
6469                        format!(
6470                            "failed to read supervisor status for module_id '{module_id}': {err}"
6471                        ),
6472                    )
6473                }).map(|status| (status, warming))
6474            })
6475            .transpose()
6476    }
6477
6478    fn guard_module_control_op(
6479        &self,
6480        frame: &Frame,
6481        module_id: &str,
6482        op: &str,
6483    ) -> Result<Option<Frame>, RouterError> {
6484        if self.module_grants_op(module_id, op, frame.header.corr)? {
6485            return Ok(None);
6486        }
6487
6488        Ok(Some(control_error_frame(
6489            frame,
6490            "op_not_allowed",
6491            format!("module_id '{module_id}' did not grant control op '{op}'"),
6492        )?))
6493    }
6494
6495    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6496        let Some(registration) = self
6497            .registry
6498            .get_module(module_id)
6499            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6500        else {
6501            return Ok(false);
6502        };
6503        Ok(module_registration_grants_op(&registration.control_ops, op))
6504    }
6505
6506    fn handle_status_update(
6507        &self,
6508        endpoint: ModuleEndpointId,
6509        frame: Frame,
6510    ) -> Result<Vec<Frame>, RouterError> {
6511        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6512            Ok(update) => update,
6513            Err(err) => {
6514                // Forward-compat: a newer module may push a channel-0 op this subc
6515                // version doesn't know. The control contract says unknown push ops
6516                // are IGNORED, never answered with an error. Only a malformed body
6517                // for an op we DO know is a real error worth surfacing.
6518                if is_known_module_push_op(&frame.body) {
6519                    return Ok(vec![control_error_frame(
6520                        &frame,
6521                        "invalid_control_body",
6522                        format!("malformed module control push body: {err}"),
6523                    )?]);
6524                }
6525                return Ok(Vec::new());
6526            }
6527        };
6528
6529        match update {
6530            ModuleControlPush::RouteStatus {
6531                route_channel,
6532                route_epoch,
6533                status,
6534            } => {
6535                self.forwarding
6536                    .cache_status(endpoint, route_channel, route_epoch, status)
6537                    .map_err(RouterError::Forwarding)?;
6538            }
6539        }
6540        Ok(Vec::new())
6541    }
6542
6543    fn handle_route_poll(
6544        &self,
6545        ctx: &RouteCtx,
6546        frame: Frame,
6547        route_channel: u16,
6548        route_epoch: u32,
6549        kind: PollKind,
6550    ) -> Result<Vec<Frame>, RouterError> {
6551        let snapshot = self
6552            .forwarding
6553            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6554            .map_err(RouterError::Forwarding)?;
6555        let response = match (kind, snapshot) {
6556            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6557                ClientControlResponse::RoutePoll {
6558                    route_channel,
6559                    route_epoch,
6560                    status,
6561                    live: None,
6562                }
6563            }
6564            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6565                route_channel,
6566                route_epoch,
6567                status: None,
6568                live: None,
6569            },
6570            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6571                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6572                // is what makes reporting `true` correct rather than a
6573                // confident guess. `process_live` returns None only when the
6574                // module id has no supervisor snapshot at all -- an
6575                // externally-started module the daemon did not spawn -- and
6576                // for those the supervisor has no opinion to offer, ever. It
6577                // is never None for a supervised module in an unknown state:
6578                // a supervised module always has a snapshot, and the answer
6579                // comes from `state == Running && process_alive`.
6580                //
6581                // The route is Bound, so the module completed a HELLO on a
6582                // live connection; "the process this route points at is
6583                // running" is therefore attested by the binding rather than
6584                // assumed. Reporting `false` for an unsupervised module would
6585                // be the actual lie -- it would tell a client its healthy
6586                // route is dead because the daemon does not manage the
6587                // process.
6588                //
6589                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6590                // module whose liveness is genuinely unknown, e.g. a snapshot
6591                // that has not been populated yet -- THIS DEFAULT BECOMES
6592                // WRONG and must split: unsupervised stays true, unknown
6593                // becomes null so the client can tell the two apart. The
6594                // response field is already `Option<bool>`, so the wire can
6595                // carry that distinction today.
6596                let live = self
6597                    .process_liveness
6598                    .as_ref()
6599                    .and_then(|source| source.process_live(&module_id))
6600                    .unwrap_or(true);
6601                ClientControlResponse::RoutePoll {
6602                    route_channel,
6603                    route_epoch,
6604                    status: None,
6605                    live: Some(live),
6606                }
6607            }
6608            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6609                route_channel,
6610                route_epoch,
6611                status: None,
6612                live: Some(false),
6613            },
6614        };
6615
6616        Ok(vec![control_response_body_frame(
6617            &frame,
6618            &response,
6619            "ClientControlResponse::RoutePoll",
6620        )?])
6621    }
6622
6623    pub(crate) fn observe_module_control_completion(
6624        &self,
6625        completion: ModuleControlRpcCompletion,
6626    ) -> bool {
6627        match completion {
6628            ModuleControlRpcCompletion::Unknown => false,
6629            ModuleControlRpcCompletion::Settled => true,
6630            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6631                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6632                info!(
6633                    module_id = %module_id,
6634                    latency_ms,
6635                    "late health.check answer proves the module is alive"
6636                );
6637                match self
6638                    .supervisor
6639                    .record_late_health_answer(&module_id, latency_ms)
6640                {
6641                    Ok(true) => {}
6642                    Ok(false) => debug!(
6643                        module_id = %module_id,
6644                        latency_ms,
6645                        "late health.check answer has no active supervisor snapshot"
6646                    ),
6647                    Err(err) => warn!(
6648                        module_id = %module_id,
6649                        latency_ms,
6650                        error = %err,
6651                        "failed to record late health.check answer"
6652                    ),
6653                }
6654                true
6655            }
6656        }
6657    }
6658
6659    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6660    /// the module connection whose frame is being handled, or to the client that
6661    /// relay was opened for.
6662    ///
6663    /// This runs on the MODULE connection's frame handler, where returning `Err`
6664    /// ends that connection -- and a module connection carries every client's
6665    /// routes to that module, so ending it costs the whole fleet its tools.
6666    /// `ConnectionClosing` carries the id of the connection that is closing, and
6667    /// when that id is a CLIENT's, the condition is entirely about that one
6668    /// client's route.open. A client-scoped condition has no authority over a
6669    /// shared module connection, so it is logged and the single relay is dropped:
6670    /// the client is going away, and `complete_pending_relay` already removed the
6671    /// relay before failing, so there is nothing left to settle. Anything that
6672    /// relay still reserved is released by that client's own connection teardown,
6673    /// which is already under way -- that is what "closing" means.
6674    ///
6675    /// Every other failure is a statement about THIS connection and stays fatal:
6676    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6677    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6678    /// frames correctly.
6679    fn refuse_to_end_module_connection_for_a_client(
6680        &self,
6681        module_connection_id: ConnectionId,
6682        corr: u64,
6683        err: ForwardingError,
6684    ) -> Result<(), RouterError> {
6685        if let ForwardingError::ConnectionClosing { connection_id } = err {
6686            if connection_id != module_connection_id {
6687                warn!(
6688                    module_connection_id = module_connection_id.get(),
6689                    client_connection_id = connection_id.get(),
6690                    corr,
6691                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6692                );
6693                return Ok(());
6694            }
6695        }
6696        Err(RouterError::Forwarding(err))
6697    }
6698
6699    fn handle_module_relay_response(
6700        &self,
6701        connection_id: ConnectionId,
6702        frame: Frame,
6703    ) -> Result<Vec<Frame>, RouterError> {
6704        let mut secondary_error = None;
6705        let outcome = match frame.header.ty {
6706            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6707                Ok(probe) if probe.op == "route.bind" => {
6708                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6709                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6710                            RouteBindRelayOutcome::Accepted
6711                        }
6712                        Ok(other) => {
6713                            let message =
6714                                format!("route.bind response carried unexpected body: {other:?}");
6715                            secondary_error = Some(control_error_frame(
6716                                &frame,
6717                                "invalid_control_body",
6718                                message.clone(),
6719                            )?);
6720                            RouteBindRelayOutcome::ModuleGone(message)
6721                        }
6722                        Err(err) => {
6723                            let message = format!("malformed route.bind response body: {err}");
6724                            secondary_error = Some(control_error_frame(
6725                                &frame,
6726                                "invalid_control_body",
6727                                message.clone(),
6728                            )?);
6729                            RouteBindRelayOutcome::ModuleGone(message)
6730                        }
6731                    }
6732                }
6733                Ok(probe) => {
6734                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6735                    {
6736                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6737                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6738                            "malformed {} response body: {err}",
6739                            probe.op
6740                        )),
6741                    };
6742                    let completion = self
6743                        .forwarding
6744                        .complete_module_control_rpc(
6745                            connection_id,
6746                            frame.header.corr,
6747                            Some(&probe.op),
6748                            outcome,
6749                        )
6750                        .map_err(RouterError::Forwarding)?;
6751                    if !self.observe_module_control_completion(completion) {
6752                        debug!(
6753                            connection_id = connection_id.get(),
6754                            corr = frame.header.corr,
6755                            op = %probe.op,
6756                            "dropping late or unknown module-control RPC response"
6757                        );
6758                    }
6759                    return Ok(Vec::new());
6760                }
6761                Err(err) => {
6762                    if let Some(expected_op) = self
6763                        .forwarding
6764                        .pending_module_control_op(connection_id, frame.header.corr)
6765                        .map_err(RouterError::Forwarding)?
6766                    {
6767                        let completion = self
6768                            .forwarding
6769                            .complete_module_control_rpc(
6770                                connection_id,
6771                                frame.header.corr,
6772                                None,
6773                                ModuleControlRpcOutcome::MalformedResponse(format!(
6774                                    "malformed {expected_op} response body: {err}"
6775                                )),
6776                            )
6777                            .map_err(RouterError::Forwarding)?;
6778                        if !self.observe_module_control_completion(completion) {
6779                            debug!(
6780                                connection_id = connection_id.get(),
6781                                corr = frame.header.corr,
6782                                "dropping late malformed module-control RPC response"
6783                            );
6784                        }
6785                        return Ok(Vec::new());
6786                    }
6787                    let message = format!("malformed route.bind response body: {err}");
6788                    secondary_error = Some(control_error_frame(
6789                        &frame,
6790                        "invalid_control_body",
6791                        message.clone(),
6792                    )?);
6793                    RouteBindRelayOutcome::ModuleGone(message)
6794                }
6795            },
6796            FrameType::Error => {
6797                if self
6798                    .forwarding
6799                    .pending_module_control_op(connection_id, frame.header.corr)
6800                    .map_err(RouterError::Forwarding)?
6801                    .is_some()
6802                {
6803                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6804                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6805                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6806                            "malformed module-control ERROR body: {err}"
6807                        )),
6808                    };
6809                    let completion = self
6810                        .forwarding
6811                        .complete_module_control_rpc(
6812                            connection_id,
6813                            frame.header.corr,
6814                            None,
6815                            outcome,
6816                        )
6817                        .map_err(RouterError::Forwarding)?;
6818                    if !self.observe_module_control_completion(completion) {
6819                        debug!(
6820                            connection_id = connection_id.get(),
6821                            corr = frame.header.corr,
6822                            "dropping late or unknown module-control RPC error"
6823                        );
6824                    }
6825                    return Ok(Vec::new());
6826                }
6827                match serde_json::from_slice::<ErrorBody>(&frame.body) {
6828                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
6829                    Err(err) => {
6830                        let message = format!("malformed route.bind ERROR body: {err}");
6831                        secondary_error = Some(control_error_frame(
6832                            &frame,
6833                            "invalid_control_body",
6834                            message.clone(),
6835                        )?);
6836                        RouteBindRelayOutcome::ModuleGone(message)
6837                    }
6838                }
6839            }
6840            ty => {
6841                return Ok(vec![control_error_frame(
6842                    &frame,
6843                    "unsupported_control_frame",
6844                    format!("unsupported module channel-0 frame {ty:?}"),
6845                )?])
6846            }
6847        };
6848
6849        let settled =
6850            self.forwarding
6851                .complete_pending_relay(connection_id, frame.header.corr, outcome);
6852        let completion = match settled {
6853            Ok(completion) => completion,
6854            Err(err) => {
6855                self.refuse_to_end_module_connection_for_a_client(
6856                    connection_id,
6857                    frame.header.corr,
6858                    err,
6859                )?;
6860                return Ok(secondary_error.into_iter().collect());
6861            }
6862        };
6863        if let Some(target) = completion.abandoned.as_ref() {
6864            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6865        }
6866        if !completion.settled {
6867            debug!(
6868                connection_id = connection_id.get(),
6869                corr = frame.header.corr,
6870                frame_type = ?frame.header.ty,
6871                "dropping late or unknown route.bind relay response"
6872            );
6873        }
6874        Ok(secondary_error.into_iter().collect())
6875    }
6876
6877    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6878        debug!(connection_id = connection_id.get(), "handling GOODBYE");
6879        // GOODBYE ends the connection's logical session even when its socket
6880        // stays open. Use disconnect teardown so verdicts, client notices and
6881        // scope authority are released at the same lifecycle boundary.
6882        self.cleanup_connection_with_end_reason(
6883            connection_id,
6884            RegistrationEndReason::ExplicitGoodbye,
6885        )
6886        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6887        Ok(Vec::new())
6888    }
6889}
6890
6891impl Default for ControlHandler {
6892    fn default() -> Self {
6893        Self::new(Arc::new(Registry::default()))
6894    }
6895}
6896
6897impl crate::supervise::SwapPromotionObserver for ControlHandler {
6898    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6899        self.apply_registration_capabilities(registration);
6900    }
6901}
6902
6903fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6904    CapabilityRequirementStatus {
6905        consumer: status.consumer,
6906        capability: status.capability,
6907        need: match status.need {
6908            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6909            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6910        },
6911        verdict: status.verdict.as_str().to_string(),
6912        episode_seq: status.episode_seq,
6913        config_satisfiable: status.config_satisfiable,
6914        runtime_available: status.runtime_available,
6915        detail: status.detail,
6916    }
6917}
6918
6919fn append_capability_problem_detail(
6920    detail: Option<String>,
6921    capability_detail: Option<String>,
6922) -> Option<String> {
6923    match (detail, capability_detail) {
6924        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6925        (Some(detail), None) => Some(detail),
6926        (None, Some(capability_detail)) => Some(capability_detail),
6927        (None, None) => None,
6928    }
6929}
6930
6931fn subc_ops() -> Vec<String> {
6932    SUBC_CONTROL_OPS
6933        .iter()
6934        .map(|op| (*op).to_string())
6935        .collect()
6936}
6937
6938fn module_subc_ops() -> Vec<String> {
6939    SUBC_CONTROL_OPS
6940        .iter()
6941        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6942        .map(|op| (*op).to_string())
6943        .collect()
6944}
6945
6946#[cfg(test)]
6947fn module_baseline_control_ops() -> Vec<String> {
6948    MODULE_BASELINE_CONTROL_OPS
6949        .iter()
6950        .map(|op| (*op).to_string())
6951        .collect()
6952}
6953
6954fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6955    let mut seen = HashSet::new();
6956    let mut effective = Vec::new();
6957    for op in MODULE_BASELINE_CONTROL_OPS {
6958        if seen.insert((*op).to_string()) {
6959            effective.push((*op).to_string());
6960        }
6961    }
6962    for op in declared.unwrap_or_default() {
6963        if seen.insert(op.clone()) {
6964            effective.push(op);
6965        }
6966    }
6967    effective
6968}
6969
6970fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6971    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6972}
6973
6974fn target_module_id(target: &RouteTarget) -> &str {
6975    match target {
6976        RouteTarget::ToolProvider { module_id }
6977        | RouteTarget::ManagementSurface { module_id }
6978        | RouteTarget::InternalService { module_id, .. } => module_id,
6979    }
6980}
6981
6982fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6983    roles.iter().any(|role| match (target, role) {
6984        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6985        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6986        (
6987            RouteTarget::InternalService { service_id, .. },
6988            ProviderRole::InternalService {
6989                service_id: provided,
6990                ..
6991            },
6992        ) => service_id == provided,
6993        _ => false,
6994    })
6995}
6996
6997fn is_routable_role(role: &ProviderRole) -> bool {
6998    matches!(
6999        role,
7000        ProviderRole::ToolProvider { .. }
7001            | ProviderRole::ManagementSurface { .. }
7002            | ProviderRole::InternalService { .. }
7003    )
7004}
7005
7006#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7007enum ControlRequestBodyError {
7008    UnknownOp,
7009    InvalidBody,
7010}
7011
7012#[derive(Debug, Deserialize)]
7013struct ControlOpProbe {
7014    op: String,
7015}
7016
7017/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7018/// this set is treated as a forward-compat unknown and ignored rather than errored.
7019const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7020
7021fn is_known_module_push_op(body: &[u8]) -> bool {
7022    serde_json::from_slice::<ControlOpProbe>(body)
7023        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7024        .unwrap_or(false)
7025}
7026
7027fn is_known_module_request_op(body: &[u8]) -> bool {
7028    serde_json::from_slice::<ControlOpProbe>(body)
7029        .map(|probe| is_module_to_subc_op(&probe.op))
7030        .unwrap_or(false)
7031}
7032
7033fn is_module_to_subc_op(op: &str) -> bool {
7034    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7035}
7036
7037fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7038    debug!(
7039        op = %op,
7040        connection_id = connection_id.get(),
7041        corr,
7042        "control dispatch"
7043    );
7044}
7045
7046fn log_slow_control_dispatch(
7047    dispatch_started_at: Option<StdInstant>,
7048    op: &'static str,
7049    connection_id: ConnectionId,
7050    corr: u64,
7051    route_open_timings: Option<&RouteOpenTimings>,
7052) {
7053    let Some(dispatch_started_at) = dispatch_started_at else {
7054        return;
7055    };
7056    // Use the same monotonic clock as the phase timers and relay deadline.
7057    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7058    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7059        if let Some(timings) = route_open_timings {
7060            warn!(
7061                op = %op,
7062                connection_id = connection_id.get(),
7063                corr,
7064                elapsed_ms = elapsed.as_millis() as u64,
7065                module_id = ?timings.module_id,
7066                principal = %timings.principal,
7067                admission_ms = timings.admission.as_millis() as u64,
7068                project_root_ms = timings.project_root.as_millis() as u64,
7069                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7070                bind_send_ms = timings.bind_send.as_millis() as u64,
7071                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7072                "slow control dispatch"
7073            );
7074            return;
7075        }
7076        warn!(
7077            op = %op,
7078            connection_id = connection_id.get(),
7079            corr,
7080            elapsed_ms = elapsed.as_millis() as u64,
7081            "slow control dispatch"
7082        );
7083    }
7084}
7085
7086/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7087#[cfg(test)]
7088type TestCallerLookup =
7089    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7090
7091/// Every new client operation needs an explicit privilege decision here.
7092fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7093    match request {
7094        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7095            Some(PrivilegedOperation::Restart { module_id })
7096        }
7097        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7098            Some(PrivilegedOperation::Swap { module_id })
7099        }
7100        ClientControlRequest::SupervisorReload { module_id } => {
7101            Some(PrivilegedOperation::Reload { module_id })
7102        }
7103        ClientControlRequest::SupervisorRescan { preview } => {
7104            if *preview {
7105                None
7106            } else {
7107                Some(PrivilegedOperation::Rescan)
7108            }
7109        }
7110        // Only the well-formed apply asks for approval. A preview changes
7111        // nothing, and any other combination is refused before it touches
7112        // anything, so prompting the operator for it would ask about a request
7113        // that cannot run.
7114        ClientControlRequest::SupervisorRescanChecked {
7115            preview,
7116            candidate,
7117            expect,
7118        } => (!*preview && candidate.is_none() && expect.is_some())
7119            .then_some(PrivilegedOperation::Rescan),
7120        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7121            Some(PrivilegedOperation::SetEnabled {
7122                module_id,
7123                enabled: *enabled,
7124            })
7125        }
7126        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7127            Some(PrivilegedOperation::ReleaseReserved { module_id })
7128        }
7129        ClientControlRequest::ServerDescribe {}
7130        | ClientControlRequest::CatalogList { .. }
7131        | ClientControlRequest::RouteOpen { .. }
7132        | ClientControlRequest::RoutePoll { .. }
7133        | ClientControlRequest::SupervisorList {}
7134        | ClientControlRequest::SupervisorHealth {}
7135        | ClientControlRequest::SupervisorHealthProbe { .. }
7136        | ClientControlRequest::SupervisorRoutes { .. }
7137        | ClientControlRequest::SupervisorProvenance { .. }
7138        | ClientControlRequest::SupervisorStderrTail { .. }
7139        | ClientControlRequest::SupervisorTerminals { .. }
7140        | ClientControlRequest::SupervisorSpawnSnapshot {}
7141        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7142    }
7143}
7144
7145fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7146    match request {
7147        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7148        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7149        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7150        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7151        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7152        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7153        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7154        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7155        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7156        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7157        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7158        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7159        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7160        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7161        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7162        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7163        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7164        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7165        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7166        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7167    }
7168}
7169
7170/// The `scope=` field of the `route.open accepted` line, from the stamp that
7171/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7172/// so a ref holding control characters cannot forge a log line, and cut to 64
7173/// characters with a trailing `…` to bound the line. The bind proof and the
7174/// scope's attributes never appear.
7175fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7176    use subc_protocol::scope::ScopeKind;
7177    const MAX_REF_CHARS: usize = 64;
7178    let kind = match stamp.kind {
7179        ScopeKind::Head => "head",
7180        ScopeKind::Worker => "worker",
7181        ScopeKind::Ephemeral => "ephemeral",
7182    };
7183    let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7184    if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7185        scope_ref.push('…');
7186    }
7187    format!(
7188        "{kind}/{}/{scope_ref:?}@{}",
7189        principal_label(&stamp.owner),
7190        stamp.scope_epoch
7191    )
7192}
7193
7194fn principal_label(principal: &Principal) -> String {
7195    match principal {
7196        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7197        Principal::Direct => "direct".to_string(),
7198        other => format!("{other:?}"),
7199    }
7200}
7201
7202fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7203    match request {
7204        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7205        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7206        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7207        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7208        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7209    }
7210}
7211
7212fn parse_client_control_request(
7213    body: &[u8],
7214) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7215    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7216        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7217            Ok(probe)
7218                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7219                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7220            {
7221                ControlRequestBodyError::InvalidBody
7222            }
7223            Ok(_) => ControlRequestBodyError::UnknownOp,
7224            Err(_) => ControlRequestBodyError::InvalidBody,
7225        };
7226        (err, classification)
7227    })
7228}
7229
7230fn parse_module_control_request_from_module(
7231    body: &[u8],
7232) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7233    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7234        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7235            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7236            Ok(_) => ControlRequestBodyError::UnknownOp,
7237            Err(_) => ControlRequestBodyError::InvalidBody,
7238        };
7239        (err, classification)
7240    })
7241}
7242
7243#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7244enum ProviderRoleKind {
7245    ToolProvider,
7246    PipelineStage,
7247    ManagementSurface,
7248    InternalService,
7249}
7250
7251fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7252    match role {
7253        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7254        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7255        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7256        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7257    }
7258}
7259
7260fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7261    roles.iter().map(provider_role_kind).collect()
7262}
7263
7264/// Most refused scope records named individually in the log per sync; the
7265/// `refused` count on the accepted line is always complete.
7266const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7267
7268/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7269#[derive(Debug, Default, PartialEq, Eq)]
7270struct ScopeOutcomeCounts {
7271    created: usize,
7272    replaced: usize,
7273    updated: usize,
7274    unchanged: usize,
7275    refused: usize,
7276}
7277
7278impl ScopeOutcomeCounts {
7279    fn of(results: &[ScopeRecordResult]) -> Self {
7280        let mut counts = Self::default();
7281        for result in results {
7282            let slot = match result.outcome {
7283                ScopeRecordOutcome::Created => &mut counts.created,
7284                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7285                ScopeRecordOutcome::Updated => &mut counts.updated,
7286                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7287                ScopeRecordOutcome::Refused => &mut counts.refused,
7288            };
7289            *slot += 1;
7290        }
7291        counts
7292    }
7293}
7294
7295#[cfg(test)]
7296mod scope_outcome_count_tests {
7297    use super::*;
7298
7299    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7300        ScopeRecordResult {
7301            scope_ref: "r".to_string(),
7302            scope_epoch: 1,
7303            outcome,
7304            code: None,
7305            message: None,
7306            version: None,
7307            parent_state: None,
7308        }
7309    }
7310
7311    /// Each outcome lands in its own count, so a refused record can never be
7312    /// hidden inside the total the log already printed.
7313    #[test]
7314    fn every_outcome_is_counted_in_its_own_field() {
7315        let results = [
7316            result(ScopeRecordOutcome::Created),
7317            result(ScopeRecordOutcome::Created),
7318            result(ScopeRecordOutcome::Replaced),
7319            result(ScopeRecordOutcome::Updated),
7320            result(ScopeRecordOutcome::Unchanged),
7321            result(ScopeRecordOutcome::Refused),
7322            result(ScopeRecordOutcome::Refused),
7323            result(ScopeRecordOutcome::Refused),
7324        ];
7325        assert_eq!(
7326            ScopeOutcomeCounts::of(&results),
7327            ScopeOutcomeCounts {
7328                created: 2,
7329                replaced: 1,
7330                updated: 1,
7331                unchanged: 1,
7332                refused: 3,
7333            }
7334        );
7335    }
7336}
7337
7338/// Return whether a catalog change can create a newly violating live route.
7339/// Removing an attested claim is intentionally excluded: it makes fewer routes
7340/// forbidden and therefore must leave the existing route census untouched.
7341fn capability_census_trigger(
7342    old: Option<&CapabilityDeclarations>,
7343    new: Option<&CapabilityDeclarations>,
7344) -> bool {
7345    let old_provides = old
7346        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7347        .unwrap_or_default();
7348    let old_denies = old
7349        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7350        .unwrap_or_default();
7351    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7352        provides: Vec::new(),
7353        requires: Vec::new(),
7354        must_never_reach: Vec::new(),
7355    });
7356
7357    new.provides
7358        .iter()
7359        .any(|capability| !old_provides.contains(capability))
7360        || new
7361            .must_never_reach
7362            .iter()
7363            .any(|capability| !old_denies.contains(capability))
7364}
7365
7366/// Find the first capability an attested opener denies that an attested target
7367/// claims. Both manifests are live registry records, never cached or client data.
7368fn denied_capability<'a>(
7369    opening_manifest: &'a ModuleManifest,
7370    target_manifest: &ModuleManifest,
7371) -> Option<&'a str> {
7372    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7373    let target_capabilities = target_manifest.capabilities.as_ref()?;
7374    opening_capabilities
7375        .must_never_reach
7376        .iter()
7377        .find(|denied| {
7378            target_capabilities
7379                .provides
7380                .iter()
7381                .any(|provided| provided == *denied)
7382        })
7383        .map(String::as_str)
7384}
7385
7386fn catalog_update_frozen_field_message(
7387    registered: &ModuleManifest,
7388    provides: &[ProviderRole],
7389) -> Option<String> {
7390    let old_has_provides = !registered.provides.is_empty();
7391    let new_has_provides = !provides.is_empty();
7392    if old_has_provides != new_has_provides {
7393        return Some(format!(
7394            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7395            registered.module_id
7396        ));
7397    }
7398
7399    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7400        return Some(format!(
7401            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7402            registered.module_id
7403        ));
7404    }
7405
7406    let registered_concurrency = manifest_concurrency(registered);
7407    let mut candidate = registered.clone();
7408    candidate.provides = provides.to_vec();
7409    let candidate_concurrency = manifest_concurrency(&candidate);
7410    if candidate_concurrency != registered_concurrency {
7411        return Some(format!(
7412            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7413            registered.module_id, registered_concurrency, candidate_concurrency
7414        ));
7415    }
7416
7417    // control_ops live beside the manifest in the HELLO body, not inside
7418    // ModuleManifest, so a provides-only catalog.update cannot change them.
7419    None
7420}
7421
7422fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7423    manifest.provides.iter().any(is_routable_role)
7424}
7425
7426/// Returns the routable-provider concurrency subc should enforce for this manifest.
7427///
7428/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7429/// InternalService has no role-specific concurrency field, so it retains the
7430/// existing ModuleManaged default for backward compatibility.
7431fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7432    manifest
7433        .provides
7434        .iter()
7435        .find_map(|provider| match provider {
7436            ProviderRole::ToolProvider { concurrency, .. }
7437            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7438            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7439        })
7440        .unwrap_or(Concurrency::ModuleManaged)
7441}
7442
7443/// True when the manifest carries a ManagementSurface role whose concurrency
7444/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7445/// bytes because the typed manifest deliberately erases that distinction: the
7446/// default exists for wire compatibility, and this probe exists so the default
7447/// stays observable. Any parse irregularity returns false -- the caller only
7448/// logs, and a malformed body already failed registration upstream.
7449fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7450    let has_management_surface = manifest
7451        .provides
7452        .iter()
7453        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7454    if !has_management_surface {
7455        return false;
7456    }
7457    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7458        return false;
7459    };
7460    let Some(provides) = raw
7461        .get("manifest")
7462        .and_then(|manifest| manifest.get("provides"))
7463        .and_then(serde_json::Value::as_array)
7464    else {
7465        return false;
7466    };
7467    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7468    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7469    // against the management_surface_manifest_without_concurrency golden, not
7470    // recalled (the externally-tagged guess was this function's first bug).
7471    provides.iter().any(|role| {
7472        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7473            && role.get("concurrency").is_none()
7474    })
7475}
7476
7477fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7478    if peer_version != PROTOCOL_VERSION {
7479        return Err(format!(
7480            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7481        ));
7482    }
7483    Ok(PROTOCOL_VERSION)
7484}
7485
7486fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7487    Frame::build_with_version(
7488        response_version(frame),
7489        FrameType::Pong,
7490        frame.header.flags,
7491        0,
7492        0,
7493        frame.header.corr,
7494        Vec::new(),
7495    )
7496    .map_err(RouterError::FrameBuild)
7497}
7498
7499pub(crate) fn control_error_frame(
7500    frame: &Frame,
7501    code: &'static str,
7502    message: impl Into<String>,
7503) -> Result<Frame, RouterError> {
7504    control_error_body_frame(
7505        frame,
7506        ErrorBody {
7507            code: code.to_string(),
7508            message: message.into(),
7509            detail: None,
7510        },
7511    )
7512}
7513
7514fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7515    let body = serde_json::to_vec(&error).map_err(|err| {
7516        RouterError::backend(
7517            0,
7518            frame.header.corr,
7519            format!("failed to encode control ERROR: {err}"),
7520        )
7521    })?;
7522
7523    Frame::build_with_version(
7524        response_version(frame),
7525        FrameType::Error,
7526        control_flags(),
7527        0,
7528        0,
7529        frame.header.corr,
7530        body,
7531    )
7532    .map_err(RouterError::FrameBuild)
7533}
7534
7535fn control_response_body_frame<T: Serialize>(
7536    frame: &Frame,
7537    reply: &T,
7538    label: &'static str,
7539) -> Result<Frame, RouterError> {
7540    let body = serde_json::to_vec(reply).map_err(|err| {
7541        RouterError::backend(
7542            0,
7543            frame.header.corr,
7544            format!("failed to encode {label}: {err}"),
7545        )
7546    })?;
7547
7548    Frame::build_with_version(
7549        response_version(frame),
7550        FrameType::Response,
7551        control_flags(),
7552        0,
7553        0,
7554        frame.header.corr,
7555        body,
7556    )
7557    .map_err(RouterError::FrameBuild)
7558}
7559
7560/// Map a forwarding failure to the wire code a client sees.
7561///
7562/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7563/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7564/// chosen here decides whether a caller retries or gives up.
7565///
7566/// That makes attribution the load-bearing property, not merely having a code. A
7567/// permanent fault published as a retryable one produces a fleet-wide retry storm
7568/// against something that can never recover; a transient fault published as
7569/// permanent gives up on work that would have succeeded. Both look correct in a
7570/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7571/// the mapping per variant rather than merely asserting that some code exists.
7572///
7573/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7574/// two codes on the same side of the boundary passes it. Measured rather than
7575/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7576/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7577/// a test named for something else that happens to assert the string.
7578///
7579/// That accidental coverage is deliberately left alone rather than promoted to a
7580/// named test, because it guards a property this function does not promise.
7581/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7582/// specific code within a class, so identity is free to change and only the
7583/// partition is a contract. Splitting it out would assert a guarantee nothing
7584/// depends on — and a suite that promises more than the code does is the harder
7585/// thing to correct later, because the next reader cannot tell which assertions
7586/// are load-bearing.
7587///
7588/// Pin identity here the moment a consumer branches on a specific code.
7589fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7590    match err {
7591        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7592        ForwardingError::NoModuleConnection => "target_unavailable",
7593        ForwardingError::ModuleReloading { .. } => "module_reloading",
7594        ForwardingError::ClientRouteChannelExhausted { .. }
7595        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7596        ForwardingError::StaleModuleEndpoint
7597        | ForwardingError::UnknownReservation { .. }
7598        | ForwardingError::ConnectionClosing { .. }
7599        | ForwardingError::ClientEgressClosed { .. }
7600        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7601        // Only a swap candidate's registration can produce this, and it means
7602        // exactly what a second active HELLO for a live id means.
7603        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7604        ForwardingError::RelayCorrelationExhausted
7605        | ForwardingError::RouteOpenBuild(_)
7606        | ForwardingError::Poisoned => "forwarding_error",
7607    }
7608}
7609
7610fn response_version(frame: &Frame) -> u8 {
7611    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7612        frame.header.ver
7613    } else {
7614        PROTOCOL_VERSION
7615    }
7616}
7617
7618fn control_flags() -> Flags {
7619    Flags::new(false, Priority::Passive, false)
7620}
7621
7622/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7623/// channel. The target is the module (a client never saw the route), so this
7624/// takes the module path: delivered late rather than dropped when the module's
7625/// queue is momentarily full, and never closing its connection.
7626fn send_goodbye_target_best_effort(
7627    counters: &DaemonCounters,
7628    target: &GoodbyeTarget,
7629    context: &'static str,
7630) {
7631    let Ok(frame) = Frame::build_with_version(
7632        target.negotiated_ver,
7633        FrameType::Goodbye,
7634        control_flags(),
7635        target.channel,
7636        target.epoch,
7637        0,
7638        Vec::new(),
7639    ) else {
7640        return;
7641    };
7642    crate::forwarding::send_module_route_goodbye(
7643        counters,
7644        &target.sink,
7645        frame,
7646        target.module_id.as_deref(),
7647        context,
7648    );
7649}
7650
7651pub(crate) fn send_route_control_pushes(
7652    forwarding: &ForwardingTable,
7653    routes: Vec<EndpointRoute>,
7654    push: ClientControlPush,
7655) {
7656    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7657    for route in routes {
7658        let target = route.goodbye_target;
7659        if let Some((existing, channels)) = targets
7660            .iter_mut()
7661            .find(|(existing, _)| existing.connection_id == target.connection_id)
7662        {
7663            debug_assert_eq!(
7664                existing.negotiated_ver, target.negotiated_ver,
7665                "one connection cannot negotiate multiple frame versions"
7666            );
7667            if !channels.contains(&target.channel) {
7668                channels.push(target.channel);
7669            }
7670            continue;
7671        }
7672        let channel = target.channel;
7673        targets.push((target, vec![channel]));
7674    }
7675    for (target, mut channels) in targets {
7676        channels.sort_unstable();
7677        let mut push = push.clone();
7678        match &mut push {
7679            ClientControlPush::RouteClosing {
7680                channels: covered, ..
7681            }
7682            | ClientControlPush::RouteClosed {
7683                channels: covered, ..
7684            } => *covered = channels,
7685        }
7686        let body = match serde_json::to_vec(&push) {
7687            Ok(body) => body,
7688            Err(err) => {
7689                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7690                continue;
7691            }
7692        };
7693        let frame = match Frame::build_with_version(
7694            target.negotiated_ver,
7695            FrameType::Push,
7696            control_flags(),
7697            0,
7698            0,
7699            0,
7700            body.clone(),
7701        ) {
7702            Ok(frame) => frame,
7703            Err(err) => {
7704                warn!(
7705                    route_channel = target.channel,
7706                    error = %err,
7707                    "failed to build route lifecycle control PUSH frame"
7708                );
7709                continue;
7710            }
7711        };
7712        if let Err(err) = target.sink.try_send(frame) {
7713            if target.close_on_delivery_failure() {
7714                warn!(
7715                    target_connection_id = target.connection_id.get(),
7716                    route_channel = target.channel,
7717                    error = %err,
7718                    "route lifecycle control PUSH was not delivered to client; closing target connection"
7719                );
7720                let _ = forwarding.escalate_client_delivery_failure(
7721                    target.connection_id,
7722                    target.channel,
7723                    target.epoch,
7724                    CloseReason::new(
7725                        "route_lifecycle_push_delivery_failed",
7726                        format!(
7727                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7728                            target.channel
7729                        ),
7730                    ),
7731                    crate::forwarding::UndeliveredFrame {
7732                        module_id: target.module_id.as_deref(),
7733                        sink: &target.sink,
7734                    },
7735                );
7736            }
7737        }
7738    }
7739}
7740
7741#[cfg(test)]
7742mod tests {
7743    mod activity_read {
7744        include!("activity_read_tests.rs");
7745    }
7746
7747    mod lifecycle_activity {
7748        include!("control_lifecycle_tests.rs");
7749    }
7750    #[cfg(unix)]
7751    #[tokio::test]
7752    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7753        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7754        let path = dir.join("subc.jsonc");
7755        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7756            "program":"/bin/sleep","args":["60"],"protocol":"none",
7757            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7758        }}}).to_string()).unwrap();
7759        let mut configured = crate::daemon_config::load(&path)
7760            .unwrap()
7761            .unwrap()
7762            .modules
7763            .pop()
7764            .unwrap();
7765        let registry = std::sync::Arc::new(crate::Registry::default());
7766        let handle = crate::SupervisorHandle::new();
7767        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7768            .with_handle(handle.clone());
7769        let module = supervisor
7770            .supervise_configured_with_health(
7771                configured.module_spec(),
7772                true,
7773                configured.health.clone(),
7774                None,
7775                configured.restart,
7776            )
7777            .unwrap();
7778        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7779        let before = module.status().unwrap().pid;
7780        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7781        configured.health.cadence = std::time::Duration::from_secs(3600);
7782        let health_only = handler
7783            .reconcile_supervised_modules(
7784                &supervisor,
7785                vec![configured.clone()],
7786                false,
7787                Principal::Direct,
7788            )
7789            .await
7790            .unwrap();
7791        assert!(
7792            health_only.0.changed_pending_reload.is_empty(),
7793            "health policy is already applied live"
7794        );
7795        assert_eq!(module.status().unwrap().pid, before);
7796        assert_eq!(
7797            module.configuration().unwrap().1.http,
7798            configured.health.http
7799        );
7800        configured.args = vec!["61".into()];
7801        let launch = handler
7802            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7803            .await
7804            .unwrap();
7805        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
7806        assert_eq!(
7807            module.status().unwrap().pid,
7808            before,
7809            "a launch edit is stored until reload"
7810        );
7811        module.drain().await.unwrap();
7812    }
7813    use cortexkit_test_support::ScratchDir;
7814    use std::{
7815        collections::BTreeMap,
7816        fmt,
7817        path::PathBuf,
7818        sync::{Arc, Mutex},
7819        time::Duration,
7820    };
7821
7822    use serde_json::{json, Value};
7823    use subc_protocol::{
7824        manifest::{
7825            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7826            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7827        },
7828        session::HealthStatus,
7829        FrameType,
7830    };
7831
7832    use super::*;
7833    use crate::{
7834        forwarding::{DataRoute, DataRouteState},
7835        registry::ChannelState,
7836        router::FrameSink,
7837        stderr_tail::DEFAULT_MAX_LINE_BYTES,
7838        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7839        RouteCtx, Router,
7840    };
7841    use tokio::{
7842        sync::mpsc,
7843        time::{sleep, Instant},
7844    };
7845    use tracing::{
7846        field::{Field, Visit},
7847        Event, Subscriber,
7848    };
7849    use tracing_subscriber::{layer::Context, prelude::*, Layer};
7850
7851    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
7852    ///
7853    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
7854    /// is only populated for `tests/*.rs` integration test binaries -- this file
7855    /// compiles as part of the library target, which gets neither. This test's
7856    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
7857    /// and the sibling binary lives two directories up at
7858    /// `<target-dir>/<profile>/fake-aft-stub`.
7859    ///
7860    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
7861    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
7862    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
7863    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
7864    /// `NotFound`, which reads as a broken test rather than an unbuilt
7865    /// dependency -- so state the cause and the remedy instead. Deliberately a
7866    /// panic and not a silent skip: a test that quietly passes when it could not
7867    /// run is worse than one that fails, because it reports health it never
7868    /// verified.
7869    fn fake_aft_stub_path() -> PathBuf {
7870        let mut path = std::env::current_exe().expect("current_exe available in tests");
7871        path.pop(); // .../deps/
7872        path.pop(); // .../<profile>/
7873        path.push(if cfg!(windows) {
7874            "fake-aft-stub.exe"
7875        } else {
7876            "fake-aft-stub"
7877        });
7878        assert!(
7879            path.exists(),
7880            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7881             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7882            path.display()
7883        );
7884        path
7885    }
7886
7887    /// Whether clients retry `code` in place: the predicate itself, never a copy
7888    /// of its set. A copied list breaks silently when a code is added to or
7889    /// removed from the real one, and a stale copy here would let exactly the
7890    /// failure this test exists to catch pass.
7891    fn client_retries(code: &str) -> bool {
7892        subc_protocol::error_codes::is_retryable_route_open(code)
7893    }
7894
7895    /// A code is not a label — clients branch on it, so publishing the wrong KIND
7896    /// of failure is worse than publishing none. A permanent fault dressed as
7897    /// retryable makes every client in the fleet retry forever against something
7898    /// that cannot recover; a transient fault dressed as permanent abandons work
7899    /// that would have succeeded.
7900    ///
7901    /// Asserting "a code exists" cannot catch either, because the string is free
7902    /// to say anything. This enumerates every variant and pins which side of the
7903    /// retry boundary it lands on, so a new variant must be classified here
7904    /// deliberately rather than inheriting whichever arm it was appended to.
7905    #[test]
7906    fn retryability_of_forwarding_codes_matches_the_failure() {
7907        // Transient by nature: the target is booting, reloading, or its endpoint
7908        // was swapped mid-flight. Retrying is how these resolve.
7909        let transient = [
7910            ForwardingError::NoModuleConnection,
7911            ForwardingError::ModuleReloading {
7912                module_id: "m".into(),
7913            },
7914            ForwardingError::StaleModuleEndpoint,
7915            ForwardingError::UnknownReservation {
7916                client_channel: 1,
7917                module_channel: 1,
7918            },
7919            ForwardingError::ConnectionClosing {
7920                connection_id: ConnectionId::new(1),
7921            },
7922            ForwardingError::ClientEgressClosed {
7923                connection_id: ConnectionId::new(1),
7924            },
7925            ForwardingError::ModuleEgressUnavailable {
7926                connection_id: ConnectionId::new(1),
7927            },
7928        ];
7929        for err in transient {
7930            let code = forwarding_error_code(&err);
7931            assert!(
7932                client_retries(code),
7933                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7934            );
7935        }
7936
7937        // Not fixed by retrying. Channel and correlation exhaustion need the
7938        // caller to close routes, and a poisoned lock is a daemon that cannot
7939        // recover at all — the worst thing to advertise as retryable, since every
7940        // client would storm a daemon that will never answer.
7941        let permanent = [
7942            ForwardingError::ConnectionRoleConflict {
7943                connection_id: ConnectionId::new(1),
7944            },
7945            ForwardingError::ClientRouteChannelExhausted {
7946                connection_id: ConnectionId::new(1),
7947            },
7948            ForwardingError::ModuleRouteChannelExhausted {
7949                endpoint: ModuleEndpointId {
7950                    connection_id: ConnectionId::new(1),
7951                    generation: 1,
7952                },
7953            },
7954            ForwardingError::RelayCorrelationExhausted,
7955            ForwardingError::RouteOpenBuild("x".into()),
7956            ForwardingError::Poisoned,
7957        ];
7958        for err in permanent {
7959            let code = forwarding_error_code(&err);
7960            assert!(
7961                !client_retries(code),
7962                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7963            );
7964        }
7965    }
7966
7967    /// The principal is the daemon's answer to "who is calling", and modules
7968    /// branch on it: aft gates bash on it, cerebellum gates browser control,
7969    /// plexus gates connector invocation. So a stamp is an authorization input in
7970    /// another process, not a label — and both possible answers SUCCEED, which is
7971    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
7972    /// first-party capability to something that never proved it; a supervised one
7973    /// stamped `Direct` silently strips a module of capability it is entitled to.
7974    ///
7975    /// Neither shows up in a test that only checks the bind succeeded. Before this
7976    /// test the only coverage was accidental —
7977    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
7978    /// stamped principal on its way past, so narrowing that wire-shape test to its
7979    /// stated subject would have deleted the last assertion on this value. It
7980    /// still asserts the stamp, which is now redundancy rather than the only
7981    /// guard: both fail under the same mutation, and this one names the reason.
7982    /// SCOPE: this handler's supervisor has spawned nothing, so
7983    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
7984    /// is unreachable here. Both assertions below are refusals, and a mutant that
7985    /// refuses everything would satisfy them.
7986    ///
7987    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
7988    /// spawns a supervised consumer, reads its live nonce, and asserts the module
7989    /// observed `principal.kind == "reserved"` carrying that module_id — verified
7990    /// at source rather than assumed, since a citation is a claim about another
7991    /// file and ages like one. Recorded because a harness that structurally
7992    /// cannot reach an arm reports "none" for that arm identically to one that
7993    /// covers it and found nothing.
7994    #[tokio::test]
7995    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7996        let handler = ControlHandler::default();
7997        let frame =
7998            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7999
8000        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8001        // any process holding the connection file. Nothing was proved, so nothing
8002        // may be granted beyond the unattested floor.
8003        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8004        assert_eq!(
8005            stamped,
8006            Principal::Direct,
8007            "a caller that proved nothing must not be stamped as a supervised module"
8008        );
8009
8010        // A claimed module_id with a nonce no supervised child was given is a
8011        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8012        // quietly demoted to Direct, or an impersonation attempt looks identical
8013        // to an ordinary unattested connection.
8014        let forged = handler
8015            .route_open_principal(
8016                &frame,
8017                Some(ConsumerIdentity {
8018                    module_id: "aft".to_string(),
8019                    launch_nonce: "not-a-real-nonce".to_string(),
8020                }),
8021            )
8022            .unwrap();
8023        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8024        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8025    }
8026
8027    /// The test above hands `route_open_principal` an identity it built itself,
8028    /// which proves the stamping rule and nothing about where the identity comes
8029    /// from. The real producer is a wire body, and the two are joined by a serde
8030    /// field name that nothing else asserts.
8031    ///
8032    /// That join fails quietly in one specific way: an unrecognised key is simply
8033    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8034    /// yields `None` and every supervised module silently drops to `Direct`.
8035    /// Capability-wise that is the safe direction, but it surfaces far from its
8036    /// cause — as a module mysteriously refused bash — and it would pass every
8037    /// test that builds its own input.
8038    ///
8039    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8040    /// would break every client the moment the daemon gains a field, trading a
8041    /// quiet demotion for a hard refusal on additive change. Asserting the join
8042    /// instead means a rename breaks a test here rather than the fleet.
8043    #[test]
8044    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8045        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8046        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8047        let ClientControlRequest::RouteOpen {
8048            consumer_identity, ..
8049        } = parsed
8050        else {
8051            panic!("route.open body must parse as RouteOpen");
8052        };
8053        assert_eq!(
8054            consumer_identity,
8055            Some(ConsumerIdentity {
8056                module_id: "aft".to_string(),
8057                launch_nonce: "n".to_string(),
8058            }),
8059            "the wire field name must reach the value route_open_principal reads"
8060        );
8061    }
8062
8063    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8064        ModuleManifest::builder(module_id, "0.1.0")
8065            .protocol_ver(protocol_ver)
8066            .provides(vec![ProviderRole::ToolProvider {
8067                tools: vec![Tool {
8068                    name: "read".to_string(),
8069                    description: None,
8070                    execution_mode: ExecutionMode::Pure,
8071                    schema: json!({"type": "object"}),
8072                }],
8073                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8074                concurrency: Concurrency::ModuleManaged,
8075                emits_push: true,
8076                sub_supervises: true,
8077            }])
8078            .build()
8079    }
8080
8081    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8082        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8083    }
8084
8085    fn hello_frame_with_control_ops(
8086        module_id: &str,
8087        protocol_ver: u8,
8088        corr: u64,
8089        control_ops: Option<Vec<String>>,
8090    ) -> Frame {
8091        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8092    }
8093
8094    fn hello_frame_with_nonce(
8095        module_id: &str,
8096        protocol_ver: u8,
8097        corr: u64,
8098        launch_nonce: Option<&str>,
8099    ) -> Frame {
8100        hello_frame_full(
8101            module_id,
8102            protocol_ver,
8103            corr,
8104            None,
8105            launch_nonce.map(ToOwned::to_owned),
8106        )
8107    }
8108
8109    fn hello_frame_full(
8110        module_id: &str,
8111        protocol_ver: u8,
8112        corr: u64,
8113        control_ops: Option<Vec<String>>,
8114        launch_nonce: Option<String>,
8115    ) -> Frame {
8116        let body = serde_json::to_vec(&ModuleHelloBody {
8117            manifest: manifest(module_id, protocol_ver),
8118            protocol_ver,
8119            control_ops,
8120            launch_nonce,
8121        })
8122        .unwrap();
8123        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8124    }
8125
8126    fn non_routable_hello_frame_with_control_ops(
8127        module_id: &str,
8128        corr: u64,
8129        control_ops: Option<Vec<String>>,
8130    ) -> Frame {
8131        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8132        manifest.provides.clear();
8133        let body = serde_json::to_vec(&ModuleHelloBody {
8134            manifest,
8135            protocol_ver: PROTOCOL_VERSION,
8136            control_ops,
8137            launch_nonce: None,
8138        })
8139        .unwrap();
8140        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8141    }
8142
8143    fn capability_grammar_hello_frame(
8144        capabilities: Value,
8145        runtime_computed: Option<Value>,
8146        corr: u64,
8147    ) -> Frame {
8148        let mut body = serde_json::to_value(ModuleHelloBody {
8149            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8150            protocol_ver: PROTOCOL_VERSION,
8151            control_ops: None,
8152            launch_nonce: None,
8153        })
8154        .expect("HELLO body serializes");
8155        body["manifest"]["capabilities"] = capabilities;
8156        if let Some(runtime_computed) = runtime_computed {
8157            body["runtime_computed"] = runtime_computed;
8158        }
8159        Frame::build(
8160            FrameType::Hello,
8161            control_flags(),
8162            0,
8163            0,
8164            corr,
8165            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8166        )
8167        .expect("HELLO frame builds")
8168    }
8169
8170    fn channel_request(channel: u16, corr: u64) -> Frame {
8171        Frame::build(
8172            FrameType::Request,
8173            Flags::new(true, Priority::Interactive, false),
8174            channel,
8175            0,
8176            corr,
8177            b"opaque".to_vec(),
8178        )
8179        .unwrap()
8180    }
8181
8182    fn route_ctx(
8183        connection_id: ConnectionId,
8184    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8185        let (tx, rx) = mpsc::channel(8);
8186        (
8187            RouteCtx {
8188                connection_id,
8189                egress: FrameSink::new(tx),
8190            },
8191            rx,
8192        )
8193    }
8194
8195    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8196        serde_json::from_slice(&frame.body).unwrap()
8197    }
8198
8199    /// Register a module over a connection that has a sink and return the
8200    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8201    /// module's own sink rather than returning it as a reply, so the ack is
8202    /// taken off `rx` here and whatever the test reads next is what followed it.
8203    async fn hello_via_sink(
8204        handler: &ControlHandler,
8205        ctx: &RouteCtx,
8206        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8207        hello: Frame,
8208    ) -> Frame {
8209        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8210        assert!(
8211            replies.is_empty(),
8212            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8213        );
8214        let ack = rx
8215            .try_recv()
8216            .expect("HELLO_ACK is queued on the module sink")
8217            .frame;
8218        assert_eq!(ack.header.ty, FrameType::HelloAck);
8219        ack
8220    }
8221
8222    fn parse_error(frame: &Frame) -> Value {
8223        serde_json::from_slice(&frame.body).unwrap()
8224    }
8225
8226    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8227        serde_json::from_slice(&frame.body).unwrap()
8228    }
8229
8230    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8231        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8232            route_channel,
8233            route_epoch: 0,
8234            kind,
8235        })
8236        .unwrap();
8237        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8238    }
8239
8240    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8241        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8242            module_id: module_id.to_string(),
8243        })
8244        .unwrap();
8245        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8246    }
8247
8248    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8249        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8250    }
8251
8252    fn route_open_frame_with_consumer_capabilities(
8253        corr: u64,
8254        module_id: &str,
8255        project_root: ScratchDir,
8256        consumer_capabilities: Option<Vec<String>>,
8257    ) -> Frame {
8258        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8259            target: RouteTarget::ToolProvider {
8260                module_id: module_id.to_string(),
8261            },
8262            identity: BindIdentity::new(
8263                project_root.path().to_path_buf(),
8264                "unit".to_string(),
8265                "session".to_string(),
8266            ),
8267            consumer_identity: None,
8268            consumer_capabilities,
8269            role_versions: None,
8270            admission_facts: None,
8271            scope: None,
8272        })
8273        .unwrap();
8274        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8275    }
8276
8277    fn route_open_frame_with_role_versions(
8278        corr: u64,
8279        module_id: &str,
8280        project_root: ScratchDir,
8281        role_versions: Option<BTreeMap<String, String>>,
8282    ) -> Frame {
8283        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8284            target: RouteTarget::ToolProvider {
8285                module_id: module_id.to_string(),
8286            },
8287            identity: BindIdentity::new(
8288                project_root.path().to_path_buf(),
8289                "unit".to_string(),
8290                format!("session-{corr}"),
8291            ),
8292            consumer_identity: None,
8293            consumer_capabilities: None,
8294            role_versions,
8295            admission_facts: None,
8296            scope: None,
8297        })
8298        .unwrap();
8299        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8300    }
8301
8302    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8303        entries
8304            .iter()
8305            .map(|(role, version)| (role.to_string(), version.to_string()))
8306            .collect()
8307    }
8308
8309    fn route_open_frame_with_admission_facts(
8310        corr: u64,
8311        module_id: &str,
8312        project_root: ScratchDir,
8313        consumer_identity: Option<subc_control::ConsumerIdentity>,
8314        facts: Option<Value>,
8315    ) -> Frame {
8316        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8317            target: RouteTarget::ToolProvider {
8318                module_id: module_id.to_string(),
8319            },
8320            identity: BindIdentity::new(
8321                project_root.path().to_path_buf(),
8322                "unit".to_string(),
8323                format!("session-{corr}"),
8324            ),
8325            consumer_identity,
8326            consumer_capabilities: None,
8327            role_versions: None,
8328            admission_facts: facts,
8329            scope: None,
8330        })
8331        .unwrap();
8332        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8333    }
8334
8335    #[derive(Clone, Default)]
8336    struct EventCapture {
8337        events: Arc<Mutex<Vec<CapturedEvent>>>,
8338    }
8339
8340    #[derive(Clone, Debug)]
8341    struct CapturedEvent {
8342        target: String,
8343        level: tracing::Level,
8344        fields: BTreeMap<String, String>,
8345    }
8346
8347    impl EventCapture {
8348        fn events(&self) -> Vec<CapturedEvent> {
8349            self.events.lock().unwrap().clone()
8350        }
8351    }
8352
8353    impl<S> Layer<S> for EventCapture
8354    where
8355        S: Subscriber,
8356    {
8357        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8358            let mut visitor = EventFieldVisitor::default();
8359            event.record(&mut visitor);
8360            self.events.lock().unwrap().push(CapturedEvent {
8361                target: event.metadata().target().to_string(),
8362                level: *event.metadata().level(),
8363                fields: visitor.fields,
8364            });
8365        }
8366    }
8367
8368    #[derive(Default)]
8369    struct EventFieldVisitor {
8370        fields: BTreeMap<String, String>,
8371    }
8372
8373    impl Visit for EventFieldVisitor {
8374        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8375            self.fields
8376                .insert(field.name().to_string(), format!("{value:?}"));
8377        }
8378    }
8379
8380    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8381        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8382            status,
8383            detail: Some("warming".to_string()),
8384            metrics: Some(json!({"queue_depth": 3})),
8385        })
8386        .unwrap();
8387        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8388    }
8389
8390    fn route_bind_ack(corr: u64) -> Frame {
8391        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8392        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8393    }
8394
8395    fn unique_project_root(label: &str) -> ScratchDir {
8396        ScratchDir::new(label)
8397    }
8398
8399    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8400        match parse_route_poll(frame) {
8401            ClientControlResponse::RoutePoll {
8402                status: None,
8403                live: Some(live),
8404                ..
8405            } => assert_eq!(live, expected_live),
8406            other => panic!("unexpected route.poll response: {other:?}"),
8407        }
8408    }
8409
8410    fn bind_liveness_route(
8411        registry: &Registry,
8412        forwarding: &ForwardingTable,
8413        module_id: &str,
8414    ) -> (RouteCtx, u16, u32) {
8415        let module_connection = ConnectionId::new(101);
8416        let client_connection = ConnectionId::new(202);
8417        let registration = registry
8418            .register_with_control_ops(
8419                manifest(module_id, PROTOCOL_VERSION),
8420                PROTOCOL_VERSION,
8421                module_connection,
8422                module_baseline_control_ops(),
8423            )
8424            .unwrap();
8425        let (module_tx, _module_rx) = mpsc::channel(8);
8426        let endpoint = forwarding
8427            .register_module_connection(
8428                module_connection,
8429                module_id.to_string(),
8430                PROTOCOL_VERSION,
8431                manifest_concurrency(&registration.manifest),
8432                FrameSink::new(module_tx),
8433            )
8434            .unwrap();
8435        let (client_ctx, _client_rx) = route_ctx(client_connection);
8436        let pending = forwarding
8437            .begin_route_bind_relay_for_test(
8438                client_connection,
8439                client_ctx.egress.clone(),
8440                1,
8441                module_id,
8442            )
8443            .unwrap();
8444        assert_eq!(pending.endpoint, endpoint);
8445        let route_channel = pending.client_channel;
8446        let route_epoch = pending.client_epoch;
8447        forwarding
8448            .complete_pending_relay(
8449                module_connection,
8450                pending.corr,
8451                RouteBindRelayOutcome::Accepted,
8452            )
8453            .unwrap();
8454        (client_ctx, route_channel, route_epoch)
8455    }
8456
8457    struct FakeProcessLiveness {
8458        live: Option<bool>,
8459    }
8460
8461    impl ModuleProcessLiveness for FakeProcessLiveness {
8462        fn process_live(&self, _module_id: &str) -> Option<bool> {
8463            self.live
8464        }
8465    }
8466
8467    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8468    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8469    {
8470        let registry = Arc::new(Registry::default());
8471        let supervisor_handle = SupervisorHandle::new();
8472        let supervisor = Supervisor::new_for_test(
8473            Arc::clone(&registry),
8474            RestartPolicy::new(1, Duration::from_millis(10)),
8475        )
8476        .with_handle(supervisor_handle.clone());
8477        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8478        let module = supervisor
8479            .spawn(ModuleSpec {
8480                module_id: "stderr-tail-wire".to_string(),
8481                program: fake_aft_stub_path(),
8482                args: Vec::new(),
8483                env: vec![
8484                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8485                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8486                ],
8487                reserved: false,
8488                reserved_prefixes: Vec::new(),
8489                protocol: ModuleProtocol::Subc,
8490                overlap: Default::default(),
8491            })
8492            .unwrap();
8493
8494        let deadline = Instant::now() + Duration::from_secs(5);
8495        loop {
8496            let tail = module.stderr_tail(None, None);
8497            if tail
8498                .entries
8499                .iter()
8500                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8501                && tail.entries.iter().any(|entry| {
8502                    matches!(
8503                        entry,
8504                        TailEntry::Line {
8505                            truncated: true,
8506                            ..
8507                        }
8508                    )
8509                })
8510            {
8511                break;
8512            }
8513            assert!(
8514                Instant::now() < deadline,
8515                "module did not produce a truncated line and restart boundary: {tail:?}"
8516            );
8517            sleep(Duration::from_millis(10)).await;
8518        }
8519
8520        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8521        let request = ClientControlRequest::SupervisorStderrTail {
8522            module_id: "stderr-tail-wire".to_string(),
8523            max_lines: None,
8524            max_bytes: None,
8525        };
8526        let frame = Frame::build(
8527            FrameType::Request,
8528            control_flags(),
8529            0,
8530            0,
8531            1,
8532            serde_json::to_vec(&request).unwrap(),
8533        )
8534        .unwrap();
8535        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8536        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8537        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8538            serde_json::from_slice(&responses[0].body).unwrap()
8539        else {
8540            panic!("expected supervisor.stderr_tail response");
8541        };
8542
8543        assert!(
8544            tail.entries
8545                .iter()
8546                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8547            "the control response lost the restart boundary"
8548        );
8549        let Some(StderrTailEntry::Line {
8550            text,
8551            truncated,
8552            at_ms,
8553        }) = tail.entries.iter().find(|entry| {
8554            matches!(
8555                entry,
8556                StderrTailEntry::Line {
8557                    truncated: true,
8558                    ..
8559                }
8560            )
8561        })
8562        else {
8563            panic!("the control response lost the truncated line");
8564        };
8565        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8566        assert!(*truncated);
8567        assert!(
8568            at_ms.is_some(),
8569            "the control response lost the line's capture time"
8570        );
8571    }
8572
8573    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8574    /// read done on the worker thread would stall every other task until it
8575    /// finished; the read must run off the worker so this test's own task keeps
8576    /// running while the read is paused.
8577    #[tokio::test(flavor = "current_thread")]
8578    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8579        let dir = ScratchDir::new("terminals-off-worker");
8580        let journal_path = dir.join("terminals.jsonl");
8581        let registry = Arc::new(Registry::default());
8582        let supervisor_handle = SupervisorHandle::new();
8583        let supervisor =
8584            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8585                .with_handle(supervisor_handle.clone())
8586                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8587        let module = supervisor
8588            .spawn(ModuleSpec {
8589                module_id: "terminal-off-worker".to_string(),
8590                program: fake_aft_stub_path(),
8591                args: Vec::new(),
8592                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8593                reserved: false,
8594                reserved_prefixes: Vec::new(),
8595                protocol: ModuleProtocol::Subc,
8596                overlap: Default::default(),
8597            })
8598            .unwrap();
8599        let deadline = Instant::now() + Duration::from_secs(5);
8600        while module.terminal_history().entries.len() != 2 {
8601            assert!(Instant::now() < deadline, "module did not record two exits");
8602            sleep(Duration::from_millis(10)).await;
8603        }
8604
8605        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8606        let handler =
8607            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8608        let frame = Frame::build(
8609            FrameType::Request,
8610            control_flags(),
8611            0,
8612            0,
8613            1,
8614            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8615                module_id: "terminal-off-worker".to_string(),
8616            })
8617            .unwrap(),
8618        )
8619        .unwrap();
8620        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8621        let spawned_at = std::time::Instant::now();
8622        let read = tokio::spawn({
8623            let handler = Arc::clone(&handler);
8624            async move { handler.handle_control_frame(&ctx, frame).await }
8625        });
8626        // Waiting for the pause from a blocking thread keeps this task pending,
8627        // so the runtime's single worker is free to run the read task.
8628        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8629            .await
8630            .unwrap()
8631            .expect("the history read reached its pause");
8632        let elapsed = spawned_at.elapsed();
8633        assert!(
8634            elapsed < Duration::from_secs(2) && !read.is_finished(),
8635            "this task could not run while the history read was paused \
8636             (resumed after {elapsed:?}, read finished: {})",
8637            read.is_finished()
8638        );
8639
8640        drop(release);
8641        let responses = read.await.unwrap().unwrap();
8642        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8643        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8644            panic!("expected supervisor.terminals response");
8645        };
8646        assert_eq!(terminals.entries.len(), 2);
8647        assert_eq!(terminals.journal_skipped_lines, 0);
8648        assert_eq!(terminals.journal_read_errors, 0);
8649    }
8650
8651    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8652    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8653        let registry = Arc::new(Registry::default());
8654        let supervisor_handle = SupervisorHandle::new();
8655        let supervisor =
8656            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8657                .with_handle(supervisor_handle.clone());
8658        let module = supervisor
8659            .spawn(ModuleSpec {
8660                module_id: "terminal-golden".to_string(),
8661                program: fake_aft_stub_path(),
8662                args: Vec::new(),
8663                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8664                reserved: false,
8665                reserved_prefixes: Vec::new(),
8666                protocol: ModuleProtocol::Subc,
8667                overlap: Default::default(),
8668            })
8669            .unwrap();
8670
8671        let deadline = Instant::now() + Duration::from_secs(5);
8672        while module.terminal_history().entries.len() != 2 {
8673            assert!(
8674                Instant::now() < deadline,
8675                "module did not retain two terminal exits: {:?}",
8676                module.terminal_history()
8677            );
8678            sleep(Duration::from_millis(10)).await;
8679        }
8680
8681        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8682        let request = ClientControlRequest::SupervisorTerminals {
8683            module_id: "terminal-golden".to_string(),
8684        };
8685        let frame = Frame::build(
8686            FrameType::Request,
8687            control_flags(),
8688            0,
8689            0,
8690            1,
8691            serde_json::to_vec(&request).unwrap(),
8692        )
8693        .unwrap();
8694        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8695        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8696        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8697        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8698            panic!("expected supervisor.terminals response");
8699        };
8700        assert_eq!(terminals.entries.len(), 2);
8701        assert_eq!(terminals.dropped, 0);
8702
8703        let mut rendered = serde_json::to_value(response).unwrap();
8704        // Wall-clock fields are the observation contract, but not stable fixture
8705        // bytes; normalize only them after the real handler has shaped the response.
8706        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8707        for (index, entry) in rendered["entries"]
8708            .as_array_mut()
8709            .expect("terminal response entries array")
8710            .iter_mut()
8711            .enumerate()
8712        {
8713            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8714        }
8715
8716        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8717            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8718        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8719        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8720            std::fs::write(&golden_path, &serialized).unwrap();
8721        }
8722        let expected: Value =
8723            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8724        assert_eq!(rendered, expected);
8725    }
8726
8727    #[test]
8728    fn hello_registers_manifest_and_returns_ack() {
8729        let registry = Arc::new(Registry::default());
8730        let handler = ControlHandler::new(Arc::clone(&registry));
8731        let conn = ConnectionId::new(1);
8732
8733        let responses = handler
8734            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8735            .unwrap();
8736
8737        assert_eq!(responses.len(), 1);
8738        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8739        assert_eq!(responses[0].header.channel, 0);
8740        assert_eq!(responses[0].header.corr, 7);
8741        let ack = parse_ack(&responses[0]);
8742        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8743        assert!(ack
8744            .subc_capabilities
8745            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8746        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8747        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8748        assert!(ack
8749            .subc_ops
8750            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8751        assert!(ack
8752            .subc_ops
8753            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8754
8755        let registration = registry.get_module("aft").unwrap().unwrap();
8756        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8757        assert_eq!(registration.state, ChannelState::Active);
8758        assert_eq!(registration.connection_id, conn);
8759        assert_eq!(registration.control_ops, module_baseline_control_ops());
8760    }
8761
8762    #[test]
8763    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8764        let invalid_identifiers = [
8765            ("case_change", "credentials-Provider/v1"),
8766            ("leading_zero", "credentials-provider/v01"),
8767            ("trailing_hyphen", "credentials-provider-/v1"),
8768            ("consecutive_hyphens", "credentials--provider/v1"),
8769            ("uppercase", "Credentials-provider/v1"),
8770            ("missing_v", "credentials-provider/1"),
8771            ("whitespace", "credentials provider/v1"),
8772            ("zero_version", "credentials-provider/v0"),
8773            ("out_of_range_version", "credentials-provider/v4294967296"),
8774            (
8775                "overlength_name",
8776                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8777            ),
8778        ];
8779        let mut cases = invalid_identifiers
8780            .into_iter()
8781            .map(|(name, identifier)| {
8782                (
8783                    format!("identifier_{name}"),
8784                    "capabilities.provides[0]".to_string(),
8785                    identifier.to_string(),
8786                    json!({ "provides": [identifier] }),
8787                    None,
8788                )
8789            })
8790            .collect::<Vec<_>>();
8791        cases.extend([
8792            (
8793                "unknown_need".to_string(),
8794                "capabilities.requires[0].need".to_string(),
8795                "deferred".to_string(),
8796                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8797                None,
8798            ),
8799            (
8800                "duplicate_provides".to_string(),
8801                "capabilities.provides[1]".to_string(),
8802                "credentials-provider/v1".to_string(),
8803                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8804                None,
8805            ),
8806            (
8807                "duplicate_must_never_reach".to_string(),
8808                "capabilities.must_never_reach[1]".to_string(),
8809                "credentials-provider/v1".to_string(),
8810                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8811                None,
8812            ),
8813            (
8814                "duplicate_requires_same_need".to_string(),
8815                "capabilities.requires[1]".to_string(),
8816                "credentials-provider/v1".to_string(),
8817                json!({ "requires": [
8818                    { "capability": "credentials-provider/v1", "need": "required" },
8819                    { "capability": "credentials-provider/v1", "need": "required" }
8820                ] }),
8821                None,
8822            ),
8823            (
8824                "duplicate_requires_conflicting_need".to_string(),
8825                "capabilities.requires[1]".to_string(),
8826                "credentials-provider/v1".to_string(),
8827                json!({ "requires": [
8828                    { "capability": "credentials-provider/v1", "need": "required" },
8829                    { "capability": "credentials-provider/v1", "need": "optional" }
8830                ] }),
8831                None,
8832            ),
8833            (
8834                "capabilities_root_pointer".to_string(),
8835                "runtime_computed[0]".to_string(),
8836                "/capabilities".to_string(),
8837                json!({}),
8838                Some(json!(["/capabilities"])),
8839            ),
8840            (
8841                "capabilities_descendant_pointer".to_string(),
8842                "runtime_computed[0]".to_string(),
8843                "/capabilities/provides".to_string(),
8844                json!({}),
8845                Some(json!(["/capabilities/provides"])),
8846            ),
8847            (
8848                "malformed_pointer_without_leading_slash".to_string(),
8849                "runtime_computed[0]".to_string(),
8850                "capabilities".to_string(),
8851                json!({}),
8852                Some(json!(["capabilities"])),
8853            ),
8854            (
8855                "malformed_pointer_escape".to_string(),
8856                "runtime_computed[0]".to_string(),
8857                "/roles/~2/tools".to_string(),
8858                json!({}),
8859                Some(json!(["/roles/~2/tools"])),
8860            ),
8861            (
8862                "unknown_capabilities_field".to_string(),
8863                "capabilities.future".to_string(),
8864                "<array>".to_string(),
8865                json!({ "future": [] }),
8866                None,
8867            ),
8868        ]);
8869
8870        for (index, (name, field, value, capabilities, runtime_computed)) in
8871            cases.into_iter().enumerate()
8872        {
8873            let registry = Arc::new(Registry::default());
8874            let handler = ControlHandler::new(Arc::clone(&registry));
8875            let response = handler
8876                .handle_control(
8877                    ConnectionId::new((index + 1) as u64),
8878                    capability_grammar_hello_frame(
8879                        capabilities,
8880                        runtime_computed,
8881                        index as u64 + 1,
8882                    ),
8883                )
8884                .expect("invalid HELLO returns a refusal");
8885
8886            assert_eq!(response.len(), 1, "{name} must emit one refusal");
8887            let error = parse_error(&response[0]);
8888            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8889            let message = error["message"]
8890                .as_str()
8891                .expect("error message is a string");
8892            assert!(
8893                message.contains(&field),
8894                "{name}: field missing from {message}"
8895            );
8896            assert!(
8897                message.contains(&value),
8898                "{name}: value missing from {message}"
8899            );
8900            assert_eq!(
8901                registry
8902                    .active_registration_count()
8903                    .expect("registry reads"),
8904                0,
8905                "{name}: refused HELLO must not create a catalog entry"
8906            );
8907        }
8908    }
8909
8910    #[test]
8911    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8912        let registry = Arc::new(Registry::default());
8913        let handler = ControlHandler::new(Arc::clone(&registry));
8914        let capabilities = json!({
8915            "provides": ["credentials-provider/v1"],
8916            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8917            "must_never_reach": ["federation-transport/v1"]
8918        });
8919        let response = handler
8920            .handle_control(
8921                ConnectionId::new(99),
8922                capability_grammar_hello_frame(
8923                    capabilities.clone(),
8924                    Some(json!(["/roles/0/tools"])),
8925                    99,
8926                ),
8927            )
8928            .expect("valid HELLO registers");
8929        assert_eq!(response[0].header.ty, FrameType::HelloAck);
8930
8931        let request = Frame::build(
8932            FrameType::Request,
8933            control_flags(),
8934            0,
8935            0,
8936            100,
8937            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8938                .expect("catalog request serializes"),
8939        )
8940        .expect("catalog request frame builds");
8941        let response = handler
8942            .handle_catalog_list(request, None)
8943            .expect("catalog list succeeds");
8944        let ClientControlResponse::CatalogList { modules, .. } =
8945            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8946        else {
8947            panic!("catalog request must return catalog.list");
8948        };
8949        assert_eq!(modules.len(), 1);
8950        assert_eq!(
8951            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8952            capabilities
8953        );
8954    }
8955
8956    #[test]
8957    fn catalog_list_mirrors_management_operation_description() {
8958        let registry = Arc::new(Registry::default());
8959        let handler = ControlHandler::new(Arc::clone(&registry));
8960        let description = "List managed records and return their identifiers and metadata.";
8961        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8962        manifest.provides = vec![ProviderRole::ManagementSurface {
8963            operations: vec![ManagementOperation {
8964                name: "records.list".to_string(),
8965                kind: ManagementOperationKind::Query,
8966                description: Some(description.to_string()),
8967            }],
8968            config_schema: json!({"type": "object"}),
8969            observability: vec![ObservabilitySurface {
8970                name: "records.stats".to_string(),
8971                kind: ObservabilityKind::Snapshot,
8972            }],
8973            identity_scope: vec![IdentityScope::Project],
8974            concurrency: Concurrency::ModuleManaged,
8975        }];
8976        registry
8977            .register_with_control_ops(
8978                manifest,
8979                PROTOCOL_VERSION,
8980                ConnectionId::new(99),
8981                Vec::new(),
8982            )
8983            .expect("described management manifest registers");
8984
8985        let request = Frame::build(
8986            FrameType::Request,
8987            control_flags(),
8988            0,
8989            0,
8990            100,
8991            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8992                .expect("catalog request serializes"),
8993        )
8994        .expect("catalog request frame builds");
8995        let response = handler
8996            .handle_catalog_list(request, None)
8997            .expect("catalog list succeeds");
8998        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8999        assert_eq!(
9000            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9001            "catalog.list must preserve the declared operation description verbatim"
9002        );
9003    }
9004
9005    #[test]
9006    fn catalog_list_mirrors_registered_operation_delivery() {
9007        let registry = Arc::new(Registry::default());
9008        let handler = ControlHandler::new(Arc::clone(&registry));
9009        let delivery =
9010            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9011        let mut body = serde_json::to_value(ModuleHelloBody {
9012            manifest: manifest("watch-management", PROTOCOL_VERSION),
9013            protocol_ver: PROTOCOL_VERSION,
9014            control_ops: None,
9015            launch_nonce: None,
9016        })
9017        .unwrap();
9018        body["manifest"]["provides"] = json!([{
9019            "role": "management_surface",
9020            "operations": [
9021                {"name": "status", "kind": "query"},
9022                {"name": "future", "kind": "query"},
9023                {"name": "reset", "kind": "mutate"}
9024            ],
9025            "config_schema": {}, "observability": [], "identity_scope": []
9026        }]);
9027        body["manifest"]["operation_delivery"] = delivery.clone();
9028        let hello = Frame::build(
9029            FrameType::Hello,
9030            control_flags(),
9031            0,
9032            0,
9033            99,
9034            serde_json::to_vec(&body).unwrap(),
9035        )
9036        .unwrap();
9037        let response = handler
9038            .handle_control(ConnectionId::new(99), hello)
9039            .unwrap();
9040        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9041
9042        let (_, registrations) = registry.list_modules().unwrap();
9043        let registered = &registrations[0].manifest;
9044        assert!(registered.is_watch_operation("status"));
9045        assert!(!registered.is_watch_operation("future"));
9046        assert!(!registered.is_watch_operation("reset"));
9047        assert!(!registered.is_watch_operation("missing"));
9048
9049        let request = Frame::build(
9050            FrameType::Request,
9051            control_flags(),
9052            0,
9053            0,
9054            100,
9055            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9056        )
9057        .unwrap();
9058        let response = handler.handle_catalog_list(request, None).unwrap();
9059        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9060        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9061        let ClientControlResponse::CatalogList { modules, .. } =
9062            serde_json::from_slice(&response[0].body).unwrap()
9063        else {
9064            panic!("expected catalog.list");
9065        };
9066        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9067    }
9068
9069    #[test]
9070    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9071        let registry = Arc::new(Registry::default());
9072        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9073            [("vault".to_string(), true), ("squatter".to_string(), true)],
9074            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9075        );
9076        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9077        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9078            provides: vec!["credentials-provider/v1".to_string()],
9079            requires: Vec::new(),
9080            must_never_reach: Vec::new(),
9081        });
9082        let frame = Frame::build(
9083            FrameType::Hello,
9084            control_flags(),
9085            0,
9086            0,
9087            77,
9088            serde_json::to_vec(&ModuleHelloBody {
9089                manifest: squatter,
9090                protocol_ver: PROTOCOL_VERSION,
9091                control_ops: None,
9092                launch_nonce: None,
9093            })
9094            .expect("HELLO serializes"),
9095        )
9096        .expect("HELLO frame builds");
9097        let response = handler
9098            .handle_control(ConnectionId::new(77), frame)
9099            .expect("reserved claim receives a typed refusal");
9100        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9101        assert_eq!(
9102            registry
9103                .active_registration_count()
9104                .expect("registry reads"),
9105            0,
9106            "a reserved capability refusal must not leave a catalog entry"
9107        );
9108    }
9109
9110    #[test]
9111    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9112        for settlement in ["timeout", "inconclusive", "drop"] {
9113            let breakers = RouteBindBreakers::default();
9114            let RouteBindAdmission::Admitted {
9115                guard: mut old,
9116                probe: false,
9117            } = breakers.admit("prov")
9118            else {
9119                panic!("ordinary relay admitted")
9120            };
9121            let RouteBindAdmission::Admitted {
9122                guard: mut opener, ..
9123            } = breakers.admit("prov")
9124            else {
9125                panic!("second relay admitted")
9126            };
9127            assert!(
9128                !opener
9129                    .record_timeout(1, Duration::ZERO)
9130                    .unwrap()
9131                    .reopened_after_probe
9132            );
9133            let RouteBindAdmission::Admitted {
9134                guard: mut probe,
9135                probe: true,
9136            } = breakers.admit("prov")
9137            else {
9138                panic!("one cooldown probe admitted")
9139            };
9140            match settlement {
9141                "timeout" => assert!(
9142                    !old.record_timeout(1, Duration::ZERO)
9143                        .unwrap()
9144                        .reopened_after_probe
9145                ),
9146                "inconclusive" => old.record_inconclusive(),
9147                "drop" => drop(old),
9148                _ => unreachable!(),
9149            }
9150            assert!(
9151                matches!(
9152                    breakers.admit("prov"),
9153                    RouteBindAdmission::Refused {
9154                        probe_in_flight: true,
9155                        ..
9156                    }
9157                ),
9158                "{settlement} of a pre-open relay cannot release the real probe"
9159            );
9160            assert!(
9161                probe
9162                    .record_timeout(1, Duration::ZERO)
9163                    .unwrap()
9164                    .reopened_after_probe
9165            );
9166            assert!(matches!(
9167                breakers.admit("prov"),
9168                RouteBindAdmission::Admitted { probe: true, .. }
9169            ));
9170        }
9171        let breakers = RouteBindBreakers::default();
9172        let admit = || match breakers.admit("prov") {
9173            RouteBindAdmission::Admitted { guard, .. } => guard,
9174            _ => panic!("relay admitted"),
9175        };
9176        admit().record_timeout(1, Duration::ZERO);
9177        let mut old_probe = admit();
9178        breakers.reset_for_new_module_connection("prov");
9179        admit().record_timeout(1, Duration::ZERO);
9180        let _new_probe = admit();
9181        old_probe.record_inconclusive();
9182        assert!(matches!(
9183            breakers.admit("prov"),
9184            RouteBindAdmission::Refused {
9185                probe_in_flight: true,
9186                ..
9187            }
9188        ));
9189    }
9190
9191    #[tokio::test]
9192    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9193        for candidate in [false, true] {
9194            let registry = Arc::new(Registry::default());
9195            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9196                [("vault".to_string(), true), ("squatter".to_string(), true)],
9197                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9198            );
9199            let conn = ConnectionId::new(77);
9200            let (ctx, mut rx) = route_ctx(conn);
9201            let initial = capability_manifest("squatter", &[], &[]);
9202            if candidate {
9203                registry
9204                    .register_candidate_with_control_ops(
9205                        initial.clone(),
9206                        PROTOCOL_VERSION,
9207                        conn,
9208                        module_baseline_control_ops(),
9209                    )
9210                    .unwrap();
9211                handler
9212                    .forwarding
9213                    .register_candidate_module_connection(
9214                        conn,
9215                        "squatter".to_string(),
9216                        PROTOCOL_VERSION,
9217                        manifest_concurrency(&initial),
9218                        ctx.egress.clone(),
9219                    )
9220                    .unwrap();
9221            } else {
9222                hello_via_sink(
9223                    &handler,
9224                    &ctx,
9225                    &mut rx,
9226                    hello_frame_with_manifest(initial.clone(), 1),
9227                )
9228                .await;
9229            }
9230            let response = handler
9231                .handle_control_frame(
9232                    &ctx,
9233                    catalog_update_with_capabilities_frame(
9234                        2,
9235                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
9236                            .capabilities
9237                            .unwrap(),
9238                    ),
9239                )
9240                .await
9241                .unwrap();
9242            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9243            assert_eq!(
9244                registry
9245                    .get_module_by_connection(conn)
9246                    .unwrap()
9247                    .unwrap()
9248                    .manifest,
9249                initial
9250            );
9251        }
9252    }
9253
9254    #[test]
9255    fn server_describe_surfaces_required_capability_verdict_fields() {
9256        let registry = Arc::new(Registry::default());
9257        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9258            [
9259                ("consumer".to_string(), true),
9260                ("provider".to_string(), false),
9261            ],
9262            BTreeMap::new(),
9263        );
9264        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9265        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9266            provides: Vec::new(),
9267            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9268                capability: "credentials-provider/v1".to_string(),
9269                need: subc_protocol::manifest::CapabilityNeed::Required,
9270            }],
9271            must_never_reach: Vec::new(),
9272        });
9273        let hello = Frame::build(
9274            FrameType::Hello,
9275            control_flags(),
9276            0,
9277            0,
9278            78,
9279            serde_json::to_vec(&ModuleHelloBody {
9280                manifest: consumer,
9281                protocol_ver: PROTOCOL_VERSION,
9282                control_ops: None,
9283                launch_nonce: None,
9284            })
9285            .expect("HELLO serializes"),
9286        )
9287        .expect("HELLO frame builds");
9288        handler
9289            .handle_control(ConnectionId::new(78), hello)
9290            .expect("consumer registers");
9291        let describe = Frame::build(
9292            FrameType::Request,
9293            control_flags(),
9294            0,
9295            0,
9296            79,
9297            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9298                .expect("request serializes"),
9299        )
9300        .expect("describe frame builds");
9301        let response = handler
9302            .handle_server_describe(describe)
9303            .expect("server.describe succeeds");
9304        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9305        let requirement = &rendered["capability_requirements"][0];
9306        assert_eq!(requirement["consumer"], "consumer");
9307        assert_eq!(requirement["verdict"], "never_provided");
9308        assert_eq!(requirement["episode_seq"], 1);
9309        assert_eq!(requirement["config_satisfiable"], false);
9310        assert_eq!(requirement["runtime_available"], false);
9311        assert!(requirement["detail"]
9312            .as_str()
9313            .expect("detail string")
9314            .contains("credentials-provider/v1"));
9315    }
9316
9317    /// Register a consumer that requires `project-identity/v1` and the provider
9318    /// of it, optionally mark daemon shutdown as begun, then deregister the
9319    /// provider as its connection closes. Returns the `capability.requirement`
9320    /// events that deregistration logged and the verdict the evaluator then
9321    /// holds for the requirement (the one `route.open` refusals read).
9322    fn deregister_required_provider(
9323        daemon_shutdown_begun: bool,
9324    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9325        const CAPABILITY: &str = "project-identity/v1";
9326        let registry = Arc::new(Registry::default());
9327        let daemon_shutdown = DaemonShutdownFlag::default();
9328        let handler = ControlHandler::new(Arc::clone(&registry))
9329            .with_capability_config(
9330                [
9331                    ("consumer".to_string(), true),
9332                    ("provider".to_string(), true),
9333                ],
9334                BTreeMap::new(),
9335            )
9336            .with_daemon_shutdown_flag(daemon_shutdown.clone());
9337        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9338        consumer.capabilities = Some(CapabilityDeclarations {
9339            provides: Vec::new(),
9340            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9341                capability: CAPABILITY.to_string(),
9342                need: subc_protocol::manifest::CapabilityNeed::Required,
9343            }],
9344            must_never_reach: Vec::new(),
9345        });
9346        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9347        let provider_conn = ConnectionId::new(182);
9348        for (conn, manifest, corr) in [
9349            (ConnectionId::new(181), consumer, 181),
9350            (provider_conn, provider, 182),
9351        ] {
9352            let hello = handler
9353                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9354                .expect("module registers");
9355            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9356        }
9357        assert_eq!(
9358            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9359            Some(CapabilityVerdict::Provided),
9360            "the registered provider must satisfy the requirement first"
9361        );
9362        if daemon_shutdown_begun {
9363            daemon_shutdown.set_for_test();
9364        }
9365
9366        let capture = EventCapture::default();
9367        let _subscriber =
9368            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9369        handler
9370            .cleanup_connection(provider_conn)
9371            .expect("provider deregisters");
9372        let events = capture
9373            .events()
9374            .into_iter()
9375            .filter(|event| {
9376                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9377            })
9378            .collect();
9379        (
9380            events,
9381            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9382        )
9383    }
9384
9385    #[test]
9386    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9387        let (events, verdict) = deregister_required_provider(true);
9388        assert_eq!(
9389            verdict,
9390            Some(CapabilityVerdict::NeverProvided),
9391            "shutdown must not change the verdict route.open refusals read"
9392        );
9393        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9394        let event = &events[0];
9395        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9396        assert_eq!(
9397            event.fields.get("cause").map(String::as_str),
9398            Some("\"daemon_shutdown\""),
9399            "{event:?}"
9400        );
9401        assert_eq!(
9402            event.fields.get("verdict").map(String::as_str),
9403            Some("\"never_provided\""),
9404            "{event:?}"
9405        );
9406    }
9407
9408    #[test]
9409    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9410        let (events, verdict) = deregister_required_provider(false);
9411        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9412        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9413        let event = &events[0];
9414        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9415        assert_eq!(event.fields.get("cause"), None, "{event:?}");
9416        assert_eq!(
9417            event.fields.get("verdict").map(String::as_str),
9418            Some("\"never_provided\""),
9419            "{event:?}"
9420        );
9421    }
9422
9423    #[test]
9424    fn catalog_list_omits_capabilities_for_legacy_manifest() {
9425        let registry = Arc::new(Registry::default());
9426        let handler = ControlHandler::new(Arc::clone(&registry));
9427        let hello = handler
9428            .handle_control(
9429                ConnectionId::new(101),
9430                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9431            )
9432            .expect("legacy HELLO registers");
9433        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9434
9435        let request = Frame::build(
9436            FrameType::Request,
9437            control_flags(),
9438            0,
9439            0,
9440            102,
9441            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9442                .expect("catalog request serializes"),
9443        )
9444        .expect("catalog request frame builds");
9445        let response = handler
9446            .handle_catalog_list(request, None)
9447            .expect("catalog list succeeds");
9448        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9449        assert!(
9450            body["modules"][0].get("capabilities").is_none(),
9451            "legacy manifest must retain an absent capabilities field on catalog.list"
9452        );
9453    }
9454
9455    #[test]
9456    fn hello_ack_omits_storage_when_no_storage_config() {
9457        let registry = Arc::new(Registry::default());
9458        let handler = ControlHandler::new(Arc::clone(&registry));
9459        let responses = handler
9460            .handle_control(
9461                ConnectionId::new(1),
9462                hello_frame("aft", PROTOCOL_VERSION, 7),
9463            )
9464            .unwrap();
9465        let ack = parse_ack(&responses[0]);
9466        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9467        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9468    }
9469
9470    #[tokio::test]
9471    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9472        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9473        let registry = Arc::new(Registry::default());
9474        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
9475        let responses = handler
9476            .handle_control(
9477                ConnectionId::new(1),
9478                hello_frame("aft", PROTOCOL_VERSION, 7),
9479            )
9480            .unwrap();
9481        let ack = parse_ack(&responses[0]);
9482        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9483
9484        let described = handler
9485            .handle_control_frame(
9486                &route_ctx(ConnectionId::new(2)).0,
9487                Frame::build(
9488                    FrameType::Request,
9489                    control_flags(),
9490                    0,
9491                    0,
9492                    9,
9493                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9494                )
9495                .unwrap(),
9496            )
9497            .await
9498            .unwrap();
9499        let ClientControlResponse::ServerDescribe { machine_id, .. } =
9500            serde_json::from_slice(&described[0].body).unwrap()
9501        else {
9502            panic!("server.describe answered with another shape");
9503        };
9504        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9505    }
9506
9507    #[test]
9508    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9509        // With a central sqlite storage policy, each registering module gets its
9510        // own resolved descriptor in HELLO_ACK, keyed by its module id.
9511        let registry = Arc::new(Registry::default());
9512        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
9513            crate::daemon_config::StorageConfig::Sqlite {
9514                data_home: std::path::PathBuf::from("/data"),
9515            },
9516        ));
9517
9518        let responses = handler
9519            .handle_control(
9520                ConnectionId::new(1),
9521                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9522            )
9523            .unwrap();
9524        let ack = parse_ack(&responses[0]);
9525        assert_eq!(
9526            ack.storage,
9527            Some(serde_json::json!({
9528                "module_id": "alfonso-routing",
9529                "storage_namespace": "default",
9530                "isolation": { "kind": "module" },
9531                "backend": {
9532                    "backend": "sqlite",
9533                    "path": "/data/cortexkit/alfonso-routing/store.db"
9534                }
9535            })),
9536            "the delivered descriptor is the module's own sqlite store path"
9537        );
9538    }
9539
9540    #[test]
9541    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9542        let registry = Arc::new(Registry::default());
9543        let handler = ControlHandler::new(Arc::clone(&registry));
9544        let conn = ConnectionId::new(1);
9545        let responses = handler
9546            .handle_control(
9547                conn,
9548                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9549            )
9550            .unwrap();
9551        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9552        let registration = registry.get_module("aft").unwrap().unwrap();
9553        assert_eq!(registration.control_ops, module_baseline_control_ops());
9554
9555        let frame =
9556            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9557        assert!(handler
9558            .guard_module_control_op(&frame, "aft", "route.bind")
9559            .unwrap()
9560            .is_none());
9561        let error = handler
9562            .guard_module_control_op(&frame, "aft", "test.synthetic")
9563            .unwrap()
9564            .expect("synthetic ungranted op should be rejected");
9565        assert_eq!(error.header.ty, FrameType::Error);
9566        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9567    }
9568
9569    #[test]
9570    fn hello_control_ops_some_adds_optional_grants() {
9571        let registry = Arc::new(Registry::default());
9572        let handler = ControlHandler::new(Arc::clone(&registry));
9573        handler
9574            .handle_control(
9575                ConnectionId::new(1),
9576                hello_frame_with_control_ops(
9577                    "aft",
9578                    PROTOCOL_VERSION,
9579                    7,
9580                    Some(vec![
9581                        "future.synthetic".to_string(),
9582                        "route.bind".to_string(),
9583                    ]),
9584                ),
9585            )
9586            .unwrap();
9587        let registration = registry.get_module("aft").unwrap().unwrap();
9588        assert_eq!(
9589            registration.control_ops,
9590            vec![
9591                "route.bind".to_string(),
9592                "route.status".to_string(),
9593                "future.synthetic".to_string(),
9594            ]
9595        );
9596        let frame =
9597            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9598        assert!(handler
9599            .guard_module_control_op(&frame, "aft", "future.synthetic")
9600            .unwrap()
9601            .is_none());
9602    }
9603
9604    #[tokio::test]
9605    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9606        let registry = Arc::new(Registry::default());
9607        let forwarding = Arc::new(ForwardingTable::default());
9608        let handler =
9609            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9610        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9611        hello_via_sink(
9612            &handler,
9613            &module_ctx,
9614            &mut module_rx,
9615            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9616        )
9617        .await;
9618
9619        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9620        let responses = handler
9621            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9622            .await
9623            .unwrap();
9624        assert_eq!(responses.len(), 1);
9625        assert_eq!(responses[0].header.ty, FrameType::Error);
9626        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9627        assert!(module_rx.try_recv().is_err());
9628    }
9629
9630    #[tokio::test]
9631    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9632        let registry = Arc::new(Registry::default());
9633        let forwarding = Arc::new(ForwardingTable::default());
9634        let handler =
9635            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9636        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9637        hello_via_sink(
9638            &handler,
9639            &module_ctx,
9640            &mut module_rx,
9641            hello_frame_with_control_ops(
9642                "aft",
9643                PROTOCOL_VERSION,
9644                7,
9645                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9646            ),
9647        )
9648        .await;
9649
9650        let project_root = unique_project_root("demux");
9651        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9652        let route_handler = handler.clone();
9653        let route_task = tokio::spawn(async move {
9654            route_handler
9655                .handle_control_frame(
9656                    &route_client_ctx,
9657                    route_open_frame(100, "aft", project_root),
9658                )
9659                .await
9660                .unwrap()
9661        });
9662        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9663            .await
9664            .unwrap()
9665            .unwrap();
9666        assert!(matches!(
9667            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9668            ModuleControlRequest::RouteBind { .. }
9669        ));
9670
9671        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9672        let health_handler = handler.clone();
9673        let health_task = tokio::spawn(async move {
9674            health_handler
9675                .handle_control_frame(
9676                    &health_client_ctx,
9677                    supervisor_health_probe_frame(101, "aft"),
9678                )
9679                .await
9680                .unwrap()
9681        });
9682        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9683            .await
9684            .unwrap()
9685            .unwrap();
9686        assert_eq!(
9687            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9688            ModuleControlRequest::HealthCheck {}
9689        );
9690
9691        handler
9692            .handle_control_frame(
9693                &module_ctx,
9694                health_response(health_frame.header.corr, HealthStatus::Degraded),
9695            )
9696            .await
9697            .unwrap();
9698        let health_response = health_task.await.unwrap();
9699        assert_eq!(health_response.len(), 1);
9700        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9701            ClientControlResponse::SupervisorHealthProbe {
9702                module_id,
9703                status,
9704                detail,
9705                metrics,
9706            } => {
9707                assert_eq!(module_id, "aft");
9708                assert_eq!(status, HealthStatus::Degraded);
9709                assert_eq!(detail.as_deref(), Some("warming"));
9710                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9711            }
9712            other => panic!("unexpected health response: {other:?}"),
9713        }
9714
9715        handler
9716            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9717            .await
9718            .unwrap();
9719        let route_response = route_task.await.unwrap();
9720        assert!(route_response.is_empty());
9721        let published = route_client_rx.recv().await.unwrap();
9722        assert!(matches!(
9723            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9724            ClientControlResponse::RouteOpen { .. }
9725        ));
9726    }
9727
9728    /// Start one `route.open` on `client_connection` and return its still-running
9729    /// handler task together with the `route.bind` the module received for it.
9730    /// The handler blocks until the module answers, so it has to run as a task
9731    /// while the test drives the module side.
9732    async fn relay_route_open(
9733        handler: &ControlHandler,
9734        client_connection: ConnectionId,
9735        client_egress: &FrameSink,
9736        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9737        corr: u64,
9738        module_id: &str,
9739        project_root_label: &str,
9740    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9741        let ctx = RouteCtx {
9742            connection_id: client_connection,
9743            egress: client_egress.clone(),
9744        };
9745        let handler = handler.clone();
9746        let project_root = unique_project_root(project_root_label);
9747        let module_id = module_id.to_string();
9748        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9749        let task = tokio::spawn(async move {
9750            let _guard = tracing::dispatcher::set_default(&dispatch);
9751            handler
9752                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9753                .await
9754                .unwrap()
9755        });
9756        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9757            .await
9758            .expect("module receives the relayed route.bind")
9759            .expect("module egress is open");
9760        (task, bind.frame)
9761    }
9762
9763    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9764        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9765            ModuleControlRequest::RouteBind {
9766                route_channel,
9767                epoch,
9768                ..
9769            } => (route_channel, epoch),
9770            other => panic!("expected a route.bind request, got {other:?}"),
9771        }
9772    }
9773
9774    fn published_route(frame: &Frame) -> (u16, u32) {
9775        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9776            ClientControlResponse::RouteOpen {
9777                route_channel,
9778                route_epoch,
9779            } => (route_channel, route_epoch),
9780            other => panic!("expected a route.open response, got {other:?}"),
9781        }
9782    }
9783
9784    /// Reproduction of a production outage. A client had `route.open`s in
9785    /// flight to a module and was already marked closing -- its egress had refused a
9786    /// module frame, so the daemon asked its connection to end -- while its sink
9787    /// was still open. When the module acked those binds, the daemon refused to
9788    /// commit a route for a closing client, and that refusal was returned from
9789    /// the MODULE connection's frame handler, where a router error that has no
9790    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
9791    /// the supervisor correctly did not respawn a clean exit, and every seat lost
9792    /// its tools for hours -- one client's teardown took down a connection
9793    /// carrying ~170 other routes.
9794    ///
9795    /// The window is opened here by calling the production path that opens it
9796    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
9797    /// state that matters is "in `closing_connections`, sink still open, relay
9798    /// still pending", and it lasts only from the close request until the
9799    /// connection loop reacts to it; a socket-level test can flood a client into
9800    /// that escalation but cannot pin the module's ack inside the window. Closing
9801    /// the socket instead takes the other path entirely -- connection teardown
9802    /// removes the pending relay under the same lock, so the ack finds nothing.
9803    #[tokio::test]
9804    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9805        let registry = Arc::new(Registry::default());
9806        let forwarding = Arc::new(ForwardingTable::default());
9807        let handler =
9808            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9809
9810        let module_connection = ConnectionId::new(30);
9811        let (module_ctx, mut module_rx) = route_ctx(module_connection);
9812        hello_via_sink(
9813            &handler,
9814            &module_ctx,
9815            &mut module_rx,
9816            hello_frame("aft", PROTOCOL_VERSION, 7),
9817        )
9818        .await;
9819
9820        let dying_client = ConnectionId::new(31);
9821        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9822
9823        // A published route on the dying client. The escalation below only marks
9824        // a connection closing for a route it has already published.
9825        let (first_task, first_bind) = relay_route_open(
9826            &handler,
9827            dying_client,
9828            &dying_ctx.egress,
9829            &mut module_rx,
9830            100,
9831            "aft",
9832            "closing-first",
9833        )
9834        .await;
9835        handler
9836            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9837            .await
9838            .unwrap();
9839        assert!(first_task.await.unwrap().is_empty());
9840        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9841
9842        // A second route.open from the same client, relayed and awaiting its ack.
9843        let (second_task, second_bind) = relay_route_open(
9844            &handler,
9845            dying_client,
9846            &dying_ctx.egress,
9847            &mut module_rx,
9848            101,
9849            "aft",
9850            "closing-second",
9851        )
9852        .await;
9853        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9854
9855        // The window: the client is closing, its sink is still open, and its
9856        // second bind is still pending.
9857        assert!(forwarding
9858            .escalate_client_delivery_failure(
9859                dying_client,
9860                first_channel,
9861                first_epoch,
9862                CloseReason::new(
9863                    "module_to_client_delivery_failed",
9864                    "client egress refused a module frame",
9865                ),
9866                crate::forwarding::UndeliveredFrame {
9867                    module_id: None,
9868                    sink: &dying_ctx.egress,
9869                },
9870            )
9871            .unwrap());
9872        assert!(!dying_ctx.egress.is_closed());
9873
9874        // The frame that used to end the module connection.
9875        let ack = handler
9876            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9877            .await;
9878        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9879        if module_loop_error.is_some() {
9880            // What the server's connection loop does with a router error that has
9881            // no ERROR-frame translation: end the connection, which releases the
9882            // module's registration and every route on it.
9883            handler.cleanup_connection(module_connection).unwrap();
9884        }
9885        // Read the module's next frame before opening the co-tenant's route, so
9886        // the GOODBYE assertion below is about THIS ack and not about later
9887        // traffic. `None` means the module was told nothing.
9888        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9889            .await
9890            .ok()
9891            .flatten();
9892
9893        // 1. The module connection is still registered.
9894        assert!(
9895            registry
9896                .get_module_by_connection(module_connection)
9897                .unwrap()
9898                .is_some(),
9899            "one client's closing connection ended the shared module connection: \
9900             {module_loop_error:?}"
9901        );
9902        // ...and still serving: another client can open and use a route on it.
9903        let cotenant = ConnectionId::new(32);
9904        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9905        let (cotenant_task, cotenant_bind) = relay_route_open(
9906            &handler,
9907            cotenant,
9908            &cotenant_ctx.egress,
9909            &mut module_rx,
9910            102,
9911            "aft",
9912            "closing-cotenant",
9913        )
9914        .await;
9915        handler
9916            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9917            .await
9918            .unwrap();
9919        assert!(cotenant_task.await.unwrap().is_empty());
9920        let (cotenant_channel, cotenant_epoch) =
9921            published_route(&cotenant_rx.recv().await.unwrap());
9922        assert!(matches!(
9923            forwarding
9924                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9925                .unwrap(),
9926            DataRoute::Client(DataRouteState::Bound(_))
9927        ));
9928
9929        // 2. The module was told to drop the binding it created for the route
9930        //    that will never be published.
9931        let goodbye = post_ack_module_frame
9932            .expect("module receives a GOODBYE for the abandoned route channel");
9933        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9934        assert_eq!(goodbye.header.channel, abandoned_channel);
9935        assert_eq!(goodbye.header.epoch, abandoned_epoch);
9936
9937        // 3. The dying client received nothing: no route was ever published to
9938        //    it. Its route.open is answered as unavailable, which the connection
9939        //    loop would write to a socket that is already going away.
9940        assert!(dying_rx.try_recv().is_err());
9941        let second_response = second_task.await.unwrap();
9942        assert_eq!(second_response.len(), 1);
9943        assert_eq!(
9944            parse_error(&second_response[0])["code"],
9945            "target_unavailable"
9946        );
9947    }
9948
9949    /// One `catalog.list` as a client sees it: the generation and the listed
9950    /// module ids with their readiness.
9951    async fn catalog_snapshot(
9952        handler: &ControlHandler,
9953        client_ctx: &RouteCtx,
9954        corr: u64,
9955    ) -> (u64, Vec<(String, bool)>) {
9956        let body =
9957            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9958        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9959        let reply = handler
9960            .handle_control_frame(client_ctx, request)
9961            .await
9962            .unwrap()
9963            .pop()
9964            .unwrap();
9965        let ClientControlResponse::CatalogList {
9966            generation,
9967            modules,
9968            ..
9969        } = serde_json::from_slice(&reply.body).unwrap()
9970        else {
9971            panic!("not a catalog.list reply");
9972        };
9973        (
9974            generation,
9975            modules
9976                .into_iter()
9977                .map(|entry| (entry.module_id, entry.ready))
9978                .collect(),
9979        )
9980    }
9981
9982    /// A handler whose HELLO for module "aft" on connection 40 is parked
9983    /// between reserving its registry entry and installing its forwarding
9984    /// endpoint. Returns the handler, the two-party barrier (wait once to know
9985    /// the HELLO is parked, once more to release it), the HELLO's thread, and
9986    /// the module's context and receiver.
9987    fn hello_held_before_forwarding() -> (
9988        ControlHandler,
9989        Arc<std::sync::Barrier>,
9990        std::thread::JoinHandle<Vec<Frame>>,
9991        RouteCtx,
9992        mpsc::Receiver<crate::router::OutboundFrame>,
9993    ) {
9994        let hold = Arc::new(std::sync::Barrier::new(2));
9995        let handler = ControlHandler::with_forwarding(
9996            Arc::new(Registry::default()),
9997            Arc::new(ForwardingTable::default()),
9998        )
9999        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10000        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10001        let hello = {
10002            let handler = handler.clone();
10003            let module_ctx = module_ctx.clone();
10004            std::thread::spawn(move || {
10005                handler
10006                    .handle_hello(
10007                        module_ctx.connection_id,
10008                        Some(module_ctx.egress.clone()),
10009                        hello_frame("aft", PROTOCOL_VERSION, 7),
10010                    )
10011                    .unwrap()
10012            })
10013        };
10014        (handler, hold, hello, module_ctx, module_rx)
10015    }
10016
10017    /// While a HELLO has reserved its registry entry but its forwarding
10018    /// endpoint is not installed yet, `route.open` to the module can only be
10019    /// refused. `catalog.list` must not list the module in that window: a
10020    /// client that opens a route as soon as it sees a module listed and ready
10021    /// would be refused for a module the daemon had just advertised. The
10022    /// refusal in the window stays the retryable `target_unavailable`, never
10023    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10024    /// Once the HELLO completes, the module is listed and a route to it opens.
10025    #[tokio::test]
10026    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10027        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10028        let client = ConnectionId::new(41);
10029        let (client_ctx, mut client_rx) = route_ctx(client);
10030
10031        hold.wait();
10032        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10033        assert!(handler.registry.get_module("aft").unwrap().is_some());
10034        assert!(!handler
10035            .forwarding
10036            .has_live_module_connection("aft")
10037            .unwrap());
10038        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10039        assert!(
10040            listed.is_empty(),
10041            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10042        );
10043        let refused = handler
10044            .handle_control_frame(
10045                &client_ctx,
10046                route_open_frame(301, "aft", unique_project_root("listed-window")),
10047            )
10048            .await
10049            .unwrap();
10050        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10051        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10052        let duplicate = handler
10053            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10054            .await
10055            .unwrap();
10056        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10057
10058        hold.wait();
10059        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10060        let ack = module_rx.try_recv().unwrap().frame;
10061        assert_eq!(ack.header.ty, FrameType::HelloAck);
10062        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10063        assert_eq!(listed, vec![("aft".to_string(), true)]);
10064
10065        let (open_task, bind) = relay_route_open(
10066            &handler,
10067            client,
10068            &client_ctx.egress,
10069            &mut module_rx,
10070            303,
10071            "aft",
10072            "listed-after",
10073        )
10074        .await;
10075        assert!(handler
10076            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10077            .await
10078            .unwrap()
10079            .is_empty());
10080        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10081        published_route(&client_rx.try_recv().unwrap().frame);
10082    }
10083
10084    /// A catalog watcher keys on the generation: when it changes, it re-reads
10085    /// the listing. So the generation must advance exactly when the module
10086    /// appears. If it advanced while the module was still hidden, a watcher
10087    /// would read the new generation without the module, and the module would
10088    /// later appear under that same generation with nothing telling the
10089    /// watcher to look again.
10090    #[tokio::test]
10091    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10092        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10093        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10094        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10095
10096        hold.wait();
10097        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10098        assert!(listed.is_empty());
10099        assert_eq!(
10100            during, before,
10101            "the generation moved while the module was not listed"
10102        );
10103
10104        hold.wait();
10105        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10106        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10107        assert_eq!(listed, vec![("aft".to_string(), true)]);
10108        assert_eq!(
10109            after,
10110            before + 1,
10111            "the module appeared without exactly one generation bump"
10112        );
10113    }
10114
10115    /// A HELLO whose forwarding endpoint cannot be installed (here the
10116    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
10117    /// refused and leaves nothing behind: no registry entry, no catalog entry,
10118    /// and the catalog generation where it was.
10119    #[tokio::test]
10120    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10121        let registry = Arc::new(Registry::default());
10122        let handler = ControlHandler::with_forwarding(
10123            Arc::clone(&registry),
10124            Arc::new(ForwardingTable::default()),
10125        );
10126        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10127        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10128        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10129        drop(module_rx);
10130
10131        let replies = handler
10132            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10133            .await
10134            .unwrap();
10135        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10136        assert!(registry.get_module("aft").unwrap().is_none());
10137        assert!(registry
10138            .get_module_by_connection(module_ctx.connection_id)
10139            .unwrap()
10140            .is_none());
10141        assert_eq!(
10142            catalog_snapshot(&handler, &client_ctx, 301).await,
10143            (before, Vec::new())
10144        );
10145    }
10146
10147    /// A handler whose `route.open` parks after queueing the route.bind and
10148    /// before marking it enqueued, with module "aft" registered on connection
10149    /// 40. Returns the two-party barrier that parks and releases it.
10150    async fn route_open_held_after_bind_send() -> (
10151        ControlHandler,
10152        Arc<tokio::sync::Barrier>,
10153        RouteCtx,
10154        mpsc::Receiver<crate::router::OutboundFrame>,
10155    ) {
10156        let hold = Arc::new(tokio::sync::Barrier::new(2));
10157        let handler = ControlHandler::with_forwarding(
10158            Arc::new(Registry::default()),
10159            Arc::new(ForwardingTable::default()),
10160        )
10161        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10162        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10163        hello_via_sink(
10164            &handler,
10165            &module_ctx,
10166            &mut module_rx,
10167            hello_frame("aft", PROTOCOL_VERSION, 7),
10168        )
10169        .await;
10170        (handler, hold, module_ctx, module_rx)
10171    }
10172
10173    /// A fast module acks the route.bind, and the reader commits the route,
10174    /// before `route.open` has marked the relay enqueued. The daemon used to
10175    /// read the missing relay as an abandoned bind and send the module a
10176    /// GOODBYE for the route it had just committed: the client was told its
10177    /// route was open, the module unbound the channel, and the client's first
10178    /// request was dropped on the module side as unbound.
10179    #[tokio::test]
10180    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10181        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10182        let client = ConnectionId::new(41);
10183        let (client_ctx, mut client_rx) = route_ctx(client);
10184
10185        let (open_task, bind) = relay_route_open(
10186            &handler,
10187            client,
10188            &client_ctx.egress,
10189            &mut module_rx,
10190            200,
10191            "aft",
10192            "acked-before-mark",
10193        )
10194        .await;
10195        let (module_channel, module_epoch) = route_bind_channel(&bind);
10196        // route.open is now parked between queueing the bind and marking it.
10197        hold.wait().await;
10198
10199        // The module answers inside that window, and the route commits.
10200        assert!(handler
10201            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10202            .await
10203            .unwrap()
10204            .is_empty());
10205        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10206
10207        hold.wait().await;
10208        assert!(
10209            open_task.await.unwrap().is_empty(),
10210            "route.open answers accepted"
10211        );
10212
10213        // Every frame route.open could have sent is already queued, because it
10214        // has returned; the module must have been sent nothing.
10215        match module_rx.try_recv() {
10216            Err(mpsc::error::TryRecvError::Empty) => {}
10217            Ok(outbound) => panic!(
10218                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10219                outbound.frame.header.ty,
10220                outbound.frame.header.channel,
10221                outbound.frame.header.epoch
10222            ),
10223            Err(err) => panic!("module egress closed: {err}"),
10224        }
10225
10226        // The client's first request on the route is the next thing the module
10227        // reads, on the binding it just created.
10228        let router = Router::with_control_handler(Arc::new(handler.clone()));
10229        let request = Frame::build(
10230            FrameType::Request,
10231            Flags::new(true, Priority::Interactive, false),
10232            client_channel,
10233            client_epoch,
10234            201,
10235            b"first request".to_vec(),
10236        )
10237        .unwrap();
10238        router
10239            .route_for_connection(&client_ctx, request)
10240            .await
10241            .unwrap();
10242        let delivered = module_rx.try_recv().unwrap().frame;
10243        assert_eq!(delivered.header.ty, FrameType::Request);
10244        assert_eq!(
10245            (delivered.header.channel, delivered.header.epoch),
10246            (module_channel, module_epoch)
10247        );
10248        assert_eq!(delivered.body, b"first request");
10249    }
10250
10251    /// The client goes away while `route.open` is still queueing the bind, so
10252    /// the relay is dropped with no module reply. The teardown cannot send the
10253    /// GOODBYE itself (it could reach the module ahead of the bind), so
10254    /// `route.open` must send it once the bind is queued; otherwise the module
10255    /// keeps a binding the daemon will never commit.
10256    #[tokio::test]
10257    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10258        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10259        let client = ConnectionId::new(42);
10260        let (client_ctx, _client_rx) = route_ctx(client);
10261
10262        let (open_task, bind) = relay_route_open(
10263            &handler,
10264            client,
10265            &client_ctx.egress,
10266            &mut module_rx,
10267            300,
10268            "aft",
10269            "released-before-mark",
10270        )
10271        .await;
10272        let (module_channel, module_epoch) = route_bind_channel(&bind);
10273        hold.wait().await;
10274
10275        handler.cleanup_connection(client).unwrap();
10276        assert!(
10277            module_rx.try_recv().is_err(),
10278            "client teardown must not GOODBYE a bind route.open is still queueing"
10279        );
10280
10281        hold.wait().await;
10282        let response = open_task.await.unwrap();
10283        assert_eq!(response.len(), 1);
10284        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10285
10286        let goodbye = module_rx
10287            .try_recv()
10288            .expect("module is told to drop the binding the daemon abandoned")
10289            .frame;
10290        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10291        assert_eq!(
10292            (goodbye.header.channel, goodbye.header.epoch),
10293            (module_channel, module_epoch)
10294        );
10295        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10296    }
10297
10298    /// The module acks inside the same window, but the client's egress closed
10299    /// first, so the reader refuses to commit. The ack proves the module holds
10300    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
10301    /// send a second one.
10302    #[tokio::test]
10303    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10304        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10305        let client = ConnectionId::new(43);
10306        let (client_ctx, client_rx) = route_ctx(client);
10307
10308        let (open_task, bind) = relay_route_open(
10309            &handler,
10310            client,
10311            &client_ctx.egress,
10312            &mut module_rx,
10313            400,
10314            "aft",
10315            "gone-client-before-mark",
10316        )
10317        .await;
10318        let (module_channel, module_epoch) = route_bind_channel(&bind);
10319        hold.wait().await;
10320
10321        drop(client_rx);
10322        assert!(client_ctx.egress.is_closed());
10323        handler
10324            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10325            .await
10326            .unwrap();
10327
10328        hold.wait().await;
10329        let response = open_task.await.unwrap();
10330        assert_eq!(response.len(), 1);
10331        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10332
10333        let goodbye = module_rx
10334            .try_recv()
10335            .expect("module is told to drop the binding it acked")
10336            .frame;
10337        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10338        assert_eq!(
10339            (goodbye.header.channel, goodbye.header.epoch),
10340            (module_channel, module_epoch)
10341        );
10342        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10343        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10344    }
10345
10346    /// The fence at the module-loop boundary, stated as its own contract: which
10347    /// forwarding failures are allowed to end the module connection that is being
10348    /// served. A `ConnectionClosing` naming some client is about that client, and
10349    /// a module connection is shared; the same error naming the module's own
10350    /// connection is about this connection and must stay fatal, as must failures
10351    /// that are about the forwarding table itself.
10352    #[test]
10353    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10354        let handler = ControlHandler::default();
10355        let module_connection = ConnectionId::new(30);
10356        let client_connection = ConnectionId::new(31);
10357
10358        handler
10359            .refuse_to_end_module_connection_for_a_client(
10360                module_connection,
10361                77,
10362                ForwardingError::ConnectionClosing {
10363                    connection_id: client_connection,
10364                },
10365            )
10366            .expect("a closing client must never end the module connection");
10367
10368        assert!(matches!(
10369            handler.refuse_to_end_module_connection_for_a_client(
10370                module_connection,
10371                78,
10372                ForwardingError::ConnectionClosing {
10373                    connection_id: module_connection,
10374                },
10375            ),
10376            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10377                connection_id
10378            })) if connection_id == module_connection
10379        ));
10380        assert!(matches!(
10381            handler.refuse_to_end_module_connection_for_a_client(
10382                module_connection,
10383                79,
10384                ForwardingError::Poisoned,
10385            ),
10386            Err(RouterError::Forwarding(ForwardingError::Poisoned))
10387        ));
10388        assert!(matches!(
10389            handler.refuse_to_end_module_connection_for_a_client(
10390                module_connection,
10391                80,
10392                ForwardingError::StaleModuleEndpoint,
10393            ),
10394            Err(RouterError::Forwarding(
10395                ForwardingError::StaleModuleEndpoint
10396            ))
10397        ));
10398    }
10399
10400    /// The spawn-attestation guard is what stops a connected module from claiming
10401    /// another module's identity and being stamped `Reserved` for it. Every other
10402    /// test that supplies a consumer_identity supplies a CORRECT one, because a
10403    /// correct one is what the rest of the flow needs -- so the guard's rejection
10404    /// branch was never the subject of an assertion, only its acceptance branch.
10405    ///
10406    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
10407    /// whole subc-core library suite green; only the forwarding integration tests
10408    /// notice, and they notice for unrelated reasons. This test exists so the
10409    /// refusal itself is asserted where the guard lives: it fails if the identity
10410    /// check stops refusing, which is the direction that matters, since a guard
10411    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
10412    #[tokio::test]
10413    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10414        let registry = Arc::new(Registry::default());
10415        let forwarding = Arc::new(ForwardingTable::default());
10416        let supervisor = SupervisorHandle::new();
10417        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10418        let handler =
10419            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10420                .with_supervisor(supervisor);
10421
10422        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10423        hello_via_sink(
10424            &handler,
10425            &target_ctx,
10426            &mut target_rx,
10427            hello_frame("target", PROTOCOL_VERSION, 1),
10428        )
10429        .await;
10430
10431        // A real supervised module id presenting the wrong nonce. This is the
10432        // impersonation case: the attacker knows a privileged module_id, which is
10433        // public, and guesses at the nonce, which is not.
10434        let wrong_nonce = handler
10435            .handle_control_frame(
10436                &route_ctx(ConnectionId::new(91)).0,
10437                route_open_frame_with_admission_facts(
10438                    20,
10439                    "target",
10440                    unique_project_root("admission-facts"),
10441                    Some(subc_control::ConsumerIdentity {
10442                        module_id: "fed".to_string(),
10443                        launch_nonce: "not-the-real-nonce".to_string(),
10444                    }),
10445                    None,
10446                ),
10447            )
10448            .await
10449            .unwrap();
10450        assert_eq!(
10451            parse_error(&wrong_nonce[0])["code"],
10452            "bad_consumer_identity",
10453            "a mismatched launch nonce must be refused, not stamped Reserved"
10454        );
10455
10456        // A module id the supervisor never spawned at all, so no nonce exists to
10457        // compare against. An implementation that treats "no record" as "nothing
10458        // to check" fails open here while passing the case above.
10459        let never_spawned = handler
10460            .handle_control_frame(
10461                &route_ctx(ConnectionId::new(92)).0,
10462                route_open_frame_with_admission_facts(
10463                    21,
10464                    "target",
10465                    unique_project_root("admission-facts"),
10466                    Some(subc_control::ConsumerIdentity {
10467                        module_id: "never-spawned".to_string(),
10468                        launch_nonce: "any-nonce".to_string(),
10469                    }),
10470                    None,
10471                ),
10472            )
10473            .await
10474            .unwrap();
10475        assert_eq!(
10476            parse_error(&never_spawned[0])["code"],
10477            "bad_consumer_identity",
10478            "an unspawned module_id must be refused rather than accepted for lack of a record"
10479        );
10480    }
10481
10482    /// The refusal test above proves the guard says NO. Nothing proved it can say
10483    /// YES, and the difference is not academic: replacing the whole authorization
10484    /// with `false` -- admitting no consumer identity at all, revoking Reserved
10485    /// standing for every supervised module in the fleet -- leaves 110 of the 111
10486    /// library tests GREEN. The one that notices does so by HANGING, because it
10487    /// waits for a bind that can no longer happen.
10488    ///
10489    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
10490    /// or flaky test, invites a RETRY rather than an investigation, and the retry
10491    /// hangs too and gets blamed on the runner. So a total revocation of the
10492    /// daemon's trust grant would have shipped behind a symptom nobody attributes
10493    /// to code.
10494    ///
10495    /// The bias is structural rather than accidental. A REFUSAL looks like a
10496    /// failure someone writes a test for; a GRANT looks like the happy path. Every
10497    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
10498    /// suite for that reason, and this one is the purest case in the daemon.
10499    ///
10500    /// This test asserts the EFFECT rather than the absence of an error: the module
10501    /// receives a RouteBind and it carries `Reserved` naming the attested module.
10502    /// A guard that admitted nobody would produce no bind at all; one that admitted
10503    /// everybody would stamp the wrong principal, which the refusal test catches.
10504    #[tokio::test]
10505    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10506        let registry = Arc::new(Registry::default());
10507        let forwarding = Arc::new(ForwardingTable::default());
10508        let supervisor = SupervisorHandle::new();
10509        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10510        let handler =
10511            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10512                .with_supervisor(supervisor);
10513
10514        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10515        hello_via_sink(
10516            &handler,
10517            &target_ctx,
10518            &mut target_rx,
10519            hello_frame("target", PROTOCOL_VERSION, 1),
10520        )
10521        .await;
10522
10523        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10524        let route_handler = handler.clone();
10525        let route_task = tokio::spawn(async move {
10526            route_handler
10527                .handle_control_frame(
10528                    &client_ctx,
10529                    route_open_frame_with_admission_facts(
10530                        30,
10531                        "target",
10532                        unique_project_root("admission-facts"),
10533                        Some(subc_control::ConsumerIdentity {
10534                            module_id: "fed".to_string(),
10535                            launch_nonce: "fed-nonce".to_string(),
10536                        }),
10537                        None,
10538                    ),
10539                )
10540                .await
10541                .unwrap()
10542        });
10543
10544        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
10545        // the very mutation it exists to catch -- a guard that admits nobody -- no
10546        // bind is ever sent, so it HUNG rather than failing. That reproduces the
10547        // exact defect being fixed: a total revocation detected only as a stalled
10548        // suite, which reads as flakiness and invites a retry. An acceptance test
10549        // that waits for an effect must bound the wait, or a red becomes a hang.
10550        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10551            .await
10552            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10553            .expect("module control channel closed before route.bind");
10554        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10555        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10556            panic!("expected route.bind")
10557        };
10558        assert_eq!(
10559            principal,
10560            Some(Principal::Reserved {
10561                module_id: "fed".to_string()
10562            }),
10563            "a correctly attested consumer must be stamped Reserved for its own id"
10564        );
10565
10566        handler
10567            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10568            .await
10569            .unwrap();
10570        assert!(route_task.await.unwrap().is_empty());
10571        assert!(
10572            matches!(
10573                serde_json::from_slice::<ClientControlResponse>(
10574                    &client_rx.recv().await.unwrap().body
10575                )
10576                .unwrap(),
10577                ClientControlResponse::RouteOpen { .. }
10578            ),
10579            "the route must actually open, not merely avoid an error"
10580        );
10581    }
10582
10583    #[tokio::test(start_paused = true)]
10584    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10585        let registry = Arc::new(Registry::default());
10586        let forwarding = Arc::new(ForwardingTable::default());
10587        let supervisor = SupervisorHandle::new();
10588        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10589        let handler =
10590            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10591                .with_supervisor(supervisor);
10592
10593        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10594        hello_via_sink(
10595            &handler,
10596            &target_ctx,
10597            &mut target_rx,
10598            hello_frame("target", PROTOCOL_VERSION, 1),
10599        )
10600        .await;
10601
10602        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10603        let direct_handler = handler.clone();
10604        let direct_open = tokio::spawn(async move {
10605            direct_handler
10606                .handle_control_frame(
10607                    &direct_ctx,
10608                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
10609                )
10610                .await
10611                .unwrap()
10612        });
10613        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10614            .await
10615            .expect("no direct route.bind within 5s")
10616            .expect("target control channel closed before direct route.bind");
10617        handler
10618            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10619            .await
10620            .unwrap();
10621        assert!(direct_open.await.unwrap().is_empty());
10622        let _ = direct_rx.recv().await.unwrap();
10623
10624        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10625        let reserved_handler = handler.clone();
10626        let reserved_open = tokio::spawn(async move {
10627            reserved_handler
10628                .handle_control_frame(
10629                    &reserved_ctx,
10630                    route_open_frame_with_admission_facts(
10631                        3,
10632                        "target",
10633                        unique_project_root("admission-facts"),
10634                        Some(ConsumerIdentity {
10635                            module_id: "fed".to_string(),
10636                            launch_nonce: "fed-nonce".to_string(),
10637                        }),
10638                        None,
10639                    ),
10640                )
10641                .await
10642                .unwrap()
10643        });
10644        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10645            .await
10646            .expect("no reserved route.bind within 5s")
10647            .expect("target control channel closed before reserved route.bind");
10648        handler
10649            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10650            .await
10651            .unwrap();
10652        assert!(reserved_open.await.unwrap().is_empty());
10653        let _ = reserved_rx.recv().await.unwrap();
10654
10655        forwarding
10656            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10657            .unwrap();
10658        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10659        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10660            module_id: Some("target".to_string()),
10661        })
10662        .unwrap();
10663        let census_frame =
10664            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10665        let response = handler
10666            .handle_control_frame(&census_ctx, census_frame)
10667            .await
10668            .unwrap()
10669            .pop()
10670            .unwrap();
10671        let actual: Value = serde_json::from_slice(&response.body).unwrap();
10672        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10673        assert!(matches!(
10674            decoded,
10675            ClientControlResponse::SupervisorRoutes { .. }
10676        ));
10677        let routes = actual["modules"][0]["routes"].as_array().unwrap();
10678        assert_eq!(routes.len(), 2);
10679        assert!(routes.iter().all(|route| route["draining"] == true));
10680        // The census carries WHY: the reason the drain was begun with, in the
10681        // route.closing vocabulary, on every draining route this drain marked.
10682        assert!(
10683            routes.iter().all(|route| route["drain_reason"] == "reload"),
10684            "draining routes must name the drain's reason: {routes:?}"
10685        );
10686        assert!(routes.iter().any(|route| {
10687            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10688        }));
10689        assert!(routes.iter().any(|route| {
10690            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10691        }));
10692
10693        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10694            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10695        if std::env::var_os("UPDATE_GOLDEN").is_some() {
10696            std::fs::write(
10697                &golden_path,
10698                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10699            )
10700            .unwrap();
10701        }
10702        let expected: Value =
10703            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10704        assert_eq!(actual, expected);
10705    }
10706
10707    async fn query_live_roots(
10708        handler: &ControlHandler,
10709        module_ctx: &RouteCtx,
10710    ) -> ModuleControlResponseToModule {
10711        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10712        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10713        let response = handler
10714            .handle_control_frame(module_ctx, frame)
10715            .await
10716            .unwrap()
10717            .pop()
10718            .unwrap();
10719        serde_json::from_slice(&response.body).unwrap()
10720    }
10721
10722    #[tokio::test(start_paused = true)]
10723    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10724        let registry = Arc::new(Registry::default());
10725        let forwarding = Arc::new(ForwardingTable::default());
10726        let handler = ControlHandler::with_forwarding(registry, forwarding);
10727        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10728        hello_via_sink(
10729            &handler,
10730            &target_ctx,
10731            &mut target_rx,
10732            hello_frame("target", PROTOCOL_VERSION, 1),
10733        )
10734        .await;
10735        let root = unique_project_root("live-roots-known");
10736        let path = ProjectRootId::from_path_allowing_missing(root.path())
10737            .unwrap()
10738            .as_path()
10739            .to_path_buf();
10740        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10741        let open_handler = handler.clone();
10742        let opened = tokio::spawn(async move {
10743            open_handler
10744                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10745                .await
10746                .unwrap()
10747        });
10748        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10749            .await
10750            .unwrap()
10751            .unwrap();
10752        handler
10753            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10754            .await
10755            .unwrap();
10756        assert!(opened.await.unwrap().is_empty());
10757        let _ = client_rx.recv().await.unwrap();
10758
10759        let root = unique_project_root("live-roots-pending");
10760        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10761            .unwrap()
10762            .as_path()
10763            .to_path_buf();
10764        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10765        let open_handler = handler.clone();
10766        let pending = tokio::spawn(async move {
10767            open_handler
10768                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10769                .await
10770                .unwrap()
10771        });
10772        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10773            .await
10774            .unwrap()
10775            .unwrap();
10776        let actual = query_live_roots(&handler, &target_ctx).await;
10777        let ModuleControlResponseToModule::LiveRoots {
10778            roots,
10779            unknown_root_bindings,
10780            total_bindings,
10781        } = actual
10782        else {
10783            panic!("expected live roots")
10784        };
10785        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10786        assert_eq!(unknown_root_bindings, 0);
10787        assert_eq!(
10788            roots.len(),
10789            2,
10790            "root-known arm must retain each canonical root"
10791        );
10792        assert_eq!(
10793            total_bindings,
10794            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10795        );
10796        let counts = roots
10797            .iter()
10798            .map(|root| (root.project_root.clone(), root.bound, root.pending))
10799            .collect::<Vec<_>>();
10800        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10801        expected.sort_by(|a, b| a.0.cmp(&b.0));
10802        assert_eq!(
10803            counts, expected,
10804            "roots must sort by path and count pending separately"
10805        );
10806        handler
10807            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10808            .await
10809            .unwrap();
10810        assert!(pending.await.unwrap().is_empty());
10811    }
10812
10813    #[tokio::test(start_paused = true)]
10814    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10815        let forwarding = Arc::new(ForwardingTable::default());
10816        let handler =
10817            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10818        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10819        hello_via_sink(
10820            &handler,
10821            &target_ctx,
10822            &mut target_rx,
10823            hello_frame("target", PROTOCOL_VERSION, 1),
10824        )
10825        .await;
10826        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10827        let pending = forwarding
10828            .begin_route_bind_relay_for_test(
10829                client_ctx.connection_id,
10830                client_ctx.egress.clone(),
10831                2,
10832                "target",
10833            )
10834            .unwrap();
10835        forwarding
10836            .complete_pending_relay(
10837                target_ctx.connection_id,
10838                pending.corr,
10839                RouteBindRelayOutcome::Accepted,
10840            )
10841            .unwrap();
10842        let actual = query_live_roots(&handler, &target_ctx).await;
10843        let ModuleControlResponseToModule::LiveRoots {
10844            roots,
10845            unknown_root_bindings,
10846            total_bindings,
10847        } = actual
10848        else {
10849            panic!("expected live roots")
10850        };
10851        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10852        assert_eq!(
10853            unknown_root_bindings, 1,
10854            "unknown-root arm must not read as no bindings"
10855        );
10856        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10857        assert_eq!(
10858            total_bindings,
10859            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10860        );
10861    }
10862
10863    /// A module reads HELLO_ACK as its first frame and exits on anything else,
10864    /// so the ack has to be on its outbound queue before the module is
10865    /// routable. The connection loop writes a handler's replies only after the
10866    /// handler returns; this test stops in exactly that gap, runs a real
10867    /// route.open from another connection, and only then writes whatever the
10868    /// HELLO handler returned, the way the loop would. If the ack were still a
10869    /// reply, the route.bind request would reach the module first.
10870    #[tokio::test(start_paused = true)]
10871    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10872        let forwarding = Arc::new(ForwardingTable::default());
10873        let handler =
10874            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10875        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10876        let replies = handler
10877            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10878            .await
10879            .unwrap();
10880        let queued_by_hello = module_rx.len();
10881
10882        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10883        let open_handler = handler.clone();
10884        let open = tokio::spawn(async move {
10885            open_handler
10886                .handle_control_frame(
10887                    &client_ctx,
10888                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10889                )
10890                .await
10891                .unwrap()
10892        });
10893        // Let the route.open run until its route.bind is on the module's queue.
10894        let mut spins = 0;
10895        while module_rx.len() == queued_by_hello {
10896            spins += 1;
10897            assert!(spins < 10_000, "route.open never queued a route.bind");
10898            tokio::task::yield_now().await;
10899        }
10900
10901        // Now the connection loop's half: write the HELLO handler's replies.
10902        for reply in replies {
10903            module_ctx.egress.send(reply).await.unwrap();
10904        }
10905
10906        let first = module_rx.recv().await.unwrap().frame;
10907        assert_eq!(
10908            first.header.ty,
10909            FrameType::HelloAck,
10910            "the first frame a registering module reads must be its HELLO_ACK"
10911        );
10912        assert_eq!(first.header.corr, 7);
10913        let second = module_rx.recv().await.unwrap().frame;
10914        assert_eq!(second.header.ty, FrameType::Request);
10915        assert!(
10916            matches!(
10917                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10918                ModuleControlRequest::RouteBind { .. }
10919            ),
10920            "the route.bind follows the ack"
10921        );
10922        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10923
10924        handler
10925            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10926            .await
10927            .unwrap();
10928        assert!(open.await.unwrap().is_empty());
10929        let _ = client_rx.recv().await.unwrap();
10930    }
10931
10932    #[tokio::test(start_paused = true)]
10933    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10934        let handler = ControlHandler::with_forwarding(
10935            Arc::new(Registry::default()),
10936            Arc::new(ForwardingTable::default()),
10937        );
10938        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10939        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10940        hello_via_sink(
10941            &handler,
10942            &first_ctx,
10943            &mut first_rx,
10944            hello_frame("first", PROTOCOL_VERSION, 1),
10945        )
10946        .await;
10947        hello_via_sink(
10948            &handler,
10949            &second_ctx,
10950            &mut second_rx,
10951            hello_frame("second", PROTOCOL_VERSION, 2),
10952        )
10953        .await;
10954        let root = unique_project_root("second-only");
10955        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10956        let cloned = handler.clone();
10957        let open = tokio::spawn(async move {
10958            cloned
10959                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10960                .await
10961                .unwrap()
10962        });
10963        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10964            .await
10965            .unwrap()
10966            .unwrap();
10967        let first = query_live_roots(&handler, &first_ctx).await;
10968        let second = query_live_roots(&handler, &second_ctx).await;
10969        assert!(
10970            matches!(
10971                first,
10972                ModuleControlResponseToModule::LiveRoots {
10973                    total_bindings: 0,
10974                    ..
10975                }
10976            ),
10977            "cross-module scope must not expose another module's roots"
10978        );
10979        assert!(
10980            matches!(
10981                second,
10982                ModuleControlResponseToModule::LiveRoots {
10983                    total_bindings: 1,
10984                    ..
10985                }
10986            ),
10987            "second module must see its pending route"
10988        );
10989        handler
10990            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10991            .await
10992            .unwrap();
10993        assert!(open.await.unwrap().is_empty());
10994    }
10995
10996    #[tokio::test(start_paused = true)]
10997    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10998        let handler = ControlHandler::with_forwarding(
10999            Arc::new(Registry::default()),
11000            Arc::new(ForwardingTable::default()),
11001        );
11002        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11003        hello_via_sink(
11004            &handler,
11005            &target_ctx,
11006            &mut target_rx,
11007            hello_frame("target", PROTOCOL_VERSION, 1),
11008        )
11009        .await;
11010        let actual = query_live_roots(&handler, &target_ctx).await;
11011        let ModuleControlResponseToModule::LiveRoots {
11012            roots,
11013            unknown_root_bindings,
11014            total_bindings,
11015        } = actual
11016        else {
11017            panic!("expected live roots")
11018        };
11019        assert!(roots.is_empty());
11020        assert_eq!(unknown_root_bindings, 0);
11021        assert_eq!(total_bindings, 0);
11022        assert_eq!(
11023            total_bindings,
11024            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11025        );
11026    }
11027
11028    /// Read the vendored fed corpus rather than hand-building a package.
11029    ///
11030    /// A hand-built object encodes what the test author believed the carrier
11031    /// emits. These vectors are what it actually emits, and one of them exists
11032    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11033    /// ignores additive unknown fields at the traversal emit terminus."
11034    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11035        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11036            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11037        let text = std::fs::read_to_string(&path)
11038            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11039        let vectors: Vec<(String, Value)> = text
11040            .lines()
11041            .filter(|line| !line.trim().is_empty())
11042            .map(|line| {
11043                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11044                let id = entry["corpus_id"]
11045                    .as_str()
11046                    .expect("every vector carries a corpus_id")
11047                    .to_string();
11048                (id, entry["package"].clone())
11049            })
11050            .collect();
11051        // Pin the count: a corpus that silently shrinks would take its coverage
11052        // with it, and a suite reading N-1 vectors reports the same clean pass
11053        // as one reading N.
11054        assert_eq!(
11055            vectors.len(),
11056            3,
11057            "vendored fed corpus changed size; re-sync from subc-federation"
11058        );
11059
11060        // Pin what makes the corpus DISCRIMINATING, not just present.
11061        //
11062        // The relay test below takes its expected value from the corpus, so the
11063        // corpus supplies the test's power to detect a lossy relay rather than
11064        // its correctness. A relay that dropped unrecognised fields would still
11065        // be caught -- but only by a package carrying fields it does not know.
11066        // Shrink every package to the handful of keys any implementation would
11067        // recognise and the test keeps passing over an input that can no longer
11068        // fail, which is the same clean green as a corpus that shrank away.
11069        //
11070        // So assert the precondition rather than duplicating the packages here:
11071        // at least one vector must carry a field beyond the small common set.
11072        // That is one claim to maintain instead of nine, and it fails loudly if
11073        // a re-sync ever flattens the corpus.
11074        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11075        let richest = vectors
11076            .iter()
11077            .filter_map(|(_, package)| package.as_object())
11078            .map(|object| {
11079                object
11080                    .keys()
11081                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11082                    .count()
11083            })
11084            .max()
11085            .unwrap_or(0);
11086        assert!(
11087            richest >= 2,
11088            "vendored corpus no longer carries a package with unmodelled fields, \
11089             so the relay test can no longer distinguish a verbatim relay from a lossy one"
11090        );
11091
11092        vectors
11093    }
11094
11095    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
11096    ///
11097    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
11098    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
11099    /// three-key object, so a relay that quietly dropped fields it did not
11100    /// recognise would satisfy it. These vectors carry nine keys including ones
11101    /// this crate has no type for, so a typed relay fails here and only here.
11102    #[tokio::test]
11103    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11104        for (corpus_id, package) in fed_admission_facts_vectors() {
11105            let registry = Arc::new(Registry::default());
11106            let forwarding = Arc::new(ForwardingTable::default());
11107            let supervisor = SupervisorHandle::new();
11108            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11109            let handler =
11110                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11111                    .with_supervisor(supervisor)
11112                    .with_admission_facts_config(
11113                        Some("fed".to_string()),
11114                        Some(vec!["target".to_string()]),
11115                    );
11116
11117            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11118            hello_via_sink(
11119                &handler,
11120                &target_ctx,
11121                &mut target_rx,
11122                hello_frame("target", PROTOCOL_VERSION, 1),
11123            )
11124            .await;
11125
11126            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11127            let route_handler = handler.clone();
11128            let expected = package.clone();
11129            let route_task = tokio::spawn(async move {
11130                route_handler
11131                    .handle_control_frame(
11132                        &client_ctx,
11133                        route_open_frame_with_admission_facts(
11134                            20,
11135                            "target",
11136                            unique_project_root("admission-facts"),
11137                            Some(subc_control::ConsumerIdentity {
11138                                module_id: "fed".to_string(),
11139                                launch_nonce: "fed-nonce".to_string(),
11140                            }),
11141                            Some(package),
11142                        ),
11143                    )
11144                    .await
11145                    .unwrap()
11146            });
11147
11148            let bind_frame = target_rx.recv().await.unwrap();
11149            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11150            let ModuleControlRequest::RouteBind {
11151                admission_facts, ..
11152            } = bind
11153            else {
11154                panic!("{corpus_id}: expected route.bind")
11155            };
11156            assert_eq!(
11157                admission_facts,
11158                Some(expected),
11159                "{corpus_id}: relay must not add, drop or reshape any field"
11160            );
11161
11162            handler
11163                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11164                .await
11165                .unwrap();
11166            route_task.await.unwrap();
11167        }
11168    }
11169
11170    #[tokio::test]
11171    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11172        let registry = Arc::new(Registry::default());
11173        let forwarding = Arc::new(ForwardingTable::default());
11174        let supervisor = SupervisorHandle::new();
11175        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11176        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11177        let handler =
11178            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11179                .with_supervisor(supervisor)
11180                .with_admission_facts_config(
11181                    Some("fed".to_string()),
11182                    Some(vec!["target".to_string()]),
11183                );
11184
11185        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11186        hello_via_sink(
11187            &handler,
11188            &target_ctx,
11189            &mut target_rx,
11190            hello_frame("target", PROTOCOL_VERSION, 1),
11191        )
11192        .await;
11193        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11194        hello_via_sink(
11195            &handler,
11196            &other_ctx,
11197            &mut other_rx,
11198            hello_frame("other", PROTOCOL_VERSION, 2),
11199        )
11200        .await;
11201
11202        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11203        let expected_facts = facts.clone();
11204        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11205        let route_handler = handler.clone();
11206        let route_task = tokio::spawn(async move {
11207            route_handler
11208                .handle_control_frame(
11209                    &client_ctx,
11210                    route_open_frame_with_admission_facts(
11211                        10,
11212                        "target",
11213                        unique_project_root("admission-facts"),
11214                        Some(subc_control::ConsumerIdentity {
11215                            module_id: "fed".to_string(),
11216                            launch_nonce: "fed-nonce".to_string(),
11217                        }),
11218                        Some(facts.clone()),
11219                    ),
11220                )
11221                .await
11222                .unwrap()
11223        });
11224        let bind_frame = target_rx.recv().await.unwrap();
11225        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11226        let ModuleControlRequest::RouteBind {
11227            admission_facts, ..
11228        } = bind
11229        else {
11230            panic!("expected route.bind")
11231        };
11232        assert_eq!(admission_facts, Some(expected_facts));
11233        handler
11234            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11235            .await
11236            .unwrap();
11237        assert!(route_task.await.unwrap().is_empty());
11238        assert!(matches!(
11239            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11240                .unwrap(),
11241            ClientControlResponse::RouteOpen { .. }
11242        ));
11243
11244        let direct = handler
11245            .handle_control_frame(
11246                &route_ctx(ConnectionId::new(73)).0,
11247                route_open_frame_with_admission_facts(
11248                    11,
11249                    "target",
11250                    unique_project_root("admission-facts"),
11251                    None,
11252                    Some(json!({"x": 1})),
11253                ),
11254            )
11255            .await
11256            .unwrap();
11257        assert_eq!(
11258            parse_error(&direct[0])["code"],
11259            "admission_facts_not_permitted"
11260        );
11261
11262        let different_reserved = handler
11263            .handle_control_frame(
11264                &route_ctx(ConnectionId::new(77)).0,
11265                route_open_frame_with_admission_facts(
11266                    15,
11267                    "target",
11268                    unique_project_root("admission-facts"),
11269                    Some(subc_control::ConsumerIdentity {
11270                        module_id: "other".to_string(),
11271                        launch_nonce: "other-nonce".to_string(),
11272                    }),
11273                    Some(json!({"x": 1})),
11274                ),
11275            )
11276            .await
11277            .unwrap();
11278        assert_eq!(
11279            parse_error(&different_reserved[0])["code"],
11280            "admission_facts_not_permitted"
11281        );
11282
11283        let other_target = handler
11284            .handle_control_frame(
11285                &route_ctx(ConnectionId::new(74)).0,
11286                route_open_frame_with_admission_facts(
11287                    12,
11288                    "other",
11289                    unique_project_root("admission-facts"),
11290                    Some(subc_control::ConsumerIdentity {
11291                        module_id: "fed".to_string(),
11292                        launch_nonce: "fed-nonce".to_string(),
11293                    }),
11294                    Some(json!({"x": 1})),
11295                ),
11296            )
11297            .await
11298            .unwrap();
11299        assert_eq!(
11300            parse_error(&other_target[0])["code"],
11301            "admission_facts_target_not_allowed"
11302        );
11303
11304        let nonexistent = handler
11305            .handle_control_frame(
11306                &route_ctx(ConnectionId::new(75)).0,
11307                route_open_frame_with_admission_facts(
11308                    13,
11309                    "missing",
11310                    unique_project_root("admission-facts"),
11311                    None,
11312                    Some(json!({"x": 1})),
11313                ),
11314            )
11315            .await
11316            .unwrap();
11317        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11318
11319        let described = handler
11320            .handle_control_frame(
11321                &route_ctx(ConnectionId::new(76)).0,
11322                Frame::build(
11323                    FrameType::Request,
11324                    control_flags(),
11325                    0,
11326                    0,
11327                    14,
11328                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11329                )
11330                .unwrap(),
11331            )
11332            .await
11333            .unwrap();
11334        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11335            serde_json::from_slice(&described[0].body).unwrap()
11336        else {
11337            panic!("expected server.describe response")
11338        };
11339        assert!(capabilities
11340            .iter()
11341            .any(|cap| cap == "admission_facts_relay_v1"));
11342    }
11343
11344    #[tokio::test]
11345    async fn admission_facts_without_configured_carrier_are_rejected() {
11346        let registry = Arc::new(Registry::default());
11347        let forwarding = Arc::new(ForwardingTable::default());
11348        let handler = ControlHandler::with_forwarding(registry, forwarding);
11349        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11350        hello_via_sink(
11351            &handler,
11352            &target_ctx,
11353            &mut target_rx,
11354            hello_frame("target", PROTOCOL_VERSION, 1),
11355        )
11356        .await;
11357
11358        let responses = handler
11359            .handle_control_frame(
11360                &route_ctx(ConnectionId::new(79)).0,
11361                route_open_frame_with_admission_facts(
11362                    16,
11363                    "target",
11364                    unique_project_root("admission-facts"),
11365                    None,
11366                    Some(json!({"x": 1})),
11367                ),
11368            )
11369            .await
11370            .unwrap();
11371        assert_eq!(
11372            parse_error(&responses[0])["code"],
11373            "admission_facts_not_permitted"
11374        );
11375    }
11376
11377    #[tokio::test]
11378    async fn route_open_relays_consumer_capabilities_verbatim() {
11379        let registry = Arc::new(Registry::default());
11380        let forwarding = Arc::new(ForwardingTable::default());
11381        let handler =
11382            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11383        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11384        hello_via_sink(
11385            &handler,
11386            &module_ctx,
11387            &mut module_rx,
11388            hello_frame("aft", PROTOCOL_VERSION, 7),
11389        )
11390        .await;
11391
11392        let expected = vec!["elicitation".to_string(), "roots".to_string()];
11393        let expected_for_request = expected.clone();
11394        let project_root = unique_project_root("consumer-capabilities-present");
11395        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11396        let route_handler = handler.clone();
11397        let route_task = tokio::spawn(async move {
11398            route_handler
11399                .handle_control_frame(
11400                    &client_ctx,
11401                    route_open_frame_with_consumer_capabilities(
11402                        401,
11403                        "aft",
11404                        project_root,
11405                        Some(expected_for_request),
11406                    ),
11407                )
11408                .await
11409                .unwrap()
11410        });
11411        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11412            .await
11413            .unwrap()
11414            .unwrap();
11415        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11416        let ModuleControlRequest::RouteBind {
11417            consumer_capabilities,
11418            ..
11419        } = bind
11420        else {
11421            panic!("expected route.bind request, got {bind:?}");
11422        };
11423        assert_eq!(consumer_capabilities, Some(expected.clone()));
11424
11425        handler
11426            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11427            .await
11428            .unwrap();
11429        let route_response = route_task.await.unwrap();
11430        assert!(route_response.is_empty());
11431        let published = client_rx.recv().await.unwrap();
11432        assert!(matches!(
11433            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11434            ClientControlResponse::RouteOpen { .. }
11435        ));
11436    }
11437
11438    #[tokio::test]
11439    async fn route_open_without_consumer_capabilities_relays_none() {
11440        let registry = Arc::new(Registry::default());
11441        let forwarding = Arc::new(ForwardingTable::default());
11442        let handler =
11443            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11444        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11445        hello_via_sink(
11446            &handler,
11447            &module_ctx,
11448            &mut module_rx,
11449            hello_frame("aft", PROTOCOL_VERSION, 7),
11450        )
11451        .await;
11452
11453        let project_root = unique_project_root("consumer-capabilities-absent");
11454        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11455        let route_handler = handler.clone();
11456        let route_task = tokio::spawn(async move {
11457            route_handler
11458                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11459                .await
11460                .unwrap()
11461        });
11462        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11463            .await
11464            .unwrap()
11465            .unwrap();
11466        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11467        let ModuleControlRequest::RouteBind {
11468            consumer_capabilities,
11469            ..
11470        } = bind
11471        else {
11472            panic!("expected route.bind request, got {bind:?}");
11473        };
11474        assert_eq!(consumer_capabilities, None);
11475
11476        handler
11477            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11478            .await
11479            .unwrap();
11480        let route_response = route_task.await.unwrap();
11481        assert!(route_response.is_empty());
11482        let published = client_rx.recv().await.unwrap();
11483        assert!(matches!(
11484            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11485            ClientControlResponse::RouteOpen { .. }
11486        ));
11487    }
11488
11489    /// Opens a route to a freshly registered `aft` with `sent` as the
11490    /// route.open's role_versions, acks the bind, and returns the role_versions
11491    /// the module's bind carried.
11492    async fn bind_role_versions_for(
11493        sent: Option<BTreeMap<String, String>>,
11494        connection: u64,
11495    ) -> Option<BTreeMap<String, String>> {
11496        let registry = Arc::new(Registry::default());
11497        let forwarding = Arc::new(ForwardingTable::default());
11498        let handler =
11499            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11500        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11501        hello_via_sink(
11502            &handler,
11503            &module_ctx,
11504            &mut module_rx,
11505            hello_frame("aft", PROTOCOL_VERSION, 7),
11506        )
11507        .await;
11508        let project_root = unique_project_root("role-versions");
11509        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11510        let route_handler = handler.clone();
11511        let route_task = tokio::spawn(async move {
11512            route_handler
11513                .handle_control_frame(
11514                    &client_ctx,
11515                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
11516                )
11517                .await
11518                .unwrap()
11519        });
11520        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11521            .await
11522            .expect("a well-formed route.open reaches the module as a bind")
11523            .unwrap();
11524        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11525        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11526            panic!("expected route.bind request, got {bind:?}");
11527        };
11528        handler
11529            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11530            .await
11531            .unwrap();
11532        assert!(route_task.await.unwrap().is_empty());
11533        let published = client_rx.recv().await.unwrap();
11534        assert!(matches!(
11535            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11536            ClientControlResponse::RouteOpen { .. }
11537        ));
11538        role_versions
11539    }
11540
11541    #[tokio::test]
11542    async fn route_open_relays_role_versions_verbatim() {
11543        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11544        assert_eq!(
11545            bind_role_versions_for(Some(sent.clone()), 141).await,
11546            Some(sent)
11547        );
11548    }
11549
11550    /// An empty map declares nothing, so the provider sees no field rather
11551    /// than an empty object it would have to treat as a second "none".
11552    #[tokio::test]
11553    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11554        assert_eq!(bind_role_versions_for(None, 143).await, None);
11555        assert_eq!(
11556            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11557            None
11558        );
11559    }
11560
11561    #[tokio::test]
11562    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11563        let registry = Arc::new(Registry::default());
11564        let forwarding = Arc::new(ForwardingTable::default());
11565        let handler =
11566            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11567        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11568        hello_via_sink(
11569            &handler,
11570            &module_ctx,
11571            &mut module_rx,
11572            hello_frame("aft", PROTOCOL_VERSION, 7),
11573        )
11574        .await;
11575
11576        let nine: BTreeMap<String, String> = (0..9)
11577            .map(|index| (format!("role-{index}"), "v1".to_string()))
11578            .collect();
11579        for (label, malformed) in [
11580            (
11581                "invalid role name",
11582                role_versions(&[("Tool_Provider", "v1")]),
11583            ),
11584            ("invalid version", role_versions(&[("tool-provider", "v0")])),
11585            ("nine entries", nine),
11586        ] {
11587            // A refused open answers at once; one that reached the module
11588            // would wait for its bind ack and trip this timeout.
11589            let responses = tokio::time::timeout(
11590                Duration::from_secs(1),
11591                handler.handle_control_frame(
11592                    &route_ctx(ConnectionId::new(148)).0,
11593                    route_open_frame_with_role_versions(
11594                        404,
11595                        "aft",
11596                        unique_project_root("role-versions-malformed"),
11597                        Some(malformed),
11598                    ),
11599                ),
11600            )
11601            .await
11602            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11603            .unwrap();
11604            assert_eq!(responses.len(), 1, "{label}");
11605            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11606            let error = parse_error(&responses[0]);
11607            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11608            assert_eq!(
11609                error["detail"]["field"], "role_versions",
11610                "{label}: {error}"
11611            );
11612            assert!(
11613                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11614                "{label}: a malformed declaration is terminal"
11615            );
11616            assert!(
11617                module_rx.try_recv().is_err(),
11618                "{label}: the module must never see a bind"
11619            );
11620        }
11621    }
11622
11623    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
11624    /// consumer can tell this daemon forwards the field from one that would
11625    /// drop it.
11626    #[tokio::test]
11627    async fn route_role_versions_capability_is_advertised() {
11628        let handler = ControlHandler::new(Arc::new(Registry::default()));
11629        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11630        let ack = hello_via_sink(
11631            &handler,
11632            &ctx,
11633            &mut rx,
11634            hello_frame("m", PROTOCOL_VERSION, 1),
11635        )
11636        .await;
11637        let ack = parse_ack(&ack);
11638        assert!(
11639            ack.subc_capabilities
11640                .iter()
11641                .any(|c| c == "route-role-versions/v1"),
11642            "{:?}",
11643            ack.subc_capabilities
11644        );
11645
11646        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11647        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11648        let reply = handler
11649            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11650            .await
11651            .unwrap()
11652            .pop()
11653            .unwrap();
11654        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11655            serde_json::from_slice(&reply.body).unwrap()
11656        else {
11657            panic!("not a server.describe reply");
11658        };
11659        assert!(
11660            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11661            "{capabilities:?}"
11662        );
11663    }
11664
11665    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
11666    /// catalog entry without `events` (the module declares none) from a daemon
11667    /// that never reports declared events.
11668    #[tokio::test]
11669    async fn catalog_events_capability_is_advertised() {
11670        let handler = ControlHandler::new(Arc::new(Registry::default()));
11671        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11672        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11673        let reply = handler
11674            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11675            .await
11676            .unwrap()
11677            .pop()
11678            .unwrap();
11679        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11680            serde_json::from_slice(&reply.body).unwrap()
11681        else {
11682            panic!("not a server.describe reply");
11683        };
11684        assert!(
11685            capabilities
11686                .iter()
11687                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11688            "{capabilities:?}"
11689        );
11690    }
11691
11692    #[tokio::test]
11693    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11694        let handler = ControlHandler::new(Arc::new(Registry::default()));
11695        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11696        let ack = hello_via_sink(
11697            &handler,
11698            &module,
11699            &mut module_rx,
11700            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11701        )
11702        .await;
11703        let ack = parse_ack(&ack);
11704        assert!(
11705            ack.subc_capabilities
11706                .iter()
11707                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11708            "{:?}",
11709            ack.subc_capabilities
11710        );
11711        assert!(ack
11712            .subc_capabilities
11713            .iter()
11714            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11715
11716        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11717        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11718        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11719        let reply = handler
11720            .handle_control_frame(&client, frame)
11721            .await
11722            .unwrap()
11723            .pop()
11724            .unwrap();
11725        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11726            serde_json::from_slice(&reply.body).unwrap()
11727        else {
11728            panic!("not a server.describe reply");
11729        };
11730        assert!(
11731            capabilities
11732                .iter()
11733                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11734            "{capabilities:?}"
11735        );
11736        assert!(capabilities
11737            .iter()
11738            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11739    }
11740
11741    #[tokio::test]
11742    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
11743        let handler = ControlHandler::new(Arc::new(Registry::default()));
11744        let (module, mut rx) = route_ctx(ConnectionId::new(156));
11745        let ack = hello_via_sink(
11746            &handler,
11747            &module,
11748            &mut rx,
11749            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
11750        )
11751        .await;
11752        assert!(parse_ack(&ack)
11753            .subc_capabilities
11754            .iter()
11755            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11756        assert!(parse_ack(&ack)
11757            .subc_capabilities
11758            .iter()
11759            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11760        let (client, _rx) = route_ctx(ConnectionId::new(157));
11761        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11762        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11763        let reply = handler
11764            .handle_control_frame(&client, frame)
11765            .await
11766            .unwrap()
11767            .pop()
11768            .unwrap();
11769        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11770            serde_json::from_slice(&reply.body).unwrap()
11771        else {
11772            panic!("server.describe reply")
11773        };
11774        assert!(capabilities
11775            .iter()
11776            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11777        assert!(capabilities
11778            .iter()
11779            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11780    }
11781
11782    #[tokio::test]
11783    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11784        let handler = ControlHandler::new(Arc::new(Registry::default()));
11785        let (module, mut rx) = route_ctx(ConnectionId::new(154));
11786        let ack = hello_via_sink(
11787            &handler,
11788            &module,
11789            &mut rx,
11790            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11791        )
11792        .await;
11793        assert!(parse_ack(&ack)
11794            .subc_capabilities
11795            .iter()
11796            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11797        let (client, _rx) = route_ctx(ConnectionId::new(155));
11798        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11799        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11800        let reply = handler
11801            .handle_control_frame(&client, frame)
11802            .await
11803            .unwrap()
11804            .pop()
11805            .unwrap();
11806        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11807            serde_json::from_slice(&reply.body).unwrap()
11808        else {
11809            panic!("not a server.describe reply")
11810        };
11811        assert!(capabilities
11812            .iter()
11813            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11814    }
11815
11816    #[tokio::test]
11817    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11818        let registry = Arc::new(Registry::default());
11819        let forwarding = Arc::new(ForwardingTable::default());
11820        let handler =
11821            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11822                .with_health_probe_timeout(Duration::from_secs(5));
11823        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11824        hello_via_sink(
11825            &handler,
11826            &module_ctx,
11827            &mut module_rx,
11828            non_routable_hello_frame_with_control_ops(
11829                "mcp",
11830                300,
11831                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11832            ),
11833        )
11834        .await;
11835        assert!(registry
11836            .get_module("mcp")
11837            .unwrap()
11838            .unwrap()
11839            .manifest
11840            .provides
11841            .is_empty());
11842
11843        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11844        let route_response = handler
11845            .handle_control_frame(
11846                &route_client_ctx,
11847                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11848            )
11849            .await
11850            .unwrap();
11851        assert_eq!(route_response[0].header.ty, FrameType::Error);
11852        assert_eq!(
11853            parse_error(&route_response[0])["code"],
11854            "target_unavailable"
11855        );
11856        assert!(parse_error(&route_response[0])["message"]
11857            .as_str()
11858            .unwrap()
11859            .contains("does not provide the requested target"));
11860        assert!(module_rx.try_recv().is_err());
11861
11862        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11863        let health_handler = handler.clone();
11864        let health_task = tokio::spawn(async move {
11865            health_handler
11866                .handle_control_frame(
11867                    &health_client_ctx,
11868                    supervisor_health_probe_frame(302, "mcp"),
11869                )
11870                .await
11871                .unwrap()
11872        });
11873        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11874            .await
11875            .unwrap()
11876            .unwrap();
11877        assert_eq!(
11878            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11879            ModuleControlRequest::HealthCheck {}
11880        );
11881        handler
11882            .handle_control_frame(
11883                &module_ctx,
11884                health_response(health_frame.header.corr, HealthStatus::Ok),
11885            )
11886            .await
11887            .unwrap();
11888        let health_response = health_task.await.unwrap();
11889        assert_eq!(health_response[0].header.ty, FrameType::Response);
11890        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11891            ClientControlResponse::SupervisorHealthProbe {
11892                module_id, status, ..
11893            } => {
11894                assert_eq!(module_id, "mcp");
11895                assert_eq!(status, HealthStatus::Ok);
11896            }
11897            other => panic!("unexpected health response: {other:?}"),
11898        }
11899
11900        // Exercise the forwarding cleanup path directly while leaving the registry
11901        // advertisement in place. If cleanup leaves a stale control sink behind,
11902        // the next probe will enqueue onto it and wait for the long probe timeout
11903        // instead of returning an immediate no-connection error.
11904        forwarding
11905            .cleanup_connection(module_ctx.connection_id)
11906            .unwrap();
11907        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11908        let cleanup_response = tokio::time::timeout(
11909            Duration::from_millis(200),
11910            handler.handle_control_frame(
11911                &cleanup_probe_ctx,
11912                supervisor_health_probe_frame(303, "mcp"),
11913            ),
11914        )
11915        .await
11916        .expect("probe should fail immediately when the control lane is gone")
11917        .unwrap();
11918        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11919        assert_eq!(
11920            parse_error(&cleanup_response[0])["code"],
11921            "target_unavailable"
11922        );
11923        assert!(parse_error(&cleanup_response[0])["message"]
11924            .as_str()
11925            .unwrap()
11926            .contains("no module connection"));
11927
11928        handler
11929            .cleanup_connection(module_ctx.connection_id)
11930            .unwrap();
11931    }
11932
11933    #[tokio::test]
11934    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11935        let registry = Arc::new(Registry::default());
11936        let supervisor_handle = SupervisorHandle::new();
11937        let supervisor =
11938            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
11939                .with_handle(supervisor_handle.clone())
11940                .with_connection_file_path(
11941                    std::env::temp_dir()
11942                        .join(format!("subc-route-open-warming-{}", std::process::id())),
11943                );
11944        let module = supervisor
11945            .supervise_configured(
11946                ModuleSpec {
11947                    module_id: "warming".to_string(),
11948                    program: fake_aft_stub_path(),
11949                    args: Vec::new(),
11950                    env: Vec::new(),
11951                    reserved: false,
11952                    reserved_prefixes: Vec::new(),
11953                    protocol: ModuleProtocol::Subc,
11954                    overlap: Default::default(),
11955                },
11956                true,
11957            )
11958            .unwrap();
11959        assert_eq!(module.state().unwrap(), ModuleState::Running);
11960
11961        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11962        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11963        let response = handler
11964            .handle_control_frame(
11965                &ctx,
11966                route_open_frame(304, "warming", unique_project_root("warming")),
11967            )
11968            .await
11969            .unwrap();
11970        module.stop().await.unwrap();
11971
11972        assert_eq!(response[0].header.ty, FrameType::Error);
11973        let error = parse_error(&response[0]);
11974        assert_eq!(error["code"], "module_warming");
11975        assert!(error["message"]
11976            .as_str()
11977            .unwrap()
11978            .contains("state=running, enabled=true, live=false"));
11979    }
11980
11981    #[test]
11982    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11983        let handler = ControlHandler::new(Arc::new(Registry::default()));
11984        let capture = EventCapture::default();
11985        let _subscriber =
11986            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11987        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11988        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11989        let pending = (0..limit).collect::<Vec<_>>();
11990        let response = handler
11991            .route_open_capacity_refusal(
11992                &ctx,
11993                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11994                "busy",
11995                pending.len(),
11996                limit,
11997            )
11998            .unwrap();
11999        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12000        let event = capture
12001            .events()
12002            .into_iter()
12003            .find(|event| {
12004                event.target == "control"
12005                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12006            })
12007            .expect("connection admission refusal event");
12008        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12009        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12010    }
12011
12012    #[test]
12013    fn route_open_target_cap_logs_admission_reason_and_capacity() {
12014        let handler = ControlHandler::new(Arc::new(Registry::default()));
12015        let capture = EventCapture::default();
12016        let _subscriber =
12017            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12018        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12019        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12020        let guards = (0..limit)
12021            .map(|_| {
12022                handler
12023                    .route_bind_concurrency
12024                    .try_admit("busy", limit)
12025                    .unwrap()
12026            })
12027            .collect::<Vec<_>>();
12028        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12029            Err(in_flight) => in_flight,
12030            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12031        };
12032        let response = handler
12033            .route_open_target_capacity_refusal(
12034                &ctx,
12035                &route_open_frame(397, "busy", unique_project_root("target-cap")),
12036                "busy",
12037                in_flight,
12038            )
12039            .unwrap();
12040        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12041        let event = capture
12042            .events()
12043            .into_iter()
12044            .find(|event| {
12045                event.target == "control"
12046                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12047            })
12048            .expect("target admission refusal event");
12049        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12050        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12051        drop(guards);
12052    }
12053
12054    #[tokio::test(flavor = "current_thread")]
12055    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12056        let handler = ControlHandler::new(Arc::new(Registry::default()));
12057        let capture = EventCapture::default();
12058        let _subscriber =
12059            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12060        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12061        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12062
12063        let _ = handler
12064            .handle_client_control_request(
12065                &ctx,
12066                frame,
12067                ClientControlRequest::SupervisorSetEnabled {
12068                    module_id: "broca".to_string(),
12069                    enabled: false,
12070                },
12071                &mut None,
12072            )
12073            .await
12074            .unwrap();
12075
12076        let events = capture
12077            .events()
12078            .into_iter()
12079            .filter(|event| {
12080                event.target == "control"
12081                    && event.fields.get("message").map(String::as_str)
12082                        == Some("supervisor request received")
12083            })
12084            .collect::<Vec<_>>();
12085        assert_eq!(events.len(), 1, "one supervisor request log line");
12086        let event = &events[0];
12087        assert_eq!(event.level, tracing::Level::INFO);
12088        assert_eq!(
12089            event.fields.get("op"),
12090            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12091        );
12092        assert_eq!(
12093            event.fields.get("module_id"),
12094            Some(&"\"broca\"".to_string())
12095        );
12096        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12097        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12098        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12099    }
12100
12101    #[tokio::test(flavor = "current_thread")]
12102    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12103        let registry = Arc::new(Registry::default());
12104        let module_connection = ConnectionId::new(303);
12105        registry
12106            .register_with_control_ops(
12107                manifest("aft", PROTOCOL_VERSION),
12108                PROTOCOL_VERSION,
12109                module_connection,
12110                module_baseline_control_ops(),
12111            )
12112            .unwrap();
12113        let handler = ControlHandler::new(registry);
12114        let capture = EventCapture::default();
12115        let _subscriber =
12116            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12117        let (ctx, _rx) = route_ctx(module_connection);
12118        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12119
12120        let _ = handler
12121            .handle_client_control_request(
12122                &ctx,
12123                frame,
12124                ClientControlRequest::SupervisorSetEnabled {
12125                    module_id: "broca".to_string(),
12126                    enabled: false,
12127                },
12128                &mut None,
12129            )
12130            .await
12131            .unwrap();
12132
12133        let callers = capture
12134            .events()
12135            .into_iter()
12136            .filter(|event| {
12137                event.fields.get("message").map(String::as_str)
12138                    == Some("supervisor request received")
12139            })
12140            .map(|event| event.fields.get("caller").cloned())
12141            .collect::<Vec<_>>();
12142        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12143    }
12144
12145    #[tokio::test(flavor = "current_thread")]
12146    async fn supervisor_list_does_not_log_request_received() {
12147        let handler = ControlHandler::new(Arc::new(Registry::default()));
12148        let capture = EventCapture::default();
12149        let _subscriber =
12150            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12151        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12152        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12153
12154        handler
12155            .handle_client_control_request(
12156                &ctx,
12157                frame,
12158                ClientControlRequest::SupervisorList {},
12159                &mut None,
12160            )
12161            .await
12162            .unwrap();
12163
12164        assert!(capture.events().into_iter().all(|event| {
12165            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12166        }));
12167    }
12168
12169    #[tokio::test(flavor = "current_thread")]
12170    async fn supervisor_rescan_preview_does_not_log_request_received() {
12171        let handler = ControlHandler::new(Arc::new(Registry::default()));
12172        let capture = EventCapture::default();
12173        let _subscriber =
12174            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12175        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12176        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12177
12178        handler
12179            .handle_client_control_request(
12180                &ctx,
12181                frame,
12182                ClientControlRequest::SupervisorRescan { preview: true },
12183                &mut None,
12184            )
12185            .await
12186            .unwrap();
12187
12188        assert!(capture.events().into_iter().all(|event| {
12189            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12190        }));
12191    }
12192
12193    struct ApproveOperator;
12194    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12195        fn prompt(
12196            &self,
12197            _: &str,
12198            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12199        ) -> crate::operator_confirm::ProviderResult {
12200            crate::operator_confirm::ProviderResult::Approved
12201        }
12202    }
12203
12204    #[tokio::test]
12205    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12206        assert_eq!(
12207            ControlHandler::default().privileged_operator_prompt,
12208            PrivilegedOperatorPrompt::Log
12209        );
12210        for (initial, changed, running) in [
12211            ("log", "enforce", PrivilegedOperatorPrompt::Log),
12212            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12213        ] {
12214            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12215            let path = dir.join("subc.jsonc");
12216            std::fs::write(
12217                &path,
12218                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12219            )
12220            .unwrap();
12221            let config = crate::daemon_config::load(&path).unwrap().unwrap();
12222            let registry = Arc::new(Registry::default());
12223            let handle = SupervisorHandle::new();
12224            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12225                .with_handle(handle.clone());
12226            let handler = ControlHandler::new(registry)
12227                .with_supervisor(handle)
12228                .with_privileged_operator_prompt(config.privileged_operator_prompt)
12229                .with_supervisor_rescan(supervisor, &path, config.port);
12230            handler.forwarding.operator_confirms().configure(
12231                Arc::new(ApproveOperator),
12232                crate::operator_confirm::OperatorLimits::default(),
12233            );
12234            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12235
12236            for (disk_mode, needs_restart) in [
12237                (Some(initial), false),
12238                (Some(changed), true),
12239                (Some(changed), true),
12240                (None, running == PrivilegedOperatorPrompt::Enforce),
12241                (Some(initial), false),
12242            ] {
12243                let mut doc = json!({"version":1});
12244                if let Some(mode) = disk_mode {
12245                    doc["privileged_operator_prompt"] = json!(mode);
12246                }
12247                std::fs::write(&path, doc.to_string()).unwrap();
12248                for preview in [true, false] {
12249                    let request = ClientControlRequest::SupervisorRescan { preview };
12250                    let frame = Frame::build(
12251                        FrameType::Request,
12252                        control_flags(),
12253                        0,
12254                        0,
12255                        1,
12256                        serde_json::to_vec(&request).unwrap(),
12257                    )
12258                    .unwrap();
12259                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12260                    assert_eq!(frames.len(), 1);
12261                    let ClientControlResponse::SupervisorRescan { result } =
12262                        serde_json::from_slice(&frames[0].body).unwrap()
12263                    else {
12264                        panic!("expected a rescan response");
12265                    };
12266                    let expected = if needs_restart {
12267                        vec!["privileged_operator_prompt"]
12268                    } else {
12269                        Vec::new()
12270                    };
12271                    assert_eq!(result.restart_required, expected);
12272                    assert_eq!(handler.privileged_operator_prompt, running);
12273                    assert_eq!(
12274                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12275                        running
12276                    );
12277                }
12278            }
12279        }
12280    }
12281
12282    #[tokio::test]
12283    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12284        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12285        let path = dir.join("subc.jsonc");
12286        let registry = Arc::new(Registry::default());
12287        let handle = SupervisorHandle::new();
12288        let supervisor =
12289            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12290        let handler = ControlHandler::new(registry)
12291            .with_supervisor(handle)
12292            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12293            .with_supervisor_rescan(supervisor, &path, None);
12294        handler.forwarding.operator_confirms().configure(
12295            Arc::new(ApproveOperator),
12296            crate::operator_confirm::OperatorLimits::default(),
12297        );
12298        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12299        for preview in [true, false] {
12300            std::fs::write(
12301                &path,
12302                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12303            )
12304            .unwrap();
12305            let request = ClientControlRequest::SupervisorRescan { preview };
12306            let frame = Frame::build(
12307                FrameType::Request,
12308                control_flags(),
12309                0,
12310                0,
12311                1,
12312                serde_json::to_vec(&request).unwrap(),
12313            )
12314            .unwrap();
12315            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12316            assert_eq!(frames.len(), 1);
12317            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12318            assert_eq!(
12319                handler.privileged_operator_prompt,
12320                PrivilegedOperatorPrompt::Enforce
12321            );
12322        }
12323    }
12324
12325    /// One wire code has several senders, so the refusal line names the check
12326    /// that refused. This drives the shared refusal path for ordinary refusals
12327    /// with an unregistered
12328    /// target and requires the branch label on the event.
12329    #[tokio::test]
12330    async fn route_open_refusal_names_the_check_that_refused() {
12331        let handler = ControlHandler::new(Arc::new(Registry::default()));
12332        let capture = EventCapture::default();
12333        let _subscriber =
12334            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12335        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12336        let response = handler
12337            .handle_control_frame(
12338                &ctx,
12339                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12340            )
12341            .await
12342            .unwrap();
12343
12344        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12345        let event = capture
12346            .events()
12347            .into_iter()
12348            .find(|event| {
12349                event.target == "control"
12350                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12351            })
12352            .expect("route.open refusal event");
12353        assert_eq!(
12354            event.fields.get("reason"),
12355            Some(&"\"not_registered\"".to_string())
12356        );
12357    }
12358
12359    #[tokio::test]
12360    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12361        let registry = Arc::new(Registry::default());
12362        let supervisor_handle = SupervisorHandle::new();
12363        let supervisor =
12364            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12365                .with_handle(supervisor_handle.clone())
12366                .with_connection_file_path(std::env::temp_dir().join(format!(
12367                    "subc-route-open-refusal-info-{}",
12368                    std::process::id()
12369                )));
12370        let module = supervisor
12371            .supervise_configured(
12372                ModuleSpec {
12373                    module_id: "warming".to_string(),
12374                    program: fake_aft_stub_path(),
12375                    args: Vec::new(),
12376                    env: Vec::new(),
12377                    reserved: false,
12378                    reserved_prefixes: Vec::new(),
12379                    protocol: ModuleProtocol::Subc,
12380                    overlap: Default::default(),
12381                },
12382                true,
12383            )
12384            .unwrap();
12385        assert_eq!(module.state().unwrap(), ModuleState::Running);
12386
12387        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12388        assert!(handler
12389            .counters()
12390            .snapshot()
12391            .get("route_open_refused_by_code")
12392            .is_none());
12393        let capture = EventCapture::default();
12394        let _subscriber =
12395            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12396        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12397        let response = handler
12398            .handle_control_frame(
12399                &ctx,
12400                route_open_frame(394, "warming", unique_project_root("refusal-info")),
12401            )
12402            .await
12403            .unwrap();
12404        module.stop().await.unwrap();
12405
12406        assert_eq!(parse_error(&response[0])["code"], "module_warming");
12407        let event = capture
12408            .events()
12409            .into_iter()
12410            .find(|event| {
12411                event.target == "control"
12412                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12413            })
12414            .expect("route.open refusal event");
12415        assert_eq!(
12416            event.fields.get("module_id"),
12417            Some(&"\"warming\"".to_string())
12418        );
12419        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12420        assert_eq!(
12421            event.fields.get("reason"),
12422            Some(&"\"supervised_not_registered\"".to_string())
12423        );
12424        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12425        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12426        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12427        assert_eq!(
12428            handler.counters().snapshot()["route_open_refused_by_code"],
12429            json!({ "module_warming": 1 })
12430        );
12431    }
12432
12433    const OUTAGE_START: &str = "route.open refusing module: not serving";
12434    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12435
12436    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12437        capture
12438            .events()
12439            .into_iter()
12440            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12441            .collect()
12442    }
12443
12444    fn supervise_stub(
12445        registry: &Arc<Registry>,
12446        module_id: &str,
12447        enabled: bool,
12448    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12449        let supervisor_handle = SupervisorHandle::new();
12450        let supervisor =
12451            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12452                .with_handle(supervisor_handle.clone())
12453                .with_connection_file_path(std::env::temp_dir().join(format!(
12454                    "subc-route-outage-{module_id}-{}",
12455                    std::process::id()
12456                )));
12457        let module = supervisor
12458            .supervise_configured(
12459                ModuleSpec {
12460                    module_id: module_id.to_string(),
12461                    program: fake_aft_stub_path(),
12462                    args: Vec::new(),
12463                    env: Vec::new(),
12464                    reserved: false,
12465                    reserved_prefixes: Vec::new(),
12466                    protocol: ModuleProtocol::Subc,
12467                    overlap: Default::default(),
12468                },
12469                enabled,
12470            )
12471            .unwrap();
12472        (supervisor_handle, module)
12473    }
12474
12475    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12476        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12477            module_id: module_id.to_string(),
12478            drain_timeout_ms: Some(50),
12479        })
12480        .unwrap();
12481        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12482    }
12483
12484    /// Two handlers built over one forwarding table must share one outage
12485    /// tracker; separate trackers would each log their own opening line for
12486    /// the same outage.
12487    #[test]
12488    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12489        let registry = Arc::new(Registry::default());
12490        let forwarding = Arc::new(ForwardingTable::default());
12491        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12492        let second = ControlHandler::with_forwarding(registry, forwarding);
12493        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12494    }
12495
12496    /// A client can name any module id it likes. Refusing an unknown one,
12497    /// however often, must not create outage state or outage lines, or the
12498    /// tracker would be a memory sink any client could fill.
12499    #[tokio::test(flavor = "current_thread")]
12500    async fn route_open_unknown_module_refusals_add_no_outage_state() {
12501        let handler = ControlHandler::new(Arc::new(Registry::default()));
12502        let capture = EventCapture::default();
12503        let _subscriber =
12504            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12505        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12506        for corr in 0..8 {
12507            let response = handler
12508                .handle_control_frame(
12509                    &ctx,
12510                    route_open_frame(
12511                        380 + corr,
12512                        &format!("nobody-{corr}"),
12513                        unique_project_root("outage-unknown"),
12514                    ),
12515                )
12516                .await
12517                .unwrap();
12518            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12519        }
12520
12521        assert_eq!(handler.route_outages.tracked_module_count(), 0);
12522        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12523        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12524    }
12525
12526    /// Drives the refusal path end to end: a supervised module that served
12527    /// before and stopped being registered with no instruction to stop is a
12528    /// WARN, and the same module refused after an operator `supervisor.restart`
12529    /// is an INFO.
12530    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12531    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12532        let registry = Arc::new(Registry::default());
12533        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
12534        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12535        let capture = EventCapture::default();
12536        let _subscriber =
12537            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12538        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12539        // The stub never registers, so pretend it served once: otherwise every
12540        // refusal would fall in its startup window.
12541        handler.route_outages.record_accepted("outage-restart");
12542
12543        let response = handler
12544            .handle_control_frame(
12545                &ctx,
12546                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12547            )
12548            .await
12549            .unwrap();
12550        assert_eq!(response[0].header.ty, FrameType::Error);
12551        let starts = outage_lines(&capture, OUTAGE_START);
12552        assert_eq!(starts.len(), 1, "{starts:?}");
12553        assert_eq!(starts[0].level, tracing::Level::WARN);
12554        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12555        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12556        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12557        handler.route_outages.record_accepted("outage-restart");
12558        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12559
12560        let restart = handler
12561            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12562            .await
12563            .unwrap();
12564        assert_eq!(
12565            restart[0].header.ty,
12566            FrameType::Response,
12567            "{:?}",
12568            parse_error(&restart[0])
12569        );
12570        handler
12571            .handle_control_frame(
12572                &ctx,
12573                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12574            )
12575            .await
12576            .unwrap();
12577        module.stop().await.unwrap();
12578
12579        let starts = outage_lines(&capture, OUTAGE_START);
12580        assert_eq!(starts.len(), 2, "{starts:?}");
12581        assert_eq!(starts[1].level, tracing::Level::INFO);
12582        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12583    }
12584
12585    /// A restart refused before it touched the module (here: the module is
12586    /// disabled) must clear its operator mark, so the next real outage is
12587    /// still reported as a warning.
12588    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12589    async fn failed_operator_restart_leaves_no_operator_mark() {
12590        let registry = Arc::new(Registry::default());
12591        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
12592        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12593        let capture = EventCapture::default();
12594        let _subscriber =
12595            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12596        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12597        handler.route_outages.record_accepted("outage-disabled");
12598
12599        let restart = handler
12600            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12601            .await
12602            .unwrap();
12603        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12604        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12605
12606        handler
12607            .handle_control_frame(
12608                &ctx,
12609                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12610            )
12611            .await
12612            .unwrap();
12613        let starts = outage_lines(&capture, OUTAGE_START);
12614        assert_eq!(starts.len(), 1, "{starts:?}");
12615        assert_eq!(starts[0].level, tracing::Level::WARN);
12616    }
12617
12618    #[tokio::test(flavor = "current_thread")]
12619    async fn route_open_unknown_module_escapes_target_module_id() {
12620        let handler = ControlHandler::new(Arc::new(Registry::default()));
12621        let capture = EventCapture::default();
12622        let _subscriber =
12623            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12624        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12625        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12626        let response = handler
12627            .handle_control_frame(
12628                &ctx,
12629                route_open_frame(
12630                    395,
12631                    hostile_module_id,
12632                    unique_project_root("hostile-target-module-id"),
12633                ),
12634            )
12635            .await
12636            .unwrap();
12637
12638        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12639        let event = capture
12640            .events()
12641            .into_iter()
12642            .find(|event| {
12643                event.target == "control"
12644                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12645            })
12646            .expect("route.open unknown-module refusal event");
12647        let logged = event.fields.get("module_id").expect("module_id field");
12648        assert!(!logged.bytes().any(|byte| byte < 0x20));
12649        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12650    }
12651
12652    #[tokio::test(flavor = "current_thread")]
12653    async fn route_open_module_rejection_uses_daemon_counter_key() {
12654        let registry = Arc::new(Registry::default());
12655        let forwarding = Arc::new(ForwardingTable::default());
12656        let handler =
12657            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12658        let module_connection = ConnectionId::new(95);
12659        let (module_ctx, mut module_rx) = route_ctx(module_connection);
12660        hello_via_sink(
12661            &handler,
12662            &module_ctx,
12663            &mut module_rx,
12664            hello_frame("aft", PROTOCOL_VERSION, 395),
12665        )
12666        .await;
12667
12668        let client_connection = ConnectionId::new(96);
12669        let (client_ctx, _client_rx) = route_ctx(client_connection);
12670        let capture = EventCapture::default();
12671        let _subscriber =
12672            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12673        let (route_task, bind) = relay_route_open(
12674            &handler,
12675            client_connection,
12676            &client_ctx.egress,
12677            &mut module_rx,
12678            396,
12679            "aft",
12680            "hostile-module-code",
12681        )
12682        .await;
12683        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12684        let rejection = Frame::build(
12685            FrameType::Error,
12686            control_flags(),
12687            0,
12688            0,
12689            bind.header.corr,
12690            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12691        )
12692        .unwrap();
12693        handler
12694            .handle_control_frame(&module_ctx, rejection)
12695            .await
12696            .unwrap();
12697
12698        let response = route_task.await.unwrap();
12699        assert_eq!(parse_error(&response[0])["code"], hostile_code);
12700        let counters = handler.counters().snapshot();
12701        assert_eq!(
12702            counters["route_open_refused_by_code"],
12703            json!({ "module_rejected": 1 })
12704        );
12705        assert!(counters["route_open_refused_by_code"]
12706            .get(hostile_code)
12707            .is_none());
12708
12709        let event = capture
12710            .events()
12711            .into_iter()
12712            .find(|event| {
12713                event.target == "control"
12714                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12715            })
12716            .expect("route.open module-rejection refusal event");
12717        let logged = event.fields.get("module_code").expect("module_code field");
12718        assert!(!logged.bytes().any(|byte| byte < 0x20));
12719        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12720    }
12721
12722    #[tokio::test]
12723    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12724        let registry = Arc::new(Registry::default());
12725        let supervisor_handle = SupervisorHandle::new();
12726        let missing_program = std::env::temp_dir().join(format!(
12727            "subc-route-open-missing-program-{}",
12728            std::process::id()
12729        ));
12730        let supervisor =
12731            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12732                .with_handle(supervisor_handle.clone());
12733        let module = supervisor
12734            .supervise_configured(
12735                ModuleSpec {
12736                    module_id: "failed".to_string(),
12737                    program: missing_program,
12738                    args: Vec::new(),
12739                    env: Vec::new(),
12740                    reserved: false,
12741                    reserved_prefixes: Vec::new(),
12742                    protocol: ModuleProtocol::Subc,
12743                    overlap: Default::default(),
12744                },
12745                true,
12746            )
12747            .unwrap();
12748        assert_eq!(module.state().unwrap(), ModuleState::Failed);
12749
12750        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12751        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12752        let response = handler
12753            .handle_control_frame(
12754                &ctx,
12755                route_open_frame(305, "failed", unique_project_root("failed")),
12756            )
12757            .await
12758            .unwrap();
12759
12760        assert_eq!(response[0].header.ty, FrameType::Error);
12761        let error = parse_error(&response[0]);
12762        assert_eq!(error["code"], "target_unavailable");
12763        assert!(error["message"]
12764            .as_str()
12765            .unwrap()
12766            .contains("state=failed, enabled=true, live=false"));
12767    }
12768
12769    #[tokio::test]
12770    async fn route_open_role_mismatch_remains_target_unavailable() {
12771        let registry = Arc::new(Registry::default());
12772        let handler = ControlHandler::new(Arc::clone(&registry));
12773        handler
12774            .handle_control(
12775                ConnectionId::new(41),
12776                non_routable_hello_frame_with_control_ops("health-only", 306, None),
12777            )
12778            .unwrap();
12779
12780        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12781        let response = handler
12782            .handle_control_frame(
12783                &ctx,
12784                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12785            )
12786            .await
12787            .unwrap();
12788
12789        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12790        assert!(parse_error(&response[0])["message"]
12791            .as_str()
12792            .unwrap()
12793            .contains("does not provide the requested target"));
12794    }
12795
12796    #[tokio::test]
12797    async fn route_open_inactive_registration_remains_target_unavailable() {
12798        let registry = Arc::new(Registry::default());
12799        let handler = ControlHandler::new(Arc::clone(&registry));
12800        handler
12801            .handle_control(
12802                ConnectionId::new(43),
12803                hello_frame("inactive", PROTOCOL_VERSION, 308),
12804            )
12805            .unwrap();
12806        assert!(registry
12807            .set_module_state_for_test("inactive", ChannelState::Closed)
12808            .unwrap());
12809
12810        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12811        let response = handler
12812            .handle_control_frame(
12813                &ctx,
12814                route_open_frame(309, "inactive", unique_project_root("inactive")),
12815            )
12816            .await
12817            .unwrap();
12818
12819        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12820        assert!(parse_error(&response[0])["message"]
12821            .as_str()
12822            .unwrap()
12823            .contains("is not active"));
12824    }
12825
12826    #[tokio::test]
12827    async fn late_health_reply_is_recorded_through_the_module_response_path() {
12828        let registry = Arc::new(Registry::default());
12829        let forwarding = Arc::new(ForwardingTable::default());
12830        let supervisor_handle = SupervisorHandle::new();
12831        let supervisor =
12832            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
12833                .with_forwarding(Arc::clone(&forwarding))
12834                .with_handle(supervisor_handle.clone());
12835        let module = supervisor
12836            .supervise_configured(
12837                crate::ModuleSpec {
12838                    module_id: "late-health-response".to_string(),
12839                    program: PathBuf::from("disabled-module"),
12840                    args: Vec::new(),
12841                    env: Vec::new(),
12842                    reserved: false,
12843                    reserved_prefixes: Vec::new(),
12844                    protocol: ModuleProtocol::Subc,
12845                    overlap: Default::default(),
12846                },
12847                false,
12848            )
12849            .unwrap();
12850        let handler =
12851            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12852                .with_supervisor(supervisor_handle);
12853        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12854        handler
12855            .handle_control_frame(
12856                &module_ctx,
12857                hello_frame_with_control_ops(
12858                    "late-health-response",
12859                    PROTOCOL_VERSION,
12860                    7,
12861                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12862                ),
12863            )
12864            .await
12865            .unwrap();
12866        let probe_started_at = Instant::now() - Duration::from_millis(80);
12867        let pending = forwarding
12868            .begin_health_probe_rpc_for(
12869                "late-health-response",
12870                MODULE_CONTROL_OP_HEALTH_CHECK,
12871                probe_started_at,
12872                Instant::now() - Duration::from_millis(1),
12873            )
12874            .unwrap();
12875        assert!(forwarding
12876            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12877            .unwrap());
12878
12879        let responses = handler
12880            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12881            .await
12882            .unwrap();
12883
12884        assert!(responses.is_empty());
12885        let health = module.status().unwrap().health;
12886        assert_eq!(health.late_answer_count, 1);
12887        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12888    }
12889
12890    #[tokio::test]
12891    async fn health_probe_timeout_and_module_death_are_typed() {
12892        let registry = Arc::new(Registry::default());
12893        let forwarding = Arc::new(ForwardingTable::default());
12894        let handler =
12895            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12896                .with_health_probe_timeout(Duration::from_millis(50));
12897        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12898        hello_via_sink(
12899            &handler,
12900            &module_ctx,
12901            &mut module_rx,
12902            hello_frame_with_control_ops(
12903                "aft",
12904                PROTOCOL_VERSION,
12905                7,
12906                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12907            ),
12908        )
12909        .await;
12910
12911        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12912        let responses = handler
12913            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12914            .await
12915            .unwrap();
12916        assert_eq!(responses[0].header.ty, FrameType::Error);
12917        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12918        let _ = module_rx.try_recv();
12919
12920        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12921        let health_handler = handler.clone();
12922        let death_task = tokio::spawn(async move {
12923            health_handler
12924                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12925                .await
12926                .unwrap()
12927        });
12928        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12929            .await
12930            .unwrap()
12931            .unwrap();
12932        handler
12933            .cleanup_connection(module_ctx.connection_id)
12934            .unwrap();
12935        let responses = death_task.await.unwrap();
12936        assert_eq!(responses[0].header.ty, FrameType::Error);
12937        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12938    }
12939
12940    #[test]
12941    fn hello_requires_exact_protocol_version() {
12942        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12943            let registry = Arc::new(Registry::default());
12944            let handler = ControlHandler::new(Arc::clone(&registry));
12945            let responses = handler
12946                .handle_control(
12947                    ConnectionId::new(connection),
12948                    hello_frame("aft", offered, 9),
12949                )
12950                .unwrap();
12951
12952            assert_eq!(responses.len(), 1);
12953            assert_eq!(responses[0].header.ty, FrameType::Error);
12954            let error = parse_error(&responses[0]);
12955            assert_eq!(error["code"], "version_unsupported");
12956            assert!(registry.get_module("aft").unwrap().is_none());
12957            assert_eq!(registry.active_registration_count().unwrap(), 0);
12958        }
12959    }
12960
12961    #[test]
12962    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12963        let registry = Arc::new(Registry::default());
12964        let forwarding = Arc::new(ForwardingTable::default());
12965        let handler =
12966            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12967        let module_connection = ConnectionId::new(301);
12968        let registration = registry
12969            .register_with_control_ops(
12970                manifest("aft-push", PROTOCOL_VERSION),
12971                PROTOCOL_VERSION,
12972                module_connection,
12973                module_baseline_control_ops(),
12974            )
12975            .unwrap();
12976        let (module_tx, _module_rx) = mpsc::channel(8);
12977        let endpoint = forwarding
12978            .register_module_connection(
12979                module_connection,
12980                "aft-push".to_string(),
12981                PROTOCOL_VERSION,
12982                manifest_concurrency(&registration.manifest),
12983                FrameSink::new(module_tx),
12984            )
12985            .unwrap();
12986
12987        // A push op this version does not know is ignored (forward-compat), not errored.
12988        let unknown = Frame::build(
12989            FrameType::Push,
12990            control_flags(),
12991            0,
12992            0,
12993            5,
12994            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12995        )
12996        .unwrap();
12997        let out = handler.handle_status_update(endpoint, unknown).unwrap();
12998        assert!(
12999            out.is_empty(),
13000            "unknown push op must be ignored, got {out:?}"
13001        );
13002
13003        // A malformed body for a KNOWN op is a real error worth surfacing.
13004        let malformed = Frame::build(
13005            FrameType::Push,
13006            control_flags(),
13007            0,
13008            0,
13009            6,
13010            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13011        )
13012        .unwrap();
13013        let out = handler.handle_status_update(endpoint, malformed).unwrap();
13014        assert_eq!(out.len(), 1);
13015        assert_eq!(out[0].header.ty, FrameType::Error);
13016        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13017    }
13018
13019    #[test]
13020    fn hello_rejected_when_connection_already_owns_client_routes() {
13021        let registry = Arc::new(Registry::default());
13022        let forwarding = Arc::new(ForwardingTable::default());
13023        let handler =
13024            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13025        // Commits a client route on connection 202 (bound to a module on conn 101).
13026        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
13027        let client_connection = ConnectionId::new(202);
13028
13029        // That same connection now tries to register as a module: rejected, so one
13030        // connection never holds both client-route and module-endpoint state.
13031        let responses = handler
13032            .handle_control(
13033                client_connection,
13034                hello_frame("aft-second", PROTOCOL_VERSION, 9),
13035            )
13036            .unwrap();
13037        assert_eq!(responses[0].header.ty, FrameType::Error);
13038        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13039        assert!(registry.get_module("aft-second").unwrap().is_none());
13040    }
13041
13042    #[tokio::test]
13043    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13044        let registry = Arc::new(Registry::default());
13045        let forwarding = Arc::new(ForwardingTable::default());
13046        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13047        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13048        hello_via_sink(
13049            &handler,
13050            &module_ctx,
13051            &mut module_rx,
13052            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13053        )
13054        .await;
13055        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13056        let pending = forwarding
13057            .begin_route_bind_relay_for_test(
13058                client_ctx.connection_id,
13059                client_ctx.egress.clone(),
13060                2,
13061                "alpha",
13062            )
13063            .unwrap();
13064        forwarding
13065            .complete_pending_relay(
13066                module_ctx.connection_id,
13067                pending.corr,
13068                RouteBindRelayOutcome::Accepted,
13069            )
13070            .unwrap();
13071        client_rx.try_recv().unwrap();
13072        for module_id in ["beta", "alpha"] {
13073            let replies = handler
13074                .handle_control_frame(
13075                    &module_ctx,
13076                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13077                )
13078                .await
13079                .unwrap();
13080            assert_eq!(replies.len(), 1, "second HELLO must be refused");
13081            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13082        }
13083        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13084        assert!(registry.get_module("beta").unwrap().is_none());
13085        assert!(matches!(
13086            forwarding
13087                .lookup_data_route(
13088                    client_ctx.connection_id,
13089                    pending.client_channel,
13090                    pending.client_epoch,
13091                )
13092                .unwrap(),
13093            DataRoute::Client(DataRouteState::Bound(_))
13094        ));
13095        assert!(handler
13096            .hello_launch_nonces
13097            .lock()
13098            .unwrap()
13099            .presented(module_ctx.connection_id, Some("alpha-nonce")));
13100        assert!(module_rx.try_recv().is_err());
13101    }
13102
13103    #[test]
13104    fn reserved_module_hello_requires_matching_launch_nonce() {
13105        let registry = Arc::new(Registry::default());
13106        let supervisor = SupervisorHandle::new();
13107        // The supervisor recorded the nonce it injected when it spawned the reserved
13108        // module; the HELLO verifier checks against the same shared handle.
13109        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13110        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13111
13112        // A HELLO with NO nonce is rejected.
13113        let no_nonce = handler
13114            .handle_control(
13115                ConnectionId::new(1),
13116                hello_frame("vault", PROTOCOL_VERSION, 1),
13117            )
13118            .unwrap();
13119        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13120        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13121        assert!(registry.get_module("vault").unwrap().is_none());
13122
13123        // A HELLO with the WRONG nonce is rejected.
13124        let wrong = handler
13125            .handle_control(
13126                ConnectionId::new(2),
13127                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13128            )
13129            .unwrap();
13130        assert_eq!(wrong[0].header.ty, FrameType::Error);
13131        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13132        assert!(registry.get_module("vault").unwrap().is_none());
13133
13134        // A HELLO with the CORRECT nonce registers.
13135        let ok = handler
13136            .handle_control(
13137                ConnectionId::new(3),
13138                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13139            )
13140            .unwrap();
13141        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13142        assert!(registry.get_module("vault").unwrap().is_some());
13143    }
13144
13145    #[test]
13146    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13147        let registry = Arc::new(Registry::default());
13148        let supervisor = SupervisorHandle::new();
13149        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13150        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13151        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13152
13153        let squat = handler
13154            .handle_control(
13155                ConnectionId::new(1),
13156                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13157            )
13158            .unwrap();
13159        assert_eq!(squat[0].header.ty, FrameType::Error);
13160        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13161        assert!(parse_error(&squat[0])["message"]
13162            .as_str()
13163            .unwrap()
13164            .contains("fed:"));
13165
13166        let accepted_peer = handler
13167            .handle_control(
13168                ConnectionId::new(2),
13169                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13170            )
13171            .unwrap();
13172        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13173
13174        let accepted_short = handler
13175            .handle_control(
13176                ConnectionId::new(3),
13177                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13178            )
13179            .unwrap();
13180        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13181
13182        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13183            let response = handler
13184                .handle_control(
13185                    ConnectionId::new(conn),
13186                    hello_frame(module_id, PROTOCOL_VERSION, conn),
13187                )
13188                .unwrap();
13189            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13190        }
13191    }
13192
13193    #[test]
13194    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13195        let registry = Arc::new(Registry::default());
13196        let supervisor = SupervisorHandle::new();
13197        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13198        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13199        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13200        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13201
13202        let owner_nonce = handler
13203            .handle_control(
13204                ConnectionId::new(1),
13205                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13206            )
13207            .unwrap();
13208        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13209        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13210        assert!(registry.get_module("fed:special").unwrap().is_none());
13211
13212        let exact_nonce = handler
13213            .handle_control(
13214                ConnectionId::new(2),
13215                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13216            )
13217            .unwrap();
13218        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13219        assert!(registry.get_module("fed:special").unwrap().is_some());
13220    }
13221
13222    #[test]
13223    fn non_reserved_module_ignores_launch_nonce() {
13224        let registry = Arc::new(Registry::default());
13225        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
13226        // registration succeeds whether a spawned process echoes a nonce or not.
13227        let handler = ControlHandler::new(Arc::clone(&registry));
13228        let no_nonce = handler
13229            .handle_control(
13230                ConnectionId::new(1),
13231                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13232            )
13233            .unwrap();
13234        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13235        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13236
13237        let echoed_nonce = handler
13238            .handle_control(
13239                ConnectionId::new(2),
13240                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13241            )
13242            .unwrap();
13243        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13244        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13245    }
13246
13247    #[test]
13248    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13249        let handler = ControlHandler::default();
13250        let conn = ConnectionId::new(1);
13251        let malformed = Frame::build(
13252            FrameType::Hello,
13253            control_flags(),
13254            0,
13255            0,
13256            3,
13257            b"{not json".to_vec(),
13258        )
13259        .unwrap();
13260
13261        let error = handler.handle_control(conn, malformed).unwrap();
13262        assert_eq!(error[0].header.ty, FrameType::Error);
13263        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13264
13265        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13266        let pong = handler.handle_control(conn, ping).unwrap();
13267        assert_eq!(pong[0].header.ty, FrameType::Pong);
13268        assert_eq!(pong[0].header.corr, 4);
13269    }
13270
13271    #[test]
13272    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13273        let registry = Arc::new(Registry::default());
13274        let handler = ControlHandler::new(Arc::clone(&registry));
13275
13276        handler
13277            .handle_control(
13278                ConnectionId::new(1),
13279                hello_frame("aft", PROTOCOL_VERSION, 1),
13280            )
13281            .unwrap();
13282        let duplicate = handler
13283            .handle_control(
13284                ConnectionId::new(2),
13285                hello_frame("aft", PROTOCOL_VERSION, 2),
13286            )
13287            .unwrap();
13288
13289        assert_eq!(duplicate[0].header.ty, FrameType::Error);
13290        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13291        let registration = registry.get_module("aft").unwrap().unwrap();
13292        assert_eq!(registration.connection_id, ConnectionId::new(1));
13293    }
13294
13295    #[test]
13296    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13297        let registry = Arc::new(Registry::default());
13298        let forwarding = Arc::new(ForwardingTable::default());
13299        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13300        let handler =
13301            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13302                .with_process_liveness(process_liveness);
13303        let (ctx, route_channel, route_epoch) =
13304            bind_liveness_route(&registry, &forwarding, "aft-dead");
13305        let responses = handler
13306            .handle_route_poll(
13307                &ctx,
13308                route_poll_frame(41, PollKind::Liveness, route_channel),
13309                route_channel,
13310                route_epoch,
13311                PollKind::Liveness,
13312            )
13313            .unwrap();
13314
13315        assert_eq!(responses.len(), 1);
13316        assert_eq!(responses[0].header.ty, FrameType::Response);
13317        assert_route_poll_liveness(&responses[0], false);
13318    }
13319
13320    #[test]
13321    fn liveness_poll_without_process_source_uses_bound_route() {
13322        let registry = Arc::new(Registry::default());
13323        let forwarding = Arc::new(ForwardingTable::default());
13324        let handler =
13325            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13326        let (ctx, route_channel, route_epoch) =
13327            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
13328        let responses = handler
13329            .handle_route_poll(
13330                &ctx,
13331                route_poll_frame(42, PollKind::Liveness, route_channel),
13332                route_channel,
13333                route_epoch,
13334                PollKind::Liveness,
13335            )
13336            .unwrap();
13337
13338        assert_route_poll_liveness(&responses[0], true);
13339    }
13340
13341    #[test]
13342    fn liveness_poll_untracked_process_source_uses_bound_route() {
13343        let registry = Arc::new(Registry::default());
13344        let forwarding = Arc::new(ForwardingTable::default());
13345        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13346        let handler =
13347            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13348                .with_process_liveness(process_liveness);
13349        let (ctx, route_channel, route_epoch) =
13350            bind_liveness_route(&registry, &forwarding, "aft-untracked");
13351        let responses = handler
13352            .handle_route_poll(
13353                &ctx,
13354                route_poll_frame(43, PollKind::Liveness, route_channel),
13355                route_channel,
13356                route_epoch,
13357                PollKind::Liveness,
13358            )
13359            .unwrap();
13360
13361        assert_route_poll_liveness(&responses[0], true);
13362    }
13363
13364    #[tokio::test]
13365    async fn unknown_op_returns_unknown_control_op() {
13366        let handler = ControlHandler::default();
13367        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13368        let request = Frame::build(
13369            FrameType::Request,
13370            control_flags(),
13371            0,
13372            0,
13373            55,
13374            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13375        )
13376        .unwrap();
13377
13378        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13379
13380        assert_eq!(response.len(), 1);
13381        assert_eq!(response[0].header.ty, FrameType::Error);
13382        assert_eq!(response[0].header.corr, 55);
13383        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13384    }
13385
13386    #[tokio::test]
13387    async fn supervisor_provenance_rejects_unknown_exact_module() {
13388        let handler = ControlHandler::default();
13389        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13390        let request = Frame::build(
13391            FrameType::Request,
13392            control_flags(),
13393            0,
13394            0,
13395            57,
13396            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13397        )
13398        .unwrap();
13399
13400        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13401
13402        assert_eq!(response.len(), 1);
13403        assert_eq!(response[0].header.ty, FrameType::Error);
13404        assert_eq!(response[0].header.corr, 57);
13405        let error = parse_error(&response[0]);
13406        assert_eq!(error["code"], "unknown_module");
13407        assert_eq!(error["message"], "module_id 'missing' is not supervised");
13408    }
13409
13410    #[test]
13411    fn provenance_probe_override_keeps_handler_tests_deterministic() {
13412        let expected = subc_control::RunningImageAgreement::Unavailable {
13413            reason: subc_control::RunningImageUnavailableReason::HashFailed,
13414        };
13415        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13416        assert_eq!(handler.provenance_probe_override, Some(expected));
13417    }
13418
13419    #[test]
13420    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13421        let verdict = reload_verdict(
13422            std::path::Path::new("/bin/new"),
13423            Some(std::path::Path::new("/bin/old")),
13424            subc_control::RunningImageAgreement::Unavailable {
13425                reason: subc_control::RunningImageUnavailableReason::HashFailed,
13426            },
13427        );
13428        assert!(matches!(
13429            verdict.path,
13430            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13431                if configured == std::path::Path::new("/bin/new")
13432                    && spawned_from == std::path::Path::new("/bin/old")
13433        ));
13434    }
13435
13436    #[test]
13437    fn launch_spec_without_running_digest_is_unavailable_not_match() {
13438        use subc_control::{
13439            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13440        };
13441        let configured = crate::supervise::ModuleSpec {
13442            module_id: "legacy".into(),
13443            program: "/module".into(),
13444            args: Vec::new(),
13445            env: Vec::new(),
13446            reserved: false,
13447            reserved_prefixes: Vec::new(),
13448            protocol: subc_control::ModuleProtocol::Subc,
13449            overlap: Default::default(),
13450        };
13451        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13452        assert_eq!(
13453            launch_spec_agreement(&configured, false, Some(&digest)),
13454            Agreement::Unavailable {
13455                reason: Reason::NotRunning
13456            }
13457        );
13458        assert_eq!(
13459            launch_spec_agreement(&configured, true, None),
13460            Agreement::Unavailable {
13461                reason: Reason::SpawnedSpecUnavailable
13462            }
13463        );
13464        assert_eq!(
13465            launch_spec_agreement(&configured, true, Some(&digest)),
13466            Agreement::Match
13467        );
13468    }
13469
13470    #[cfg(windows)]
13471    #[test]
13472    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13473        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13474        let configured = dir.join("module.exe");
13475        std::fs::write(&configured, b"executable path fixture").unwrap();
13476        let spawned = std::fs::canonicalize(&configured).unwrap();
13477        assert_ne!(
13478            configured, spawned,
13479            "fixture must exercise different path spellings"
13480        );
13481        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13482        let lower_case = configured.to_string_lossy().to_lowercase();
13483        for spelling in [
13484            &configured,
13485            Path::new(&forward_slashes),
13486            Path::new(&lower_case),
13487        ] {
13488            let verdict = reload_verdict(
13489                spelling,
13490                Some(&spawned),
13491                subc_control::RunningImageAgreement::Unavailable {
13492                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13493                },
13494            );
13495            assert_eq!(
13496                verdict.path,
13497                ReloadPathAgreement::Match,
13498                "configured {spelling:?}, spawned {spawned:?}"
13499            );
13500        }
13501    }
13502
13503    #[cfg(windows)]
13504    #[test]
13505    fn windows_reload_path_keeps_different_existing_program_pending() {
13506        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13507        let old = dir.join("old.exe");
13508        let new = dir.join("new.exe");
13509        std::fs::write(&old, b"same executable content").unwrap();
13510        std::fs::copy(&old, &new).unwrap();
13511        let spawned = std::fs::canonicalize(&old).unwrap();
13512        let image = subc_control::RunningImageAgreement::Unavailable {
13513            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13514        };
13515        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13516        assert!(
13517            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13518            if configured == new && spawned_from == spawned)
13519        );
13520        let missing = dir.join("missing.exe");
13521        assert!(matches!(
13522            reload_verdict(&missing, Some(&spawned), image).path,
13523            ReloadPathAgreement::Mismatch { .. }
13524        ));
13525    }
13526
13527    #[test]
13528    fn reload_verdict_detects_replaced_image_at_same_path() {
13529        let image = subc_control::RunningImageAgreement::Mismatch {
13530            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13531                digest: "old".into(),
13532            },
13533            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13534                digest: "new".into(),
13535            },
13536        };
13537        let verdict = reload_verdict(
13538            std::path::Path::new("/bin/same"),
13539            Some(std::path::Path::new("/bin/same")),
13540            image.clone(),
13541        );
13542        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13543        assert_eq!(verdict.image, image);
13544    }
13545
13546    #[test]
13547    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13548        let image = subc_control::RunningImageAgreement::Unavailable {
13549            reason: subc_control::RunningImageUnavailableReason::NotRunning,
13550        };
13551        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13552        assert_eq!(
13553            verdict.path,
13554            subc_control::ReloadPathAgreement::Unavailable {
13555                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13556            }
13557        );
13558        assert_eq!(verdict.image, image);
13559
13560        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13561            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13562        };
13563        let verdict = reload_verdict(
13564            std::path::Path::new("/bin/same"),
13565            Some(std::path::Path::new("/bin/same")),
13566            unconfirmed.clone(),
13567        );
13568        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13569        assert_eq!(verdict.image, unconfirmed);
13570    }
13571
13572    #[test]
13573    fn reload_verdict_preserves_each_image_unavailability_reason() {
13574        use subc_control::RunningImageUnavailableReason as Reason;
13575
13576        for reason in [
13577            Reason::NotRunning,
13578            Reason::UnsupportedPlatform,
13579            Reason::RunningExecutableUnreadable,
13580            Reason::SpawnedPathUnreadable,
13581            Reason::HashFailed,
13582            Reason::ProcessIdentityUnconfirmed,
13583            Reason::Unknown("future_probe_reason".to_string()),
13584        ] {
13585            let image = subc_control::RunningImageAgreement::Unavailable {
13586                reason: reason.clone(),
13587            };
13588            let verdict = reload_verdict(
13589                std::path::Path::new("/bin/same"),
13590                Some(std::path::Path::new("/bin/same")),
13591                image.clone(),
13592            );
13593            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13594            assert_eq!(verdict.image, image, "{reason:?}");
13595        }
13596    }
13597
13598    #[tokio::test]
13599    async fn malformed_control_bodies_return_invalid_control_body() {
13600        let handler = ControlHandler::default();
13601        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13602
13603        for (corr, body) in [
13604            (56, br#"{"route_channel":1}"#.as_slice()),
13605            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13606            (
13607                58,
13608                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13609            ),
13610        ] {
13611            let request = Frame::build(
13612                FrameType::Request,
13613                control_flags(),
13614                0,
13615                0,
13616                corr,
13617                body.to_vec(),
13618            )
13619            .unwrap();
13620            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13621
13622            assert_eq!(response.len(), 1);
13623            assert_eq!(response[0].header.ty, FrameType::Error);
13624            assert_eq!(response[0].header.corr, corr);
13625            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13626        }
13627    }
13628
13629    #[tokio::test]
13630    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13631        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13632        let registry = Arc::new(Registry::default());
13633        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
13634        let router = Router::with_control_handler(Arc::clone(&control));
13635        let connection = router.begin_connection();
13636        let (ctx, mut rx) = route_ctx(connection.id());
13637
13638        router
13639            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13640            .await
13641            .unwrap();
13642        let response = rx.recv().await.unwrap();
13643        let ack = parse_ack(&response);
13644        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13645        let channel = 1;
13646
13647        let goodbye =
13648            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13649        router.route_for_connection(&ctx, goodbye).await.unwrap();
13650        assert!(rx.try_recv().is_err());
13651        assert!(registry.get_module("aft").unwrap().is_none());
13652
13653        router
13654            .route_for_connection(&ctx, channel_request(channel, 13))
13655            .await
13656            .unwrap();
13657        let error_frame = rx.recv().await.unwrap();
13658        assert_eq!(error_frame.header.ty, FrameType::Error);
13659        assert_eq!(error_frame.header.channel, channel);
13660        let captured = crate::router::test_log::captured_logs(&logs);
13661        assert_eq!(
13662            captured
13663                .lines()
13664                .filter(|line| {
13665                    line.contains("module_id=aft")
13666                        && line.contains("reason=explicit_goodbye")
13667                        && line.contains("module registration ended")
13668                })
13669                .count(),
13670            1,
13671            "unexpected GOODBYE registry log: {captured}"
13672        );
13673    }
13674
13675    #[tokio::test]
13676    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13677        let registry = Arc::new(Registry::default());
13678        let handler = ControlHandler::new(registry).with_capability_config(
13679            [("prov".to_string(), true), ("cons".to_string(), true)],
13680            BTreeMap::new(),
13681        );
13682        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13683        register_capability_manifest(
13684            &handler,
13685            &provider_ctx,
13686            &mut provider_rx,
13687            capability_manifest("prov", &["thing/v1"], &[]),
13688            1,
13689        )
13690        .await;
13691        let mut consumer = capability_manifest("cons", &[], &[]);
13692        consumer.capabilities.as_mut().unwrap().requires.push(
13693            subc_protocol::manifest::CapabilityRequirement {
13694                capability: "thing/v1".to_string(),
13695                need: subc_protocol::manifest::CapabilityNeed::Required,
13696            },
13697        );
13698        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13699        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13700        assert_eq!(
13701            handler.capability_evaluator.verdict("cons", "thing/v1"),
13702            Some(CapabilityVerdict::Provided)
13703        );
13704        let (mut client_rx, _) = open_route_for_capability_test(
13705            &handler,
13706            &provider_ctx,
13707            &mut provider_rx,
13708            703,
13709            3,
13710            "prov",
13711            None,
13712        )
13713        .await;
13714        let goodbye =
13715            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13716        handler
13717            .handle_control_frame(&provider_ctx, goodbye)
13718            .await
13719            .unwrap();
13720        assert_eq!(
13721            handler.capability_evaluator.verdict("cons", "thing/v1"),
13722            Some(CapabilityVerdict::NeverProvided)
13723        );
13724        let closed = client_rx
13725            .try_recv()
13726            .expect("GOODBYE pushes route.closed before route GOODBYE");
13727        assert!(
13728            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13729            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13730        );
13731        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13732        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13733    }
13734
13735    struct RouteClosedFixture {
13736        handler: ControlHandler,
13737        #[cfg(unix)]
13738        supervisor: Supervisor,
13739        module_ctx: RouteCtx,
13740        #[cfg(unix)]
13741        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13742        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13743        route: Arc<crate::forwarding::RouteBinding>,
13744    }
13745
13746    async fn route_closed_fixture() -> RouteClosedFixture {
13747        let registry = Arc::new(Registry::default());
13748        let handler = ControlHandler::new(Arc::clone(&registry));
13749        #[cfg(unix)]
13750        let supervisor = Supervisor::new(registry, RestartPolicy::default())
13751            .with_forwarding(Arc::clone(&handler.forwarding));
13752        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13753        register_capability_manifest(
13754            &handler,
13755            &module_ctx,
13756            &mut module_rx,
13757            capability_manifest("prov", &[], &[]),
13758            1,
13759        )
13760        .await;
13761        let (client_rx, bind) = open_route_for_capability_test(
13762            &handler,
13763            &module_ctx,
13764            &mut module_rx,
13765            702,
13766            2,
13767            "prov",
13768            None,
13769        )
13770        .await;
13771        let ModuleControlRequest::RouteBind {
13772            route_channel: channel,
13773            epoch: route_epoch,
13774            ..
13775        } = bind
13776        else {
13777            panic!("expected route.bind");
13778        };
13779        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13780            handler
13781                .forwarding
13782                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13783                .unwrap()
13784        else {
13785            panic!("the acknowledged route must be bound");
13786        };
13787        RouteClosedFixture {
13788            handler,
13789            #[cfg(unix)]
13790            supervisor,
13791            module_ctx,
13792            #[cfg(unix)]
13793            module_rx,
13794            client_rx,
13795            route,
13796        }
13797    }
13798
13799    impl RouteClosedFixture {
13800        #[cfg(unix)]
13801        async fn cut(&mut self) {
13802            let notices = async {
13803                let mut draining = self.module_rx.recv().await.unwrap();
13804                assert!(matches!(
13805                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13806                        &draining.body
13807                    )
13808                    .unwrap(),
13809                    subc_protocol::session::ModuleControlCommand::Draining {
13810                        reason: RouteCloseReason::Restart,
13811                        ..
13812                    }
13813                ));
13814                draining.flushed.take().unwrap().send(()).unwrap();
13815                let mut closing = self.client_rx.recv().await.unwrap();
13816                assert!(matches!(
13817                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13818                    ClientControlPush::RouteClosing {
13819                        reason: RouteCloseReason::Restart,
13820                        ..
13821                    }
13822                ));
13823                closing.flushed.take().unwrap().send(()).unwrap();
13824            };
13825            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13826            result.unwrap();
13827        }
13828
13829        fn disconnect(&mut self) -> ClientControlPush {
13830            self.handler
13831                .cleanup_connection(self.module_ctx.connection_id)
13832                .unwrap();
13833            let closed = self.client_rx.try_recv().unwrap();
13834            assert_eq!(closed.header.ty, FrameType::Push);
13835            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13836            assert!(matches!(
13837                &push,
13838                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13839                    if channels == &[self.route.client_channel]
13840            ));
13841            assert_eq!(
13842                self.client_rx.try_recv().unwrap().header.ty,
13843                FrameType::Goodbye
13844            );
13845            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13846            push
13847        }
13848    }
13849
13850    #[cfg(unix)]
13851    #[tokio::test(start_paused = true)]
13852    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13853        let mut fixture = route_closed_fixture().await;
13854        fixture.cut().await;
13855        assert!(matches!(
13856            fixture.disconnect(),
13857            ClientControlPush::RouteClosed {
13858                reason: RouteCloseReason::Restart,
13859                drained: true,
13860                ..
13861            }
13862        ));
13863    }
13864
13865    #[cfg(unix)]
13866    #[tokio::test(start_paused = true)]
13867    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13868        let mut fixture = route_closed_fixture().await;
13869        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13870        let started = tokio::time::Instant::now();
13871        fixture.cut().await;
13872        assert!(started.elapsed() >= Duration::from_secs(2));
13873        assert_eq!(fixture.route.flow.in_flight(), 1);
13874        assert!(matches!(
13875            fixture.disconnect(),
13876            ClientControlPush::RouteClosed {
13877                reason: RouteCloseReason::Restart,
13878                drained: false,
13879                ..
13880            }
13881        ));
13882    }
13883
13884    #[tokio::test]
13885    async fn module_connection_crash_reports_crash_and_not_drained() {
13886        let mut fixture = route_closed_fixture().await;
13887        // The server calls cleanup_connection on module EOF. Without a planned
13888        // daemon shutdown, that disconnect must remain a crash with no prior
13889        // route.closing notice, even when the route has no in-flight calls.
13890        assert!(matches!(
13891            fixture.disconnect(),
13892            ClientControlPush::RouteClosed {
13893                reason: RouteCloseReason::Crash,
13894                drained: false,
13895                ..
13896            }
13897        ));
13898    }
13899
13900    #[tokio::test]
13901    async fn dropping_router_connection_releases_registration() {
13902        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13903        let registry = Arc::new(Registry::default());
13904        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
13905        let router = Router::with_control_handler(Arc::clone(&control));
13906        let connection = router.begin_connection();
13907        let connection_id = connection.id();
13908        let (ctx, mut rx) = route_ctx(connection_id);
13909
13910        router
13911            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13912            .await
13913            .unwrap();
13914        let response = rx.recv().await.unwrap();
13915        let ack = parse_ack(&response);
13916        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13917        assert!(registry.get_module("aft").unwrap().is_some());
13918
13919        drop(connection);
13920
13921        assert!(registry.get_module("aft").unwrap().is_none());
13922        assert_eq!(registry.active_registration_count().unwrap(), 0);
13923
13924        control
13925            .cleanup_connection(ConnectionId::new(u64::MAX))
13926            .unwrap();
13927        let captured = crate::router::test_log::captured_logs(&logs);
13928        println!("captured registry lifecycle logs:\n{captured}");
13929        let events: Vec<_> = captured
13930            .lines()
13931            .filter(|line| {
13932                line.contains("module registered module_id=aft ")
13933                    || line.contains("module registration ended")
13934            })
13935            .collect();
13936        assert_eq!(
13937            events.len(),
13938            2,
13939            "unexpected registry lifecycle logs: {captured}"
13940        );
13941        assert!(events[0].contains("module registered module_id=aft "));
13942        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13943        assert!(events[1].contains(&format!(
13944            "module_id=aft connection_id={}",
13945            connection_id.get()
13946        )));
13947        assert!(events[1].contains("reason=connection_closed"));
13948        assert!(events[1].contains("module registration ended"));
13949        assert_eq!(
13950            captured
13951                .lines()
13952                .filter(|line| line.contains("module registered module_id=aft "))
13953                .count(),
13954            1,
13955            "legacy registration admission line must appear once: {captured}"
13956        );
13957    }
13958
13959    #[tokio::test]
13960    async fn hello_registration_keeps_legacy_module_registered_line_once() {
13961        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13962        let registry = Arc::new(Registry::default());
13963        let control = ControlHandler::new(Arc::clone(&registry));
13964        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13965        hello_via_sink(
13966            &control,
13967            &ctx,
13968            &mut rx,
13969            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13970        )
13971        .await;
13972
13973        let captured = crate::router::test_log::captured_logs(&logs);
13974        let admissions: Vec<_> = captured
13975            .lines()
13976            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13977            .collect();
13978        assert_eq!(
13979            admissions.len(),
13980            1,
13981            "legacy admission line must remain exactly once: {captured}"
13982        );
13983        assert!(admissions[0].contains("routable_provider=true"));
13984        assert!(admissions[0].contains("connection_id=777"));
13985    }
13986
13987    fn capability_manifest(
13988        module_id: &str,
13989        provides: &[&str],
13990        must_never_reach: &[&str],
13991    ) -> ModuleManifest {
13992        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13993        manifest.capabilities = Some(CapabilityDeclarations {
13994            provides: provides
13995                .iter()
13996                .map(|capability| (*capability).to_string())
13997                .collect(),
13998            requires: Vec::new(),
13999            must_never_reach: must_never_reach
14000                .iter()
14001                .map(|capability| (*capability).to_string())
14002                .collect(),
14003        });
14004        manifest
14005    }
14006
14007    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14008        Frame::build(
14009            FrameType::Hello,
14010            control_flags(),
14011            0,
14012            0,
14013            corr,
14014            serde_json::to_vec(&ModuleHelloBody {
14015                protocol_ver: manifest.protocol_ver,
14016                manifest,
14017                control_ops: None,
14018                launch_nonce: None,
14019            })
14020            .expect("capability test HELLO serializes"),
14021        )
14022        .expect("capability test HELLO frame builds")
14023    }
14024
14025    fn catalog_update_with_capabilities_frame(
14026        corr: u64,
14027        capabilities: CapabilityDeclarations,
14028    ) -> Frame {
14029        Frame::build(
14030            FrameType::Request,
14031            control_flags(),
14032            0,
14033            0,
14034            corr,
14035            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14036                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14037                capabilities: Some(capabilities),
14038                ready: None,
14039            })
14040            .expect("capability catalog.update serializes"),
14041        )
14042        .expect("capability catalog.update frame builds")
14043    }
14044
14045    async fn register_capability_manifest(
14046        handler: &ControlHandler,
14047        ctx: &RouteCtx,
14048        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14049        manifest: ModuleManifest,
14050        corr: u64,
14051    ) {
14052        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14053    }
14054
14055    async fn open_route_for_capability_test(
14056        handler: &ControlHandler,
14057        target_ctx: &RouteCtx,
14058        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14059        client_connection_id: u64,
14060        corr: u64,
14061        target_module_id: &str,
14062        consumer_identity: Option<ConsumerIdentity>,
14063    ) -> (
14064        mpsc::Receiver<crate::router::OutboundFrame>,
14065        ModuleControlRequest,
14066    ) {
14067        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14068        let route_handler = handler.clone();
14069        let target_module_id = target_module_id.to_string();
14070        let route_task = tokio::spawn(async move {
14071            route_handler
14072                .handle_control_frame(
14073                    &client_ctx,
14074                    route_open_frame_with_admission_facts(
14075                        corr,
14076                        &target_module_id,
14077                        unique_project_root("admission-facts"),
14078                        consumer_identity,
14079                        None,
14080                    ),
14081                )
14082                .await
14083                .expect("capability test route.open succeeds")
14084        });
14085        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14086            .await
14087            .expect("capability test route.open must reach route.bind")
14088            .expect("target control receiver stays open");
14089        let bind_request: ModuleControlRequest =
14090            serde_json::from_slice(&bind.body).expect("route.bind decodes");
14091        handler
14092            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14093            .await
14094            .expect("capability test route.bind ACK succeeds");
14095        assert!(route_task.await.expect("route.open task joins").is_empty());
14096        let opened = client_rx
14097            .recv()
14098            .await
14099            .expect("successful route.open publishes a response");
14100        assert!(matches!(
14101            serde_json::from_slice::<ClientControlResponse>(&opened.body),
14102            Ok(ClientControlResponse::RouteOpen { .. })
14103        ));
14104        (client_rx, bind_request)
14105    }
14106
14107    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14108        assert_eq!(frame.header.ty, FrameType::Push);
14109        assert_eq!(frame.header.channel, 0);
14110        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14111            .expect("route.closed control push decodes");
14112        let ClientControlPush::RouteClosed { channels, .. } = &push else {
14113            panic!("expected route.closed");
14114        };
14115        assert_eq!(channels.len(), 1, "exactly one violating route closed");
14116        let channels = channels.clone();
14117        assert_eq!(
14118            push,
14119            ClientControlPush::RouteClosed {
14120                module_id: target_module_id.to_string(),
14121                channels,
14122                reason: RouteCloseReason::CapabilityDenied,
14123                drained: false,
14124                abandoned: 0,
14125                excluded_subscriptions: 0,
14126                terminal: Some(false),
14127            }
14128        );
14129    }
14130
14131    #[tokio::test]
14132    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14133        let registry = Arc::new(Registry::default());
14134        let forwarding = Arc::new(ForwardingTable::default());
14135        let supervisor = SupervisorHandle::new();
14136        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14137        let handler =
14138            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14139                .with_supervisor(supervisor);
14140        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14141        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14142        register_capability_manifest(
14143            &handler,
14144            &target_ctx,
14145            &mut target_rx,
14146            capability_manifest("target", &["credentials-provider/v1"], &[]),
14147            1,
14148        )
14149        .await;
14150        register_capability_manifest(
14151            &handler,
14152            &opener_ctx,
14153            &mut opener_rx,
14154            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14155            2,
14156        )
14157        .await;
14158
14159        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14160        let replies = handler
14161            .handle_control_frame(
14162                &client_ctx,
14163                route_open_frame_with_admission_facts(
14164                    3,
14165                    "target",
14166                    unique_project_root("admission-facts"),
14167                    Some(ConsumerIdentity {
14168                        module_id: "opener".to_string(),
14169                        launch_nonce: "opener-nonce".to_string(),
14170                    }),
14171                    None,
14172                ),
14173            )
14174            .await
14175            .expect("denied route.open returns a typed frame");
14176        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14177        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14178        assert!(
14179            target_rx.try_recv().is_err(),
14180            "forbidden route.open must not relay route.bind"
14181        );
14182    }
14183
14184    #[tokio::test]
14185    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14186        let registry = Arc::new(Registry::default());
14187        let forwarding = Arc::new(ForwardingTable::default());
14188        let supervisor = SupervisorHandle::new();
14189        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14190        let handler =
14191            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14192                .with_supervisor(supervisor);
14193        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14194        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14195        register_capability_manifest(
14196            &handler,
14197            &target_ctx,
14198            &mut target_rx,
14199            capability_manifest("target", &["credentials-provider/v1"], &[]),
14200            1,
14201        )
14202        .await;
14203        register_capability_manifest(
14204            &handler,
14205            &old_opener_ctx,
14206            &mut old_opener_rx,
14207            capability_manifest("opener", &[], &[]),
14208            2,
14209        )
14210        .await;
14211        let (mut client_rx, _) = open_route_for_capability_test(
14212            &handler,
14213            &target_ctx,
14214            &mut target_rx,
14215            712,
14216            3,
14217            "target",
14218            Some(ConsumerIdentity {
14219                module_id: "opener".to_string(),
14220                launch_nonce: "opener-nonce".to_string(),
14221            }),
14222        )
14223        .await;
14224        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14225
14226        handler
14227            .cleanup_connection(old_opener_ctx.connection_id)
14228            .expect("old opener registration cleans up");
14229        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14230        register_capability_manifest(
14231            &handler,
14232            &new_opener_ctx,
14233            &mut new_opener_rx,
14234            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14235            4,
14236        )
14237        .await;
14238
14239        assert_capability_denied_push(
14240            client_rx
14241                .try_recv()
14242                .expect("HELLO deny addition must emit route.closed")
14243                .frame,
14244            "target",
14245        );
14246        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14247        assert!(matches!(
14248            target_rx.try_recv(),
14249            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14250        ));
14251    }
14252
14253    #[tokio::test]
14254    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14255        let registry = Arc::new(Registry::default());
14256        let forwarding = Arc::new(ForwardingTable::default());
14257        let supervisor = SupervisorHandle::new();
14258        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14259        let handler =
14260            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14261                .with_supervisor(supervisor);
14262        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14263        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14264        register_capability_manifest(
14265            &handler,
14266            &target_ctx,
14267            &mut target_rx,
14268            capability_manifest("target", &[], &[]),
14269            1,
14270        )
14271        .await;
14272        register_capability_manifest(
14273            &handler,
14274            &opener_ctx,
14275            &mut opener_rx,
14276            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14277            2,
14278        )
14279        .await;
14280        let (mut client_rx, _) = open_route_for_capability_test(
14281            &handler,
14282            &target_ctx,
14283            &mut target_rx,
14284            722,
14285            3,
14286            "target",
14287            Some(ConsumerIdentity {
14288                module_id: "opener".to_string(),
14289                launch_nonce: "opener-nonce".to_string(),
14290            }),
14291        )
14292        .await;
14293        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14294
14295        let replies = handler
14296            .handle_control_frame(
14297                &target_ctx,
14298                catalog_update_with_capabilities_frame(
14299                    4,
14300                    CapabilityDeclarations {
14301                        provides: vec!["credentials-provider/v1".to_string()],
14302                        requires: Vec::new(),
14303                        must_never_reach: Vec::new(),
14304                    },
14305                ),
14306            )
14307            .await
14308            .expect("claim catalog.update succeeds");
14309        assert!(matches!(
14310            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14311            Ok(ModuleControlResponseToModule::CatalogUpdate {})
14312        ));
14313        assert_capability_denied_push(
14314            client_rx
14315                .try_recv()
14316                .expect("claim addition must emit route.closed")
14317                .frame,
14318            "target",
14319        );
14320        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14321        assert!(matches!(
14322            target_rx.try_recv(),
14323            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14324        ));
14325    }
14326
14327    #[tokio::test]
14328    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14329        let registry = Arc::new(Registry::default());
14330        let forwarding = Arc::new(ForwardingTable::default());
14331        let supervisor = SupervisorHandle::new();
14332        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14333        let handler =
14334            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14335                .with_supervisor(supervisor);
14336        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14337        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14338        register_capability_manifest(
14339            &handler,
14340            &target_ctx,
14341            &mut target_rx,
14342            capability_manifest("target", &["credentials-provider/v1"], &[]),
14343            1,
14344        )
14345        .await;
14346        register_capability_manifest(
14347            &handler,
14348            &opener_ctx,
14349            &mut opener_rx,
14350            capability_manifest("opener", &[], &[]),
14351            2,
14352        )
14353        .await;
14354        let (mut client_rx, _) = open_route_for_capability_test(
14355            &handler,
14356            &target_ctx,
14357            &mut target_rx,
14358            732,
14359            3,
14360            "target",
14361            Some(ConsumerIdentity {
14362                module_id: "opener".to_string(),
14363                launch_nonce: "opener-nonce".to_string(),
14364            }),
14365        )
14366        .await;
14367        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14368
14369        handler
14370            .handle_control_frame(
14371                &target_ctx,
14372                catalog_update_with_capabilities_frame(
14373                    4,
14374                    CapabilityDeclarations {
14375                        provides: Vec::new(),
14376                        requires: Vec::new(),
14377                        must_never_reach: Vec::new(),
14378                    },
14379                ),
14380            )
14381            .await
14382            .expect("claim removal catalog.update succeeds");
14383        assert_eq!(
14384            forwarding.active_binding_count().unwrap(),
14385            1,
14386            "removing an attested target claim must leave the route census unchanged"
14387        );
14388        assert!(
14389            client_rx.try_recv().is_err(),
14390            "claim removal must not emit route.closed capability_denied"
14391        );
14392        assert!(
14393            target_rx.try_recv().is_err(),
14394            "claim removal must not send the target a route GOODBYE"
14395        );
14396    }
14397
14398    /// A direct client may open a route to a denied capability provider; this
14399    /// policy applies only to attested supervised module origins, not to direct clients.
14400    #[tokio::test]
14401    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14402        let registry = Arc::new(Registry::default());
14403        let forwarding = Arc::new(ForwardingTable::default());
14404        let supervisor = SupervisorHandle::new();
14405        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14406        let handler =
14407            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14408                .with_supervisor(supervisor);
14409        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14410        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14411        register_capability_manifest(
14412            &handler,
14413            &target_ctx,
14414            &mut target_rx,
14415            capability_manifest("target", &["credentials-provider/v1"], &[]),
14416            1,
14417        )
14418        .await;
14419        register_capability_manifest(
14420            &handler,
14421            &opener_ctx,
14422            &mut opener_rx,
14423            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14424            2,
14425        )
14426        .await;
14427
14428        let (_client_rx, bind) = open_route_for_capability_test(
14429            &handler,
14430            &target_ctx,
14431            &mut target_rx,
14432            742,
14433            3,
14434            "target",
14435            None,
14436        )
14437        .await;
14438        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14439            panic!("direct scope-honesty route must bind");
14440        };
14441        assert_eq!(principal, Some(Principal::Direct));
14442        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14443    }
14444
14445    /// A module that denies a capability receives no self-route exemption when it
14446    /// also attestedly provides that capability.
14447    #[tokio::test]
14448    async fn must_never_reach_self_route_is_capability_forbidden() {
14449        let registry = Arc::new(Registry::default());
14450        let forwarding = Arc::new(ForwardingTable::default());
14451        let supervisor = SupervisorHandle::new();
14452        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14453        let handler =
14454            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14455                .with_supervisor(supervisor);
14456        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14457        register_capability_manifest(
14458            &handler,
14459            &self_ctx,
14460            &mut self_rx,
14461            capability_manifest(
14462                "self-provider",
14463                &["credentials-provider/v1"],
14464                &["credentials-provider/v1"],
14465            ),
14466            1,
14467        )
14468        .await;
14469
14470        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14471        let replies = handler
14472            .handle_control_frame(
14473                &client_ctx,
14474                route_open_frame_with_admission_facts(
14475                    2,
14476                    "self-provider",
14477                    unique_project_root("admission-facts"),
14478                    Some(ConsumerIdentity {
14479                        module_id: "self-provider".to_string(),
14480                        launch_nonce: "self-nonce".to_string(),
14481                    }),
14482                    None,
14483                ),
14484            )
14485            .await
14486            .expect("self-route refusal returns a typed frame");
14487        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14488        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14489        assert!(
14490            self_rx.try_recv().is_err(),
14491            "self denial must not relay route.bind"
14492        );
14493    }
14494
14495    #[test]
14496    fn unsupported_channel_zero_frame_returns_error() {
14497        let handler = ControlHandler::default();
14498        let request = Frame::build(
14499            FrameType::Request,
14500            control_flags(),
14501            0,
14502            0,
14503            21,
14504            b"opaque".to_vec(),
14505        )
14506        .unwrap();
14507
14508        let response = handler
14509            .handle_control(ConnectionId::new(1), request)
14510            .unwrap();
14511
14512        assert_eq!(response[0].header.ty, FrameType::Error);
14513        assert_eq!(
14514            parse_error(&response[0])["code"],
14515            "unsupported_control_frame"
14516        );
14517    }
14518
14519    /// Blue/green swap at the control-plane boundary. The supervisor that opens
14520    /// a swap is not wired yet, so the candidate is registered here directly
14521    /// into the registry and forwarding candidate slots, the way the swap's
14522    /// HELLO admission will.
14523    mod swap {
14524        use super::*;
14525
14526        const INCUMBENT: ConnectionId = ConnectionId::new(30);
14527        const CANDIDATE: ConnectionId = ConnectionId::new(40);
14528
14529        struct Swap {
14530            registry: Arc<Registry>,
14531            forwarding: Arc<ForwardingTable>,
14532            handler: ControlHandler,
14533            incumbent_ctx: RouteCtx,
14534            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14535            candidate_ctx: RouteCtx,
14536            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14537        }
14538
14539        async fn swap_with_incumbent() -> Swap {
14540            let registry = Arc::new(Registry::default());
14541            let forwarding = Arc::new(ForwardingTable::default());
14542            let handler =
14543                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14544            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14545            hello_via_sink(
14546                &handler,
14547                &incumbent_ctx,
14548                &mut incumbent_rx,
14549                hello_frame("aft", PROTOCOL_VERSION, 7),
14550            )
14551            .await;
14552            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14553            Swap {
14554                registry,
14555                forwarding,
14556                handler,
14557                incumbent_ctx,
14558                incumbent_rx,
14559                candidate_ctx,
14560                candidate_rx,
14561            }
14562        }
14563
14564        fn register_candidate(swap: &Swap, ready: Option<bool>) {
14565            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14566            candidate_manifest.ready = ready;
14567            let registration = swap
14568                .registry
14569                .register_candidate_with_control_ops(
14570                    candidate_manifest,
14571                    PROTOCOL_VERSION,
14572                    CANDIDATE,
14573                    module_baseline_control_ops(),
14574                )
14575                .unwrap();
14576            swap.forwarding
14577                .register_candidate_module_connection(
14578                    CANDIDATE,
14579                    "aft".to_string(),
14580                    PROTOCOL_VERSION,
14581                    manifest_concurrency(&registration.manifest),
14582                    swap.candidate_ctx.egress.clone(),
14583                )
14584                .unwrap();
14585        }
14586
14587        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14588            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14589            swap.registry.promote_candidate("aft").unwrap().unwrap();
14590            cutover.incumbent.unwrap()
14591        }
14592
14593        /// A swap candidate's HELLO reserves its candidate registry entry
14594        /// before installing its candidate forwarding endpoint. The swap
14595        /// supervisor promotes a ready candidate as soon as it reads it as
14596        /// registered, and promotion cuts forwarding over to the candidate
14597        /// endpoint, so the lookup the supervisor reads
14598        /// (`Registry::published_registration`) must not return the candidate
14599        /// until that endpoint exists. Otherwise the cutover finds no candidate
14600        /// endpoint and the swap fails as if the candidate had disconnected.
14601        #[tokio::test]
14602        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14603            let mut swap = swap_with_incumbent().await;
14604            let hold = Arc::new(std::sync::Barrier::new(2));
14605            let handler = swap
14606                .handler
14607                .clone()
14608                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14609            handler
14610                .supervisor
14611                .open_swap("aft", "candidate-nonce".to_string());
14612            let hello = {
14613                let handler = handler.clone();
14614                let candidate_ctx = swap.candidate_ctx.clone();
14615                std::thread::spawn(move || {
14616                    handler
14617                        .handle_hello(
14618                            candidate_ctx.connection_id,
14619                            Some(candidate_ctx.egress.clone()),
14620                            hello_frame_with_nonce(
14621                                "aft",
14622                                PROTOCOL_VERSION,
14623                                8,
14624                                Some("candidate-nonce"),
14625                            ),
14626                        )
14627                        .unwrap()
14628                })
14629            };
14630
14631            hold.wait();
14632            // Parked: the candidate slot is claimed but not published, and
14633            // forwarding has no candidate endpoint.
14634            assert!(swap
14635                .registry
14636                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14637                .unwrap()
14638                .is_some());
14639            assert!(swap
14640                .registry
14641                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14642                .unwrap()
14643                .is_none());
14644            assert!(swap
14645                .forwarding
14646                .module_endpoint_for_connection(CANDIDATE)
14647                .unwrap()
14648                .is_none());
14649
14650            hold.wait();
14651            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14652            assert_eq!(
14653                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14654                FrameType::HelloAck
14655            );
14656            let published = swap
14657                .registry
14658                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14659                .unwrap()
14660                .expect("candidate is published once its HELLO completes");
14661            assert_eq!(published.connection_id, CANDIDATE);
14662            assert!(swap
14663                .forwarding
14664                .module_endpoint_for_connection(CANDIDATE)
14665                .unwrap()
14666                .is_some());
14667        }
14668
14669        fn keyed_total(counters: &Value, key: &str) -> u64 {
14670            counters[key]
14671                .as_object()
14672                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14673                .unwrap_or(0)
14674        }
14675
14676        #[tokio::test]
14677        async fn replacement_logs_old_end_and_new_admission_once() {
14678            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14679            let mut swap = swap_with_incumbent().await;
14680            swap.handler
14681                .supervisor
14682                .open_swap("aft", "candidate-nonce".to_string());
14683            hello_via_sink(
14684                &swap.handler,
14685                &swap.candidate_ctx,
14686                &mut swap.candidate_rx,
14687                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14688            )
14689            .await;
14690            cutover(&swap);
14691            swap.handler.cleanup_connection(INCUMBENT).unwrap();
14692
14693            let captured = crate::router::test_log::captured_logs(&logs);
14694            println!("captured replacement registry logs:\n{captured}");
14695            let new_admissions: Vec<_> = captured
14696                .lines()
14697                .filter(|line| {
14698                    line.contains("connection_id=40")
14699                        && line.contains("swap candidate registered; not routable until cutover")
14700                })
14701                .collect();
14702            assert_eq!(
14703                new_admissions.len(),
14704                1,
14705                "unexpected admission logs: {captured}"
14706            );
14707            assert!(new_admissions[0].contains("module_id=aft"));
14708            assert!(new_admissions[0].contains("connection_id=40"));
14709            assert!(
14710                new_admissions[0].contains("swap candidate registered; not routable until cutover")
14711            );
14712            assert!(new_admissions[0].contains("ready=true"));
14713
14714            let old_admissions: Vec<_> = captured
14715                .lines()
14716                .filter(|line| {
14717                    line.contains("module registered module_id=aft ")
14718                        && line.contains("connection_id=30")
14719                })
14720                .collect();
14721            assert_eq!(
14722                old_admissions.len(),
14723                1,
14724                "unexpected incumbent admission: {captured}"
14725            );
14726
14727            let old_ends: Vec<_> = captured
14728                .lines()
14729                .filter(|line| {
14730                    line.contains("connection_id=30") && line.contains("module registration ended")
14731                })
14732                .collect();
14733            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14734            assert!(old_ends[0].contains("module_id=aft"));
14735            assert!(old_ends[0].contains("connection_id=30"));
14736            assert!(old_ends[0].contains("reason=replaced"));
14737            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14738            assert_eq!(
14739                captured
14740                    .lines()
14741                    .filter(|line| line.contains("module registration promoted"))
14742                    .count(),
14743                1,
14744                "unexpected promotion logs: {captured}"
14745            );
14746        }
14747
14748        /// An ack from the incumbent for a bind it was sent before cutover,
14749        /// arriving before the incumbent is drained. The incumbent is the live
14750        /// connection carrying every other client's routes, so the ack must
14751        /// not end it: the waiting client is told to retry, the reservation is
14752        /// given back, and the incumbent is told to drop just that binding.
14753        #[tokio::test]
14754        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14755            let mut swap = swap_with_incumbent().await;
14756            let handler = swap.handler.clone();
14757
14758            // A co-tenant route, bound on the incumbent before the swap.
14759            let cotenant = ConnectionId::new(31);
14760            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14761            let (cotenant_task, cotenant_bind) = relay_route_open(
14762                &handler,
14763                cotenant,
14764                &cotenant_ctx.egress,
14765                &mut swap.incumbent_rx,
14766                100,
14767                "aft",
14768                "swap-cotenant",
14769            )
14770            .await;
14771            handler
14772                .handle_control_frame(
14773                    &swap.incumbent_ctx,
14774                    route_bind_ack(cotenant_bind.header.corr),
14775                )
14776                .await
14777                .unwrap();
14778            assert!(cotenant_task.await.unwrap().is_empty());
14779            let (cotenant_channel, cotenant_epoch) =
14780                published_route(&cotenant_rx.recv().await.unwrap());
14781
14782            // A second route.open, relayed to the incumbent and not yet acked.
14783            let caller = ConnectionId::new(32);
14784            let (caller_ctx, mut caller_rx) = route_ctx(caller);
14785            let (caller_task, caller_bind) = relay_route_open(
14786                &handler,
14787                caller,
14788                &caller_ctx.egress,
14789                &mut swap.incumbent_rx,
14790                101,
14791                "aft",
14792                "swap-caller",
14793            )
14794            .await;
14795            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14796
14797            register_candidate(&swap, None);
14798            cutover(&swap);
14799
14800            // The incumbent acks after promotion and before any drain.
14801            let ack = handler
14802                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14803                .await;
14804            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14805            if module_loop_error.is_some() {
14806                // What the connection loop does with an untranslated router
14807                // error: end the connection, releasing every route on it.
14808                handler.cleanup_connection(INCUMBENT).unwrap();
14809            }
14810
14811            // 1. The incumbent's other routes survive.
14812            assert!(
14813                cotenant_rx.try_recv().is_err(),
14814                "the co-tenant route on the incumbent was torn down by one late ack: \
14815                 {module_loop_error:?}"
14816            );
14817            assert!(matches!(
14818                swap.forwarding
14819                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14820                    .unwrap(),
14821                DataRoute::Client(DataRouteState::Bound(_))
14822            ));
14823            assert_eq!(module_loop_error, None);
14824            assert!(swap
14825                .registry
14826                .get_module_by_connection(INCUMBENT)
14827                .unwrap()
14828                .is_some());
14829
14830            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
14831            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14832                .await
14833                .expect("the incumbent is told to drop the abandoned binding")
14834                .unwrap()
14835                .frame;
14836            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14837            assert_eq!(goodbye.header.channel, abandoned_channel);
14838            assert_eq!(goodbye.header.epoch, abandoned_epoch);
14839            assert!(swap.incumbent_rx.try_recv().is_err());
14840
14841            // 3. The waiting client gets a retryable refusal and no route.
14842            let response = caller_task.await.unwrap();
14843            assert_eq!(response.len(), 1);
14844            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14845            assert!(caller_rx.try_recv().is_err());
14846
14847            // 4. The reservation pair is given back, and the pending bind
14848            //    settled exactly once: one accepted open (the co-tenant) and one
14849            //    refused open (the caller), nothing counted twice.
14850            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14851            let counters = handler.counters().snapshot();
14852            assert_eq!(
14853                keyed_total(&counters, "route_open_accepted_by_principal"),
14854                1
14855            );
14856            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14857            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14858        }
14859
14860        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
14861        /// id would resolve to the promoted candidate and every new route.open
14862        /// would be refused as reloading, leaving neither process routable.
14863        #[tokio::test]
14864        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14865            let mut swap = swap_with_incumbent().await;
14866            register_candidate(&swap, None);
14867            let incumbent = cutover(&swap);
14868            swap.forwarding
14869                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14870                .unwrap()
14871                .expect("the incumbent is still registered");
14872
14873            let client = ConnectionId::new(33);
14874            let (client_ctx, mut client_rx) = route_ctx(client);
14875            let route_handler = swap.handler.clone();
14876            let open_ctx = RouteCtx {
14877                connection_id: client,
14878                egress: client_ctx.egress.clone(),
14879            };
14880            let mut route_task = tokio::spawn(async move {
14881                route_handler
14882                    .handle_control_frame(
14883                        &open_ctx,
14884                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14885                    )
14886                    .await
14887                    .unwrap()
14888            });
14889            let bind = tokio::select! {
14890                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14891                response = &mut route_task => {
14892                    let response = response.unwrap();
14893                    panic!(
14894                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
14895                        parse_error(&response[0])["code"]
14896                    );
14897                }
14898            };
14899            swap.handler
14900                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14901                .await
14902                .unwrap();
14903            assert!(route_task.await.unwrap().is_empty());
14904            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14905            match swap
14906                .forwarding
14907                .lookup_data_route(client, channel, epoch)
14908                .unwrap()
14909            {
14910                DataRoute::Client(DataRouteState::Bound(route)) => {
14911                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14912                }
14913                other => panic!("expected a bound route on the candidate, got {other:?}"),
14914            }
14915            assert!(swap.incumbent_rx.try_recv().is_err());
14916        }
14917
14918        /// A candidate declares itself ready with `catalog.update` on its own
14919        /// connection. If the connection-keyed registry lookups searched only the
14920        /// active slot, this would answer `not_registered` and the candidate
14921        /// would never become ready.
14922        #[tokio::test]
14923        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14924            let swap = swap_with_incumbent().await;
14925            register_candidate(&swap, Some(false));
14926            let update = Frame::build(
14927                FrameType::Request,
14928                control_flags(),
14929                0,
14930                0,
14931                55,
14932                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14933                    provides: manifest("aft", PROTOCOL_VERSION).provides,
14934                    capabilities: None,
14935                    ready: Some(true),
14936                })
14937                .unwrap(),
14938            )
14939            .unwrap();
14940
14941            let replies = swap
14942                .handler
14943                .handle_control_frame(&swap.candidate_ctx, update)
14944                .await
14945                .unwrap();
14946
14947            assert_eq!(replies.len(), 1);
14948            assert_eq!(
14949                replies[0].header.ty,
14950                FrameType::Response,
14951                "candidate catalog.update was refused: {:?}",
14952                serde_json::from_slice::<Value>(&replies[0].body).ok()
14953            );
14954            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14955            assert_eq!(
14956                swap.registry
14957                    .get_module("aft")
14958                    .unwrap()
14959                    .unwrap()
14960                    .connection_id,
14961                INCUMBENT
14962            );
14963        }
14964    }
14965
14966    /// The HELLO gate while the supervisor has a swap open: only the nonce it
14967    /// minted for the candidate admits a second process, into the candidate
14968    /// slot, and that check runs ahead of the reserved-module gate.
14969    mod swap_admission {
14970        use super::*;
14971
14972        const INCUMBENT_NONCE: &str = "incumbent-nonce";
14973        const CANDIDATE_NONCE: &str = "candidate-nonce";
14974
14975        fn handler_with_incumbent(
14976            module_id: &str,
14977            reserved: bool,
14978        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14979            let registry = Arc::new(Registry::default());
14980            let supervisor = SupervisorHandle::new();
14981            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14982            if reserved {
14983                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14984            }
14985            let handler =
14986                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
14987            let incumbent = handler
14988                .handle_control(
14989                    ConnectionId::new(1),
14990                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14991                )
14992                .unwrap();
14993            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14994            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14995            (registry, supervisor, handler)
14996        }
14997
14998        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
14999        /// admits every nonce, so while a swap is open the swap gate is the only
15000        /// thing between a key-holder and the candidate slot. A nonce the
15001        /// supervisor did not mint, or none at all, is refused, and neither the
15002        /// incumbent's registration nor the candidate slot moves.
15003        #[test]
15004        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15005            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15006
15007            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15008                let replies = handler
15009                    .handle_control(
15010                        ConnectionId::new(connection),
15011                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15012                    )
15013                    .unwrap();
15014                assert_eq!(replies[0].header.ty, FrameType::Error);
15015                assert_eq!(
15016                    parse_error(&replies[0])["code"],
15017                    "swap_token_invalid",
15018                    "nonce {nonce:?}"
15019                );
15020            }
15021            assert!(registry.get_candidate("aft").unwrap().is_none());
15022            assert_eq!(
15023                registry.get_module("aft").unwrap().unwrap().connection_id,
15024                ConnectionId::new(1)
15025            );
15026
15027            // Control: the minted token is admitted, into the candidate slot,
15028            // and only once.
15029            let admitted = handler
15030                .handle_control(
15031                    ConnectionId::new(4),
15032                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15033                )
15034                .unwrap();
15035            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15036            assert_eq!(
15037                registry
15038                    .get_candidate("aft")
15039                    .unwrap()
15040                    .unwrap()
15041                    .connection_id,
15042                ConnectionId::new(4)
15043            );
15044            assert_eq!(
15045                registry.get_module("aft").unwrap().unwrap().connection_id,
15046                ConnectionId::new(1),
15047                "the candidate must not take the active slot"
15048            );
15049            let replayed = handler
15050                .handle_control(
15051                    ConnectionId::new(5),
15052                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15053                )
15054                .unwrap();
15055            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15056
15057            // The case only this gate covers: the incumbent has died mid-swap,
15058            // so its duplicate refusal is gone too, and without the gate a
15059            // key-holder would take the id's ACTIVE slot.
15060            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15061            let squatter = handler
15062                .handle_control(
15063                    ConnectionId::new(6),
15064                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15065                )
15066                .unwrap();
15067            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15068            assert!(
15069                registry.get_module("aft").unwrap().is_none(),
15070                "a squatter took the active slot of an id being swapped"
15071            );
15072        }
15073
15074        /// Design mutation arm (iii). A reserved module's candidate presents a
15075        /// nonce the reserved gate has never seen (that gate holds the
15076        /// incumbent's), so the swap gate must run first or the candidate is
15077        /// refused `reserved_module` and a reserved module can never be swapped.
15078        #[test]
15079        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15080            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15081
15082            let replies = handler
15083                .handle_control(
15084                    ConnectionId::new(2),
15085                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15086                )
15087                .unwrap();
15088
15089            assert_eq!(
15090                replies[0].header.ty,
15091                FrameType::HelloAck,
15092                "reserved candidate refused: {:?}",
15093                serde_json::from_slice::<Value>(&replies[0].body).ok()
15094            );
15095            assert_eq!(
15096                registry
15097                    .get_candidate("vault")
15098                    .unwrap()
15099                    .unwrap()
15100                    .connection_id,
15101                ConnectionId::new(2)
15102            );
15103        }
15104
15105        /// With no swap open the gate is inert: the incumbent's reserved gate
15106        /// and duplicate refusal behave exactly as before.
15107        #[test]
15108        fn without_an_open_swap_the_ordinary_gates_decide() {
15109            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15110            supervisor.close_swap("vault");
15111
15112            let candidate = handler
15113                .handle_control(
15114                    ConnectionId::new(2),
15115                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15116                )
15117                .unwrap();
15118            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15119            let duplicate = handler
15120                .handle_control(
15121                    ConnectionId::new(3),
15122                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15123                )
15124                .unwrap();
15125            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15126            assert!(registry.get_candidate("vault").unwrap().is_none());
15127        }
15128    }
15129
15130    /// `scope.sync` and `scope.describe` through the real control handler: who
15131    /// may sync is decided by the registration and launch nonce of the module
15132    /// connection, never by the request body.
15133    mod scopes {
15134        mod idle_tests {
15135            use super::*;
15136            include!("scope_idle_tests.rs");
15137        }
15138        mod route_open_timings {
15139            use super::*;
15140            include!("route_open_timing_tests.rs");
15141        }
15142        mod activity {
15143            include!("activity_scope_tests.rs");
15144        }
15145        mod delta {
15146            include!("scope_handler_delta_tests.rs");
15147        }
15148        mod call_modes {
15149            include!("scope_call_mode_tests.rs");
15150        }
15151        #[tokio::test]
15152        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15153            let rig = rig().await;
15154            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15155                "s".into(),
15156                1,
15157                "tool".into(),
15158            ))
15159            .unwrap();
15160            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15161            let (direct, _rx) = wide_ctx(999);
15162            for (ctx, expected) in [
15163                (&rig.owner, FrameType::Response),
15164                (&direct, FrameType::Error),
15165            ] {
15166                let frame =
15167                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15168                        .unwrap();
15169                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15170                assert_eq!(replies[0].header.ty, expected);
15171                if expected == FrameType::Response {
15172                    let reply: subc_control::ScopeServedByReply =
15173                        serde_json::from_slice(&replies[0].body).unwrap();
15174                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15175                } else {
15176                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15177                }
15178            }
15179            let reply = call(
15180                &rig.handler,
15181                &rig.owner,
15182                &ModuleControlRequestFromModule::ScopeDescribe {
15183                    owner: Principal::Reserved {
15184                        module_id: OWNER.into(),
15185                    },
15186                    scope_ref: "s".into(),
15187                },
15188            )
15189            .await;
15190            assert!(matches!(
15191                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15192                ModuleControlResponseToModule::ScopeDescribe { .. }
15193            ));
15194        }
15195
15196        use subc_protocol::scope::{
15197            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15198            ScopeStatus,
15199        };
15200
15201        use super::*;
15202
15203        const OWNER: &str = "prefrontal-core";
15204
15205        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15206            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15207        }
15208
15209        async fn call(
15210            handler: &ControlHandler,
15211            ctx: &RouteCtx,
15212            request: &ModuleControlRequestFromModule,
15213        ) -> Frame {
15214            let body = serde_json::to_vec(request).unwrap();
15215            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15216            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15217            assert_eq!(replies.len(), 1, "{replies:?}");
15218            replies.pop().unwrap()
15219        }
15220
15221        async fn sync(
15222            handler: &ControlHandler,
15223            ctx: &RouteCtx,
15224            generation: u64,
15225            scopes: Vec<ScopeRecord>,
15226        ) -> Result<ModuleControlResponseToModule, String> {
15227            let reply = call(
15228                handler,
15229                ctx,
15230                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15231            )
15232            .await;
15233            match reply.header.ty {
15234                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15235                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15236            }
15237        }
15238
15239        async fn describe(
15240            handler: &ControlHandler,
15241            ctx: &RouteCtx,
15242            owner: &str,
15243            scope_ref: &str,
15244        ) -> ModuleControlResponseToModule {
15245            let reply = call(
15246                handler,
15247                ctx,
15248                &ModuleControlRequestFromModule::ScopeDescribe {
15249                    owner: Principal::Reserved {
15250                        module_id: owner.to_string(),
15251                    },
15252                    scope_ref: scope_ref.to_string(),
15253                },
15254            )
15255            .await;
15256            assert_eq!(
15257                reply.header.ty,
15258                FrameType::Response,
15259                "{:?}",
15260                parse_error(&reply)
15261            );
15262            serde_json::from_slice(&reply.body).unwrap()
15263        }
15264
15265        #[tokio::test]
15266        async fn scope_describe_never_echoes_bind_proof_or_hash() {
15267            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15268            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15269            let mut rig = rig().await;
15270            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15271                .await;
15272            let _bound = rig
15273                .bound(
15274                    Some(AFT),
15275                    PLEXUS,
15276                    Some(
15277                        rig.selector("s", Some(1))
15278                            .with_bind_proof(Some(PROOF.into())),
15279                    ),
15280                )
15281                .await;
15282            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15283            let output = serde_json::to_string(&reply).unwrap();
15284            assert!(output.contains("\"status\":\"live\""));
15285            assert!(!output.contains(PROOF));
15286            assert!(!output.contains(HASH));
15287        }
15288
15289        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
15290        async fn module(
15291            handler: &ControlHandler,
15292            connection: u64,
15293            module_id: &str,
15294            nonce: Option<&str>,
15295        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15296            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15297            hello_via_sink(
15298                handler,
15299                &ctx,
15300                &mut rx,
15301                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15302            )
15303            .await;
15304            (ctx, rx)
15305        }
15306
15307        /// `direct` and every other client connection has no registration, so
15308        /// it can neither sync nor own a scope.
15309        #[tokio::test]
15310        async fn a_client_connection_cannot_sync_or_describe() {
15311            let handler = ControlHandler::new(Arc::new(Registry::default()));
15312            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15313            for request in [
15314                ModuleControlRequestFromModule::ScopeSync {
15315                    generation: 1,
15316                    scopes: vec![head("s", 1)],
15317                },
15318                ModuleControlRequestFromModule::ScopeDescribe {
15319                    owner: Principal::Direct,
15320                    scope_ref: "s".to_string(),
15321                },
15322            ] {
15323                let reply = call(&handler, &ctx, &request).await;
15324                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15325            }
15326            assert!(
15327                !handler
15328                    .scopes
15329                    .read()
15330                    .unwrap()
15331                    .describe(
15332                        &Principal::Reserved {
15333                            module_id: OWNER.to_string()
15334                        },
15335                        "s"
15336                    )
15337                    .owner_synced
15338            );
15339        }
15340
15341        /// A module the supervisor did not spawn registers without a launch
15342        /// nonce, so it is never an owner's current launch.
15343        #[tokio::test]
15344        async fn a_module_without_a_supervised_launch_cannot_sync() {
15345            let handler = ControlHandler::new(Arc::new(Registry::default()));
15346            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15347            assert_eq!(
15348                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15349                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15350            );
15351        }
15352
15353        #[tokio::test]
15354        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15355            let supervisor = SupervisorHandle::new();
15356            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15357            let handler = ControlHandler::new(Arc::new(Registry::default()))
15358                .with_supervisor(supervisor.clone());
15359            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15360            sync(&handler, &incumbent, 1, vec![head("s", 1)])
15361                .await
15362                .expect("the current launch syncs");
15363
15364            // A swap candidate registers with the swap token and is refused
15365            // while the incumbent keeps syncing.
15366            supervisor.open_swap(OWNER, "n2".to_string());
15367            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15368            assert_eq!(
15369                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15370                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15371            );
15372            sync(&handler, &incumbent, 2, vec![head("s", 1)])
15373                .await
15374                .expect("the serving owner syncs during the swap");
15375
15376            // The swap fails and is rolled back. The candidate never held sync
15377            // authority, and still cannot sync.
15378            supervisor.close_swap(OWNER);
15379            assert_eq!(
15380                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15381                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15382            );
15383            sync(&handler, &incumbent, 3, vec![head("s", 1)])
15384                .await
15385                .expect("the serving owner syncs after the rollback");
15386            handler.cleanup_connection(candidate.connection_id).unwrap();
15387
15388            // A swap that cuts over. Promotion records the candidate's nonce as
15389            // the module's spawn nonce, which is what `set_spawn_nonce` does
15390            // here; the promoted connection then takes authority at any
15391            // generation and the superseded incumbent is refused.
15392            supervisor.open_swap(OWNER, "n3".to_string());
15393            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15394            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15395            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15396                .await
15397                .expect("the promoted launch takes authority");
15398            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15399                panic!("unexpected reply {reply:?}");
15400            };
15401            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15402            assert_eq!(
15403                sync(&handler, &incumbent, 4, Vec::new()).await,
15404                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15405            );
15406        }
15407
15408        /// Authority dies with its connection: the cleanup path releases it,
15409        /// so the owner's next connection takes it at any generation.
15410        #[tokio::test]
15411        async fn closing_the_authority_connection_frees_sync_authority() {
15412            let supervisor = SupervisorHandle::new();
15413            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15414            let handler = ControlHandler::new(Arc::new(Registry::default()))
15415                .with_supervisor(supervisor.clone());
15416            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15417            sync(&handler, &first, 10, vec![head("s", 1)])
15418                .await
15419                .unwrap();
15420            handler.cleanup_connection(first.connection_id).unwrap();
15421
15422            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15423            sync(&handler, &second, 1, vec![head("s", 1)])
15424                .await
15425                .expect("the next connection takes the released authority");
15426        }
15427
15428        #[tokio::test]
15429        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15430            let supervisor = SupervisorHandle::new();
15431            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15432            let handler =
15433                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15434            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15435            sync(&handler, &first, 10, vec![head("s", 1)])
15436                .await
15437                .unwrap();
15438            handler
15439                .handle_control_frame(
15440                    &first,
15441                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15442                )
15443                .await
15444                .unwrap();
15445            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15446            sync(&handler, &second, 1, vec![head("s", 1)])
15447                .await
15448                .expect("GOODBYE releases authority even if the old socket remains open");
15449        }
15450
15451        #[tokio::test]
15452        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15453            let registry = Arc::new(Registry::default());
15454            let supervisor_handle = SupervisorHandle::new();
15455            let supervisor =
15456                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15457                    .with_handle(supervisor_handle.clone())
15458                    .with_daemon_incarnation("incarnation-7".to_string());
15459            // Configured with enabled: false, so the supervisor lists the
15460            // module without spawning a process for it.
15461            supervisor
15462                .supervise_configured(
15463                    ModuleSpec {
15464                        module_id: OWNER.to_string(),
15465                        program: PathBuf::from("/nonexistent/prefrontal-core"),
15466                        args: Vec::new(),
15467                        env: Vec::new(),
15468                        reserved: false,
15469                        reserved_prefixes: Vec::new(),
15470                        protocol: ModuleProtocol::Subc,
15471                        overlap: Default::default(),
15472                    },
15473                    false,
15474                )
15475                .unwrap();
15476            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15477            let handler =
15478                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15479            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15480
15481            // Configured but not yet synced: a reader waits for the owner.
15482            let ModuleControlResponseToModule::ScopeDescribe {
15483                status,
15484                daemon_incarnation,
15485                owner_synced,
15486                owner_configured,
15487                scope,
15488                ..
15489            } = describe(&handler, &reader, OWNER, "s").await
15490            else {
15491                panic!("not a describe reply");
15492            };
15493            assert_eq!(status, ScopeStatus::NotLive);
15494            assert_eq!(daemon_incarnation, "incarnation-7");
15495            assert!(!owner_synced);
15496            assert!(owner_configured);
15497            assert!(scope.is_none());
15498
15499            // Not a supervised module: the owner will never sync, and a reader
15500            // refuses rather than waits.
15501            let ModuleControlResponseToModule::ScopeDescribe {
15502                status,
15503                owner_configured,
15504                ..
15505            } = describe(&handler, &reader, "ghost", "s").await
15506            else {
15507                panic!("not a describe reply");
15508            };
15509            assert_eq!(status, ScopeStatus::NotLive);
15510            assert!(!owner_configured);
15511
15512            // Live, with the stamp fields and the computed owner_authorized.
15513            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15514            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15515            let ModuleControlResponseToModule::ScopeDescribe {
15516                status,
15517                scope_epoch,
15518                owner_synced,
15519                scope,
15520                ..
15521            } = describe(&handler, &reader, OWNER, "s").await
15522            else {
15523                panic!("not a describe reply");
15524            };
15525            assert_eq!(status, ScopeStatus::Live);
15526            assert_eq!(scope_epoch, Some(4));
15527            assert!(owner_synced);
15528            let stamp = scope.expect("a live scope carries its stamp");
15529            assert!(
15530                stamp.owner_authorized,
15531                "prefrontal-core is the default authority"
15532            );
15533            assert_eq!(stamp.kind, ScopeKind::Head);
15534        }
15535
15536        #[tokio::test]
15537        async fn scope_authority_owners_decides_owner_authorized() {
15538            let supervisor = SupervisorHandle::new();
15539            supervisor.set_spawn_nonce("broca", "b1".to_string());
15540            let handler = ControlHandler::new(Arc::new(Registry::default()))
15541                .with_supervisor(supervisor)
15542                .with_scope_authority_owners(vec!["broca".to_string()]);
15543            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15544            let mut gated = head("s", 1);
15545            gated.attributes.agent_id = Some("agent".to_string());
15546            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15547            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15548                describe(&handler, &broca, "broca", "s").await
15549            else {
15550                panic!("not a describe reply");
15551            };
15552            assert!(scope.unwrap().owner_authorized);
15553        }
15554
15555        /// With route admission, the stamp, the commit re-check and drains in
15556        /// place, the feature is advertised: the module ops in HELLO_ACK, and
15557        /// `scopes/v1` in HELLO_ACK and `server.describe`.
15558        #[tokio::test]
15559        async fn scope_ops_and_the_scopes_capability_are_advertised() {
15560            let handler = ControlHandler::new(Arc::new(Registry::default()));
15561            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15562            let ack = hello_via_sink(
15563                &handler,
15564                &ctx,
15565                &mut rx,
15566                hello_frame("m", PROTOCOL_VERSION, 1),
15567            )
15568            .await;
15569            let ack = parse_ack(&ack);
15570            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15571                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15572            }
15573            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15574
15575            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15576            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15577            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15578            let reply = handler
15579                .handle_control_frame(&client, frame)
15580                .await
15581                .unwrap()
15582                .pop()
15583                .unwrap();
15584            let ClientControlResponse::ServerDescribe { capabilities, .. } =
15585                serde_json::from_slice(&reply.body).unwrap()
15586            else {
15587                panic!("not a server.describe reply");
15588            };
15589            assert!(
15590                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15591                "{capabilities:?}"
15592            );
15593        }
15594
15595        // ---- route admission, stamps, commit re-check and drains ----------
15596
15597        const PLEXUS: &str = "plexus";
15598        const OTHER: &str = "other";
15599        const AFT: &str = "aft";
15600        const BROCA: &str = "broca";
15601        const MAGIC: &str = "magic-context";
15602
15603        fn nonce(module_id: &str) -> String {
15604            format!("nonce-{module_id}")
15605        }
15606
15607        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15608            let (tx, rx) = mpsc::channel(64);
15609            (
15610                RouteCtx {
15611                    connection_id: ConnectionId::new(connection),
15612                    egress: FrameSink::new(tx),
15613                },
15614                rx,
15615            )
15616        }
15617
15618        /// A daemon with a configured owner (prefrontal-core) registered on its
15619        /// own module connection, two routable targets (plexus, other), and
15620        /// launch nonces minted for the modules that open routes as carriers.
15621        struct Rig {
15622            handler: ControlHandler,
15623            forwarding: Arc<ForwardingTable>,
15624            owner: RouteCtx,
15625            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15626            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15627            generation: u64,
15628            next_connection: u64,
15629            _supervisor: Supervisor,
15630        }
15631
15632        async fn rig() -> Rig {
15633            rig_with_flow_support(true).await
15634        }
15635
15636        async fn rig_with_flow_support(flow_support: bool) -> Rig {
15637            rig_with_scope_capabilities(flow_support, false).await
15638        }
15639
15640        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15641            let mut provides = Vec::new();
15642            if flow_support {
15643                provides.push("flow-scopes/v1");
15644            }
15645            if run_support {
15646                provides.push("agent-run-scopes/v1");
15647            }
15648            rig_with_target_capabilities(&provides).await
15649        }
15650
15651        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15652            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15653            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15654            // Decoder versions alone cannot admit scoped calls: only a capability
15655            // declaration promises the required run or flow behaviour.
15656            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15657            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15658            Frame::build(
15659                FrameType::Hello,
15660                control_flags(),
15661                0,
15662                0,
15663                connection,
15664                serde_json::to_vec(&body).unwrap(),
15665            )
15666            .unwrap()
15667        }
15668
15669        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15670            let registry = Arc::new(Registry::default());
15671            let forwarding = Arc::new(ForwardingTable::default());
15672            let supervisor_handle = SupervisorHandle::new();
15673            let supervisor =
15674                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15675                    .with_handle(supervisor_handle.clone());
15676            supervisor
15677                .supervise_configured(
15678                    ModuleSpec {
15679                        module_id: OWNER.to_string(),
15680                        program: PathBuf::from("/nonexistent/prefrontal-core"),
15681                        args: Vec::new(),
15682                        env: Vec::new(),
15683                        reserved: false,
15684                        reserved_prefixes: Vec::new(),
15685                        protocol: ModuleProtocol::Subc,
15686                        overlap: Default::default(),
15687                    },
15688                    false,
15689                )
15690                .unwrap();
15691            for module_id in [OWNER, AFT, BROCA, MAGIC] {
15692                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15693            }
15694            let handler =
15695                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15696                    .with_supervisor(supervisor_handle);
15697            let (owner, mut owner_rx) = wide_ctx(1);
15698            hello_via_sink(
15699                &handler,
15700                &owner,
15701                &mut owner_rx,
15702                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15703            )
15704            .await;
15705            let mut modules = BTreeMap::new();
15706            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15707                let (ctx, mut rx) = wide_ctx(connection);
15708                let hello = hello_with_capabilities(module_id, connection, provides);
15709                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15710                modules.insert(module_id.to_string(), (ctx, rx));
15711            }
15712            Rig {
15713                handler,
15714                forwarding,
15715                owner,
15716                _owner_rx: owner_rx,
15717                modules,
15718                generation: 0,
15719                next_connection: 100,
15720                _supervisor: supervisor,
15721            }
15722        }
15723
15724        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15725            ScopeCarrier::new(Principal::Reserved {
15726                module_id: module_id.to_string(),
15727            })
15728            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15729        }
15730
15731        /// The scope most tests open under: aft carries to any module, broca
15732        /// only to plexus and other, and the owner delegates as agent-1.
15733        fn session(scope_epoch: u64) -> ScopeRecord {
15734            let mut record = head("s", scope_epoch);
15735            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15736            record.attributes.agent_id = Some("agent-1".to_string());
15737            record.attributes.delegates = true;
15738            record
15739        }
15740
15741        impl Rig {
15742            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15743                self.generation += 1;
15744                sync(&self.handler, &self.owner, self.generation, scopes)
15745                    .await
15746                    .expect("the owner's sync is accepted");
15747            }
15748
15749            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15750                ScopeSelector::new(
15751                    Principal::Reserved {
15752                        module_id: OWNER.to_string(),
15753                    },
15754                    scope_ref.to_string(),
15755                    scope_epoch,
15756                )
15757            }
15758
15759            fn open_frame(
15760                &mut self,
15761                opener: Option<&str>,
15762                target: &str,
15763                scope: Option<ScopeSelector>,
15764            ) -> (
15765                RouteCtx,
15766                mpsc::Receiver<crate::router::OutboundFrame>,
15767                Frame,
15768            ) {
15769                self.next_connection += 1;
15770                let (ctx, rx) = wide_ctx(self.next_connection);
15771                let root = unique_project_root("scoped-open");
15772                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15773                    target: RouteTarget::ToolProvider {
15774                        module_id: target.to_string(),
15775                    },
15776                    identity: BindIdentity::new(
15777                        root.path().to_path_buf(),
15778                        "unit".to_string(),
15779                        "session".to_string(),
15780                    ),
15781                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
15782                        module_id: module_id.to_string(),
15783                        launch_nonce: nonce(module_id),
15784                    }),
15785                    consumer_capabilities: None,
15786                    role_versions: None,
15787                    admission_facts: None,
15788                    scope,
15789                })
15790                .unwrap();
15791                let frame = Frame::build(
15792                    FrameType::Request,
15793                    control_flags(),
15794                    0,
15795                    0,
15796                    self.next_connection,
15797                    body,
15798                )
15799                .unwrap();
15800                (ctx, rx, frame)
15801            }
15802
15803            /// Open and expect a refusal before anything is relayed.
15804            async fn refused(
15805                &mut self,
15806                opener: Option<&str>,
15807                target: &str,
15808                scope: Option<ScopeSelector>,
15809            ) -> String {
15810                self.refusal_body(opener, target, scope).await["code"]
15811                    .as_str()
15812                    .unwrap()
15813                    .to_string()
15814            }
15815
15816            async fn refusal_body(
15817                &mut self,
15818                opener: Option<&str>,
15819                target: &str,
15820                scope: Option<ScopeSelector>,
15821            ) -> Value {
15822                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15823                let replies = tokio::time::timeout(
15824                    Duration::from_secs(2),
15825                    self.handler.handle_control_frame(&ctx, frame),
15826                )
15827                .await
15828                .expect("the open must be refused before waiting for a bind ack")
15829                .unwrap();
15830                assert_eq!(replies.len(), 1, "{replies:?}");
15831                assert_eq!(replies[0].header.ty, FrameType::Error);
15832                let (_, module_rx) = self.modules.get_mut(target).unwrap();
15833                assert!(
15834                    module_rx.try_recv().is_err(),
15835                    "a refused open relays nothing"
15836                );
15837                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15838                parse_error(&replies[0])
15839            }
15840
15841            /// Start an open and return its task and the bind the target got.
15842            async fn relayed(
15843                &mut self,
15844                opener: Option<&str>,
15845                target: &str,
15846                scope: Option<ScopeSelector>,
15847            ) -> Relayed {
15848                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15849                let handler = self.handler.clone();
15850                let task_ctx = ctx.clone();
15851                let task = tokio::spawn(async move {
15852                    handler
15853                        .handle_control_frame(&task_ctx, frame)
15854                        .await
15855                        .unwrap()
15856                });
15857                let (_, module_rx) = self.modules.get_mut(target).unwrap();
15858                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15859                    .await
15860                    .expect("the target receives the relayed route.bind")
15861                    .unwrap()
15862                    .frame;
15863                Relayed {
15864                    target: target.to_string(),
15865                    client: ctx,
15866                    client_rx: rx,
15867                    task,
15868                    bind,
15869                }
15870            }
15871
15872            async fn ack(&self, relayed: &Relayed) {
15873                let (module, _) = &self.modules[&relayed.target];
15874                self.handler
15875                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15876                    .await
15877                    .unwrap();
15878            }
15879
15880            /// Open, ack and return the bound route.
15881            async fn bound(
15882                &mut self,
15883                opener: Option<&str>,
15884                target: &str,
15885                scope: Option<ScopeSelector>,
15886            ) -> Bound {
15887                let relayed = self.relayed(opener, target, scope).await;
15888                self.ack(&relayed).await;
15889                let Relayed {
15890                    target,
15891                    client,
15892                    mut client_rx,
15893                    task,
15894                    bind,
15895                } = relayed;
15896                assert!(
15897                    task.await.unwrap().is_empty(),
15898                    "the open is answered by commit"
15899                );
15900                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15901                Bound {
15902                    target,
15903                    client,
15904                    client_rx,
15905                    channel,
15906                    epoch,
15907                    bind,
15908                }
15909            }
15910
15911            fn live(&self, route: &Bound) -> bool {
15912                matches!(
15913                    self.forwarding
15914                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15915                        .unwrap(),
15916                    DataRoute::Client(DataRouteState::Bound(_))
15917                )
15918            }
15919        }
15920
15921        struct Relayed {
15922            target: String,
15923            client: RouteCtx,
15924            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15925            task: tokio::task::JoinHandle<Vec<Frame>>,
15926            bind: Frame,
15927        }
15928
15929        struct Bound {
15930            target: String,
15931            client: RouteCtx,
15932            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15933            channel: u16,
15934            epoch: u32,
15935            bind: Frame,
15936        }
15937
15938        impl Bound {
15939            /// The reason of the `route.closed` this client was sent, after
15940            /// checking it also got a GOODBYE on exactly this route.
15941            fn closed_reason(&mut self) -> RouteCloseReason {
15942                let mut reason = None;
15943                let mut goodbye = false;
15944                while let Ok(outbound) = self.client_rx.try_recv() {
15945                    let frame = outbound.frame;
15946                    match frame.header.ty {
15947                        FrameType::Goodbye => {
15948                            assert_eq!(
15949                                (frame.header.channel, frame.header.epoch),
15950                                (self.channel, self.epoch)
15951                            );
15952                            goodbye = true;
15953                        }
15954                        FrameType::Push => {
15955                            let ClientControlPush::RouteClosed {
15956                                reason: r,
15957                                module_id,
15958                                ..
15959                            } = serde_json::from_slice(&frame.body).unwrap()
15960                            else {
15961                                panic!("unexpected push");
15962                            };
15963                            assert_eq!(module_id, self.target);
15964                            reason = Some(r);
15965                        }
15966                        other => panic!("unexpected frame {other:?}"),
15967                    }
15968                }
15969                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15970                reason.expect("the client is told why the route closed")
15971            }
15972
15973            fn untouched(&mut self) -> bool {
15974                self.client_rx.try_recv().is_err()
15975            }
15976
15977            fn stamp(&self) -> Option<ScopeStamp> {
15978                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15979                    ModuleControlRequest::RouteBind { scope, .. } => scope,
15980                    other => panic!("expected a route.bind, got {other:?}"),
15981                }
15982            }
15983        }
15984
15985        #[tokio::test]
15986        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15987        ) {
15988            let mut rig = rig().await;
15989            let mut record = session(1);
15990            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15991            record.child_owners = vec![Principal::Reserved {
15992                module_id: MAGIC.to_string(),
15993            }];
15994            rig.sync(vec![record]).await;
15995            let scope = || Some(rig_selector("s", Some(1)));
15996
15997            // Admitted: the owner, a bare carrier to any module, a targeted
15998            // carrier to its listed module.
15999            rig.bound(Some(OWNER), PLEXUS, scope()).await;
16000            rig.bound(Some(AFT), OTHER, scope()).await;
16001            rig.bound(Some(BROCA), PLEXUS, scope()).await;
16002
16003            // Refused scope_not_carrier: a targeted carrier to an unlisted
16004            // module, a module that is not listed at all (a child owner is not
16005            // a carrier), and a direct key-holder.
16006            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16007                assert_eq!(
16008                    rig.refused(opener, target, scope()).await,
16009                    error_codes::SCOPE_NOT_CARRIER,
16010                    "{opener:?} -> {target}"
16011                );
16012            }
16013        }
16014
16015        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16016            ScopeSelector::new(
16017                Principal::Reserved {
16018                    module_id: OWNER.to_string(),
16019                },
16020                scope_ref.to_string(),
16021                scope_epoch,
16022            )
16023        }
16024
16025        #[tokio::test]
16026        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16027            let mut rig = rig().await;
16028            rig.sync(vec![session(1)]).await;
16029            for opener in [OWNER, AFT] {
16030                assert_eq!(
16031                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16032                        .await,
16033                    error_codes::SCOPE_EPOCH_REQUIRED,
16034                    "{opener}"
16035                );
16036            }
16037        }
16038
16039        #[tokio::test]
16040        async fn admission_separates_not_synced_not_live_and_ended() {
16041            let mut rig = rig().await;
16042            // Before the configured owner's first sync: retryable.
16043            let code = rig
16044                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16045                .await;
16046            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16047            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16048
16049            // An owner that is not configured will never sync: terminal.
16050            let ghost = ScopeSelector::new(
16051                Principal::Reserved {
16052                    module_id: "ghost".to_string(),
16053                },
16054                "s".to_string(),
16055                Some(1),
16056            );
16057            assert_eq!(
16058                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16059                error_codes::SCOPE_NOT_LIVE
16060            );
16061
16062            rig.sync(vec![session(2)]).await;
16063            assert_eq!(
16064                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16065                    .await,
16066                error_codes::SCOPE_NOT_LIVE
16067            );
16068            for epoch in [1, 3] {
16069                assert_eq!(
16070                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16071                        .await,
16072                    error_codes::SCOPE_ENDED,
16073                    "epoch {epoch}"
16074                );
16075            }
16076            // Control: the live epoch is admitted.
16077            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16078                .await;
16079        }
16080
16081        #[tokio::test]
16082        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16083            let mut rig = rig().await;
16084            rig.sync(vec![session(1)]).await;
16085            let route = rig
16086                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16087                .await;
16088            let stamp = route.stamp().expect("a scoped bind carries the stamp");
16089            assert_eq!(stamp.scope_ref, "s");
16090            assert_eq!(stamp.scope_epoch, 1);
16091            assert_eq!(stamp.kind, ScopeKind::Head);
16092            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16093            assert!(stamp.attributes.delegates);
16094            assert!(stamp.owner_authorized);
16095            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16096            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16097
16098            // broca owns a scope of its own on its own module connection; it is
16099            // not in scope_authority_owners, so its stamp is not authorized.
16100            let (broca, mut broca_rx) = wide_ctx(50);
16101            hello_via_sink(
16102                &rig.handler,
16103                &broca,
16104                &mut broca_rx,
16105                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16106            )
16107            .await;
16108            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16109                .await
16110                .unwrap();
16111            let own = ScopeSelector::new(
16112                Principal::Reserved {
16113                    module_id: BROCA.to_string(),
16114                },
16115                "b".to_string(),
16116                Some(1),
16117            );
16118            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16119            assert!(!route.stamp().unwrap().owner_authorized);
16120        }
16121
16122        #[tokio::test]
16123        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16124            let mut rig = rig().await;
16125            let mut record = head("s", 1);
16126            record.carriers = vec![carrier(AFT, None)];
16127            let flow_id = "Flow:run-7/step_2!~";
16128            record.attributes.flow_id = Some(flow_id.to_string());
16129            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16130                .await
16131                .unwrap();
16132            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16133                panic!("not a sync reply");
16134            };
16135            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16136            let route = rig
16137                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16138                .await;
16139            assert!(rig.live(&route), "the stamped bind committed");
16140            let stamp = route.stamp().expect("a flow scope carries a stamp");
16141            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16142            assert_eq!(stamp.attributes.agent_id, None);
16143            assert!(!stamp.attributes.delegates);
16144            assert!(stamp.owner_authorized);
16145            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16146            assert_eq!(unscoped.stamp(), None);
16147        }
16148
16149        #[tokio::test]
16150        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16151            let mut rig = rig_with_flow_support(false).await;
16152            let mut record = session(1);
16153            record.attributes.flow_id = Some("flow:7".to_string());
16154            rig.sync(vec![record]).await;
16155            for opener in [OWNER, AFT] {
16156                let body = rig
16157                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16158                    .await;
16159                assert_eq!(body["code"], "target_flow_unsupported");
16160                let message = body["message"].as_str().unwrap();
16161                for required in [PLEXUS, "flow-scopes/v1"] {
16162                    assert!(message.contains(required), "{message}");
16163                }
16164            }
16165        }
16166
16167        #[tokio::test]
16168        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16169            let mut rig = rig_with_flow_support(true).await;
16170            let mut record = session(1);
16171            record.attributes.flow_id = Some("flow:7".to_string());
16172            rig.sync(vec![record]).await;
16173            for opener in [OWNER, AFT] {
16174                let route = rig
16175                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16176                    .await;
16177                assert!(rig.live(&route));
16178                assert_eq!(
16179                    route.stamp().unwrap().attributes.flow_id.as_deref(),
16180                    Some("flow:7")
16181                );
16182            }
16183        }
16184
16185        #[tokio::test]
16186        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16187            use std::future::Future;
16188
16189            let mut rig = rig_with_flow_support(true).await;
16190            let mut record = session(1);
16191            record.attributes.flow_id = Some("flow:7".to_string());
16192            rig.sync(vec![record]).await;
16193            let (client, mut client_rx, frame) =
16194                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16195            // Hold the route-open response permit so admission sees the first
16196            // target but relay reservation cannot capture an endpoint yet.
16197            for _ in 0..64 {
16198                client.egress.try_send(route_bind_ack(1)).unwrap();
16199            }
16200            let handler = rig.handler.clone();
16201            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16202            std::future::poll_fn(|cx| {
16203                assert!(open.as_mut().poll(cx).is_pending());
16204                std::task::Poll::Ready(())
16205            })
16206            .await;
16207            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16208
16209            let old_connection = rig.modules[PLEXUS].0.connection_id;
16210            rig.handler.cleanup_connection(old_connection).unwrap();
16211            let (replacement, mut replacement_rx) = wide_ctx(200);
16212            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16213            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16214            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16215            let hello = Frame::build(
16216                FrameType::Hello,
16217                control_flags(),
16218                0,
16219                0,
16220                200,
16221                serde_json::to_vec(&body).unwrap(),
16222            )
16223            .unwrap();
16224            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16225
16226            client_rx.try_recv().unwrap();
16227            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16228                .await
16229                .expect("the replacement is refused without waiting for a bind ack")
16230                .unwrap();
16231            assert_eq!(replies.len(), 1);
16232            let body = parse_error(&replies[0]);
16233            assert_eq!(body["code"], "target_flow_unsupported");
16234            for required in [PLEXUS, "flow-scopes/v1"] {
16235                assert!(body["message"].as_str().unwrap().contains(required));
16236            }
16237            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16238            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16239            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16240            assert!(rig
16241                .handler
16242                .registry
16243                .get_module_by_connection(replacement.connection_id)
16244                .unwrap()
16245                .is_some());
16246        }
16247
16248        #[tokio::test]
16249        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16250            use std::future::Future;
16251            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16252            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16253            const NEXT_HASH: &str =
16254                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16255            let mut rig = rig().await;
16256            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16257            record.carriers.push(ScopeCarrier::new(Principal::Direct));
16258            rig.sync(vec![record.clone()]).await;
16259            let (client, mut rx, frame) = rig.open_frame(
16260                None,
16261                PLEXUS,
16262                Some(
16263                    rig.selector("s", Some(1))
16264                        .with_bind_proof(Some(PROOF.into())),
16265                ),
16266            );
16267            // Hold egress reservation while the scope owner changes its bind-proof
16268            // hash, so a proof valid at admission must be checked again at relay.
16269            for _ in 0..64 {
16270                client.egress.try_send(route_bind_ack(1)).unwrap();
16271            }
16272            let handler = rig.handler.clone();
16273            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16274            std::future::poll_fn(|cx| {
16275                assert!(open.as_mut().poll(cx).is_pending());
16276                std::task::Poll::Ready(())
16277            })
16278            .await;
16279            record.bind_proof_sha256 = Some(NEXT_HASH.into());
16280            rig.sync(vec![record]).await;
16281            rx.try_recv().unwrap();
16282            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16283                .await
16284                .unwrap()
16285                .unwrap();
16286            assert_eq!(
16287                parse_error(&replies[0])["code"],
16288                error_codes::SCOPE_BIND_PROOF_MISMATCH
16289            );
16290            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16291            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16292            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16293        }
16294
16295        #[tokio::test]
16296        async fn scope_bind_proof_reserved_openers_ignore_proof() {
16297            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16298            let mut rig = rig().await;
16299            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16300                .await;
16301            for opener in [OWNER, AFT] {
16302                for proof in [None, Some("malformed+proof=")] {
16303                    let route = rig
16304                        .bound(
16305                            Some(opener),
16306                            PLEXUS,
16307                            Some(
16308                                rig.selector("s", Some(1))
16309                                    .with_bind_proof(proof.map(str::to_string)),
16310                            ),
16311                        )
16312                        .await;
16313                    assert!(rig.live(&route));
16314                }
16315            }
16316        }
16317
16318        #[tokio::test]
16319        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16320            use std::future::Future;
16321            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16322            const NEXT_HASH: &str =
16323                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16324            let mut rig = rig().await;
16325            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16326                .await;
16327            let (client, mut rx, frame) =
16328                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16329            for _ in 0..64 {
16330                client.egress.try_send(route_bind_ack(1)).unwrap();
16331            }
16332            let handler = rig.handler.clone();
16333            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16334            std::future::poll_fn(|cx| {
16335                assert!(open.as_mut().poll(cx).is_pending());
16336                std::task::Poll::Ready(())
16337            })
16338            .await;
16339            rig.sync(vec![
16340                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16341            ])
16342            .await;
16343            rx.try_recv().unwrap();
16344            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16345            let bind = std::future::poll_fn(|cx| {
16346                assert!(open.as_mut().poll(cx).is_pending());
16347                module_rx.poll_recv(cx)
16348            })
16349            .await
16350            .unwrap()
16351            .frame;
16352            handler
16353                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16354                .await
16355                .unwrap();
16356            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16357                .await
16358                .unwrap()
16359                .unwrap();
16360            assert_eq!(replies.len(), 1);
16361            // Proof rechecking allowed the relay. Independent tag settlement
16362            // still asks the opener to retry after any same-epoch content change.
16363            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16364            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16365            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16366            let route = rig
16367                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16368                .await;
16369            assert!(rig.live(&route));
16370        }
16371
16372        #[tokio::test]
16373        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16374        {
16375            let mut rig = rig_with_flow_support(false).await;
16376            rig.sync(vec![session(1)]).await;
16377            let route = rig
16378                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16379                .await;
16380            assert!(rig.live(&route));
16381            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16382            let mut record = session(1);
16383            record.attributes.flow_id = Some("flow:7".to_string());
16384            rig.sync(vec![record]).await;
16385            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16386            assert!(rig.live(&unscoped));
16387            assert_eq!(unscoped.stamp(), None);
16388        }
16389
16390        #[tokio::test]
16391        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16392            let mut rig = rig().await;
16393            let mut record = session(1);
16394            record.attributes.flow_id = Some("flow:7".to_string());
16395            rig.sync(vec![record.clone()]).await;
16396            let before = rig.forwarding.published_scope_tag(OWNER, "s");
16397            let mut owner_route = rig
16398                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16399                .await;
16400            let mut carrier_route = rig
16401                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16402                .await;
16403            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16404            rig.sync(vec![record.clone()]).await;
16405            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16406            assert!(rig.live(&owner_route) && owner_route.untouched());
16407            assert!(rig.live(&carrier_route) && carrier_route.untouched());
16408
16409            record.attributes.flow_id = Some("flow:8".to_string());
16410            rig.sync(vec![record]).await;
16411            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16412            let before = before.unwrap();
16413            assert_eq!(after.scope_epoch, before.scope_epoch);
16414            assert!(after.version > before.version);
16415            for route in [&mut owner_route, &mut carrier_route] {
16416                assert!(!rig.live(route));
16417                assert_eq!(
16418                    route.closed_reason(),
16419                    RouteCloseReason::ScopeDelegationChanged
16420                );
16421            }
16422            assert!(rig.live(&unscoped) && unscoped.untouched());
16423            // Each provider also receives a GOODBYE for its drained route;
16424            // consume it before expecting the next route.bind on that sink.
16425            for target in [PLEXUS, OTHER] {
16426                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16427                let goodbye = module_rx
16428                    .try_recv()
16429                    .expect("the provider sees the drain")
16430                    .frame;
16431                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16432                assert!(module_rx.try_recv().is_err());
16433            }
16434            let rebound = rig
16435                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16436                .await;
16437            assert_eq!(
16438                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16439                Some("flow:8")
16440            );
16441        }
16442
16443        /// The owner's sync lands between admission and the module's ack. The
16444        /// open is refused by name, the module's other routes stay up, and the
16445        /// reserved pair is released. Changed content is retryable; an ended
16446        /// scope is not.
16447        #[tokio::test]
16448        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16449            let mut rig = rig().await;
16450            rig.sync(vec![session(1)]).await;
16451            let mut cotenant = rig
16452                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16453                .await;
16454
16455            let mut changed = session(1);
16456            changed.child_owners.push(Principal::Reserved {
16457                module_id: MAGIC.to_string(),
16458            });
16459            let mut ended = None;
16460            for (code, next) in [
16461                (error_codes::SCOPE_CHANGED, vec![changed]),
16462                (error_codes::SCOPE_ENDED, Vec::new()),
16463            ] {
16464                let relayed = rig
16465                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16466                    .await;
16467                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16468                ended = Some(next.is_empty());
16469                rig.sync(next).await;
16470                rig.ack(&relayed).await;
16471                let replies = relayed.task.await.unwrap();
16472                assert_eq!(replies.len(), 1, "{replies:?}");
16473                assert_eq!(parse_error(&replies[0])["code"], code);
16474                assert_eq!(
16475                    subc_protocol::error_codes::is_retryable_route_open(code),
16476                    code == error_codes::SCOPE_CHANGED
16477                );
16478                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16479                // The module is told to drop just the binding it created.
16480                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16481                // Collected, because ending the scope also closes the co-tenant
16482                // route, whose GOODBYE comes first.
16483                let mut goodbyes = Vec::new();
16484                while let Ok(outbound) = plexus_rx.try_recv() {
16485                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16486                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16487                }
16488                assert!(
16489                    goodbyes.contains(&(bind_channel, bind_epoch)),
16490                    "{goodbyes:?}"
16491                );
16492                assert!(rig
16493                    .handler
16494                    .registry
16495                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16496                    .unwrap()
16497                    .is_some());
16498            }
16499            assert_eq!(ended, Some(true));
16500            // The co-tenant stayed up through the change, and closed only when
16501            // the scope ended, by the drain rule rather than by the commit.
16502            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16503        }
16504
16505        /// Each row of the drain table on one set of routes: the owner's, a
16506        /// bare carrier's, and a targeted carrier's to each of its targets.
16507        #[tokio::test]
16508        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16509            struct Case {
16510                name: &'static str,
16511                change: fn(&mut ScopeRecord),
16512                /// Closed routes by index: owner->plexus, aft->plexus,
16513                /// broca->plexus, broca->other.
16514                closed: [Option<RouteCloseReason>; 4],
16515            }
16516            use RouteCloseReason::*;
16517            let cases = [
16518                Case {
16519                    name: "a carrier entry removed",
16520                    change: |r| {
16521                        r.carriers.retain(|c| {
16522                            c.principal
16523                                != Principal::Reserved {
16524                                    module_id: AFT.to_string(),
16525                                }
16526                        })
16527                    },
16528                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16529                },
16530                Case {
16531                    name: "a target removed from a carrier",
16532                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16533                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
16534                },
16535                Case {
16536                    name: "a bare carrier narrowed to targets",
16537                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16538                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16539                },
16540                Case {
16541                    name: "delegates turned off",
16542                    change: |r| r.attributes.delegates = false,
16543                    closed: [Some(ScopeDelegationChanged); 4],
16544                },
16545                Case {
16546                    name: "agent_id changed",
16547                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16548                    closed: [Some(ScopeDelegationChanged); 4],
16549                },
16550                Case {
16551                    name: "a carrier added, child owners changed, the record re-sent",
16552                    change: |r| {
16553                        r.carriers.push(carrier(MAGIC, None));
16554                        r.child_owners.push(Principal::Reserved {
16555                            module_id: MAGIC.to_string(),
16556                        });
16557                    },
16558                    closed: [None; 4],
16559                },
16560                Case {
16561                    name: "a target added",
16562                    change: |r| {
16563                        r.carriers[1]
16564                            .targets
16565                            .as_mut()
16566                            .unwrap()
16567                            .push("third".to_string())
16568                    },
16569                    closed: [None; 4],
16570                },
16571                Case {
16572                    name: "delegates turned on",
16573                    change: |r| r.attributes.delegates = true,
16574                    closed: [None; 4],
16575                },
16576            ];
16577            for case in cases {
16578                let mut rig = rig().await;
16579                rig.sync(vec![session(1)]).await;
16580                let scope = || Some(rig_selector("s", Some(1)));
16581                let mut routes = [
16582                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
16583                    rig.bound(Some(AFT), PLEXUS, scope()).await,
16584                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
16585                    rig.bound(Some(BROCA), OTHER, scope()).await,
16586                ];
16587                let mut record = session(1);
16588                (case.change)(&mut record);
16589                rig.sync(vec![record]).await;
16590                for (index, expected) in case.closed.iter().enumerate() {
16591                    let route = &mut routes[index];
16592                    match expected {
16593                        Some(reason) => {
16594                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
16595                            assert_eq!(
16596                                route.closed_reason(),
16597                                *reason,
16598                                "{}: route {index}",
16599                                case.name
16600                            );
16601                        }
16602                        None => {
16603                            assert!(rig.live(route), "{}: route {index} closed", case.name);
16604                            assert!(
16605                                route.untouched(),
16606                                "{}: route {index} was told something",
16607                                case.name
16608                            );
16609                        }
16610                    }
16611                }
16612            }
16613        }
16614
16615        #[tokio::test]
16616        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16617            // Removed, and replaced by a higher epoch.
16618            for next in [Vec::new(), vec![session(2)]] {
16619                let mut rig = rig().await;
16620                rig.sync(vec![session(1)]).await;
16621                let mut route = rig
16622                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16623                    .await;
16624                rig.sync(next).await;
16625                assert!(!rig.live(&route));
16626                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16627            }
16628
16629            // A child whose parent ends: its routes close as parent-ended, the
16630            // child stays live, and routes under the parent close as ended.
16631            let mut rig = rig().await;
16632            let mut child = session(1);
16633            child.scope_ref = "child".to_string();
16634            child.kind = ScopeKind::Worker;
16635            child.parent = Some(ScopeParent::new(
16636                Principal::Reserved {
16637                    module_id: OWNER.to_string(),
16638                },
16639                "s".to_string(),
16640                1,
16641            ));
16642            rig.sync(vec![session(1), child.clone()]).await;
16643            let mut child_route = rig
16644                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16645                .await;
16646            assert_eq!(
16647                child_route.stamp().unwrap().parent_state,
16648                Some(ParentState::Linked)
16649            );
16650            rig.sync(vec![child]).await;
16651            assert!(!rig.live(&child_route));
16652            assert_eq!(
16653                child_route.closed_reason(),
16654                RouteCloseReason::ScopeParentEnded
16655            );
16656        }
16657
16658        #[tokio::test]
16659        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16660        ) {
16661            let mut rig = rig().await;
16662            rig.sync(vec![session(1)]).await;
16663            let mut route = rig
16664                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16665                .await;
16666            let before = rig.forwarding.published_scope_tag(OWNER, "s");
16667            rig.sync(vec![session(1)]).await;
16668            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16669            assert!(rig.live(&route) && route.untouched());
16670
16671            // A call in flight on the route when another carrier is added. A
16672            // forwarded REQUEST holds one credit on the route's flow until the
16673            // module answers; the router takes it exactly like this.
16674            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16675                .forwarding
16676                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16677                .unwrap()
16678            else {
16679                panic!("the route is bound");
16680            };
16681            binding.flow.acquire_tagged(9, false).await.unwrap();
16682            let mut widened = session(1);
16683            widened.carriers.push(carrier(MAGIC, None));
16684            rig.sync(vec![widened]).await;
16685            assert!(rig.live(&route) && route.untouched());
16686            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16687            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16688            // The call's credit is still held on an open flow, so its answer
16689            // will be delivered: closing the route would have closed the flow.
16690            assert_eq!(binding.flow.in_flight(), 1);
16691            binding
16692                .flow
16693                .acquire_tagged(10, false)
16694                .await
16695                .expect("the flow is still open");
16696        }
16697
16698        /// A swap's superseded endpoint keeps its routes until drained; ending
16699        /// the scope closes them there too.
16700        #[tokio::test]
16701        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16702            let mut rig = rig().await;
16703            rig.sync(vec![session(1)]).await;
16704            let mut on_incumbent = rig
16705                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16706                .await;
16707
16708            // Swap plexus: register a candidate and cut over, leaving the
16709            // incumbent superseded with the route still on it.
16710            let (candidate, _candidate_rx) = wide_ctx(9);
16711            let registration = rig
16712                .handler
16713                .registry
16714                .register_candidate_with_control_ops(
16715                    manifest(PLEXUS, PROTOCOL_VERSION),
16716                    PROTOCOL_VERSION,
16717                    candidate.connection_id,
16718                    module_baseline_control_ops(),
16719                )
16720                .unwrap();
16721            rig.forwarding
16722                .register_candidate_module_connection(
16723                    candidate.connection_id,
16724                    PLEXUS.to_string(),
16725                    PROTOCOL_VERSION,
16726                    manifest_concurrency(&registration.manifest),
16727                    candidate.egress.clone(),
16728                )
16729                .unwrap();
16730            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16731            rig.handler
16732                .registry
16733                .promote_candidate(PLEXUS)
16734                .unwrap()
16735                .unwrap();
16736            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16737
16738            rig.sync(Vec::new()).await;
16739            assert!(!rig.live(&on_incumbent));
16740            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16741            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16742            let goodbye = incumbent_rx
16743                .try_recv()
16744                .expect("the superseded endpoint is told")
16745                .frame;
16746            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16747        }
16748    }
16749}
16750
16751#[cfg(test)]
16752mod concurrency_default_exposure_tests {
16753    use super::*;
16754
16755    fn hello_body(role_json: &str) -> Vec<u8> {
16756        format!(
16757            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16758        )
16759        .into_bytes()
16760    }
16761
16762    fn manifest_from(body: &[u8]) -> ModuleManifest {
16763        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16764        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16765            .expect("manifest parses")
16766    }
16767
16768    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16769
16770    #[test]
16771    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16772        let body = hello_body(&format!(
16773            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16774        ));
16775        let manifest = manifest_from(&body);
16776        // Precondition: serde really resolved it to the default, so the typed
16777        // manifest alone cannot answer the question this probe exists for.
16778        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16779        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16780    }
16781
16782    #[test]
16783    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16784        let body = hello_body(&format!(
16785            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16786        ));
16787        let manifest = manifest_from(&body);
16788        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16789        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16790    }
16791
16792    #[test]
16793    fn non_management_roles_are_never_reported() {
16794        let body = hello_body(
16795            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16796        );
16797        let manifest = manifest_from(&body);
16798        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16799    }
16800}