Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
26        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
27        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
28    },
29    scope::{
30        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32        SCOPE_SYNC_OP,
33    },
34    session::{
35        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38        ROLE_VERSIONS_FIELD,
39    },
40    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41    Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47    caller_identity::CallerIdentity,
48    capability_requirements::{
49        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51        RuntimeModule,
52    },
53    child_roster::DaemonShutdownFlag,
54    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55    forwarding::{
56        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59    },
60    observability::{
61        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62    },
63    privileged_prompt::{prompt_text, PrivilegedOperation},
64    provenance::{
65        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66    },
67    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68    router::{RouteCtx, RouterError},
69    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71    stderr_tail::{CaptureState, TailEntry},
72    supervise::{
73        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74        SupervisorHandle, SwapHelloAdmission,
75    },
76    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79/// Lowest envelope version this subc build will negotiate.
80///
81/// Module HELLO negotiation is exact: peers must use the daemon's locked
82/// protocol version. Older and newer peers receive `version_unsupported` and
83/// are not registered.
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93    ops::ACTIVITY_RECORD,
94    ops::ACTIVITY_QUERY,
95    ops::ACTIVITY_SUBSCRIBE,
96    ops::SERVER_DESCRIBE,
97    ops::CATALOG_LIST,
98    ops::ROUTE_OPEN,
99    ops::ROUTE_POLL,
100    ops::ROUTE_CLOSING,
101    ops::ROUTE_CLOSED,
102    ops::SUPERVISOR_LIST,
103    ops::SUPERVISOR_RESTART,
104    ops::SUPERVISOR_SWAP,
105    ops::SUPERVISOR_RELOAD,
106    ops::SUPERVISOR_RESCAN,
107    ops::SUPERVISOR_RELEASE_RESERVED,
108    ops::SUPERVISOR_SET_ENABLED,
109    ops::SUPERVISOR_HEALTH_PROBE,
110    ops::SUPERVISOR_HEALTH,
111    ops::SUPERVISOR_STDERR_TAIL,
112    ops::SUPERVISOR_TERMINALS,
113    ops::SUPERVISOR_ROUTES,
114    ops::SUPERVISOR_PROVENANCE,
115    ops::SUPERVISOR_SPAWN_SNAPSHOT,
116    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
120/// announced by a capability in `server.describe` instead, and clients check
121/// that capability before sending it. `ck daemon --json` prints `subc_ops`
122/// verbatim and scripts read that output, so a new name there would change it.
123/// A malformed body for one of these is still refused `invalid_control_body`
124/// rather than as an unknown op.
125const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132    "supervisor.live_roots",
133    SCOPE_SYNC_OP,
134    SCOPE_APPLY_OP,
135    SCOPE_DESCRIBE_OP,
136    "operator.confirm",
137];
138
139/// Module-originated ops the daemon answers but does not advertise in
140/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
141/// belongs to is incomplete, so no module is told it works before it does.
142const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146/// How long subc waits for a module to ack a relayed route.bind before returning
147/// `module_timeout`. The ack waits on the module's own configure, which for AFT
148/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
149/// and DB-open work — on a cold page cache or a large repo that legitimately
150/// exceeds a couple of seconds. The default is generous because rejecting a VALID
151/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
152/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
153pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155/// How many CONSECUTIVE full-budget relay timeouts against one target module
156/// open that module's bind-relay breaker.
157///
158/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
159/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
160/// so three consecutive full-budget timeouts take ~36s to observe: every client
161/// whose open contributed to opening the breaker had already given up on its
162/// own. That is what makes opening the breaker unable to turn a call that would
163/// have succeeded into a refusal — it can only make an already-failing module
164/// fail faster.
165///
166/// Two would be reachable inside one default deadline. One would convict a
167/// module on a single cold-cache bind, which is exactly the valid-but-slow case
168/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
169pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171/// How long a module's bind-relay breaker stays open before exactly one
172/// `route.open` is let through as a probe.
173///
174/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
175/// re-pays a full-budget stall almost continuously, and the breaker stops being
176/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
177/// deadline: a client that starts retrying after the module recovers has to get
178/// a probe opportunity inside its own deadline, or the breaker converts a
179/// recovered module into a failed call — the failure it exists to prevent,
180/// pointed the other way.
181///
182/// 20s sits between those with room on both sides, and it caps what a wedged
183/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
184/// rather than one per `route.open` per connection. The stall that motivated
185/// this, with its measurements, is written up in
186/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
187/// waited the whole budget out.
188pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194    configured: &Path,
195    spawned_from: Option<&Path>,
196    image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198    let path = match spawned_from {
199        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200            ReloadPathAgreement::Match
201        }
202        Some(spawned_from) => ReloadPathAgreement::Mismatch {
203            configured: configured.to_path_buf(),
204            spawned_from: spawned_from.to_path_buf(),
205        },
206        None => ReloadPathAgreement::Unavailable {
207            reason: if matches!(
208                image,
209                subc_control::RunningImageAgreement::Unavailable {
210                    reason: subc_control::RunningImageUnavailableReason::NotRunning
211                }
212            ) {
213                ReloadPathUnavailableReason::NotRunning
214            } else {
215                ReloadPathUnavailableReason::SpawnedPathUnavailable
216            },
217        },
218    };
219    PendingReloadVerdict {
220        path,
221        image,
222        spec: None,
223    }
224}
225
226fn launch_spec_agreement(
227    configured: &crate::supervise::ModuleSpec,
228    running: bool,
229    spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232    if !running {
233        return Agreement::Unavailable {
234            reason: Reason::NotRunning,
235        };
236    }
237    match spawned {
238        None => Agreement::Unavailable {
239            reason: Reason::SpawnedSpecUnavailable,
240        },
241        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242            Agreement::Match
243        }
244        Some(_) => Agreement::Mismatch,
245    }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249    if configured == spawned_from {
250        return true;
251    }
252    #[cfg(windows)]
253    {
254        // Windows spawn records the resolved verbatim path (\\?\...), while
255        // configuration keeps its original spelling. Resolve both with the OS
256        // instead of stripping prefixes or folding case, which could conflate
257        // distinct names in case-sensitive directories. Different resolved
258        // paths still differ even if their files have identical content.
259        match (
260            std::fs::canonicalize(configured),
261            std::fs::canonicalize(spawned_from),
262        ) {
263            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264            _ => false,
265        }
266    }
267    #[cfg(not(windows))]
268    {
269        false
270    }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275    build: DaemonBuildProvenance,
276    pid: Option<u32>,
277    started_at_ms: Option<u64>,
278    start_clock: Option<crate::clock::StartClock>,
279    executable_path: Option<PathBuf>,
280    executable_identity: Option<SpawnedFileIdentity>,
281    process_start_time: Option<u64>,
282    probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286    fn default() -> Self {
287        Self {
288            build: DaemonBuildProvenance {
289                build_git_sha: None,
290                build_lock_digest: None,
291            },
292            pid: None,
293            started_at_ms: None,
294            start_clock: None,
295            executable_path: None,
296            executable_identity: None,
297            process_start_time: None,
298            probe: ExecutableIdentityProbe::default(),
299        }
300    }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305    supervisor: Supervisor,
306    config_path: PathBuf,
307    configured_port: Option<u16>,
308    storage_config: Option<crate::daemon_config::StorageConfig>,
309    admission_facts_carrier_module_id: Option<String>,
310    admission_facts_targets: Option<Vec<String>>,
311    scope_authority_owners: Vec<String>,
312    privileged_operator_prompt: PrivilegedOperatorPrompt,
313    /// Counts executed rescans that set out to change the supervised modules'
314    /// staged or applied configuration, counted before the first change so a
315    /// rescan that fails partway still counts. With the daemon incarnation it
316    /// names the state a `supervisor.rescan_checked` preview was computed
317    /// against.
318    generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321/// Refusal labels passed to `observe_route_open_refusal` that mean the target
322/// module is not serving right now, and so open or extend an outage in the
323/// route outage tracker. Every one of them is only reachable after the target
324/// was found in the registry, which is what keeps an arbitrary client-chosen
325/// id from ever creating tracker state.
326///
327/// Deliberately absent: `not_registered` and `removed` (the id may be
328/// anything a client sent, and a removed module is gone on purpose),
329/// `protocol_none` (such a module never serves routes, so nothing is out),
330/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
331/// capability and admission-facts refusals (they refuse the caller, not a
332/// module outage), and `relay_reservation_failed` (its code ranges over
333/// capacity limits as well as a vanished connection). Capacity, breaker,
334/// relay-timeout and module-rejection refusals do not pass through that
335/// function at all; the breaker logs its own transitions.
336///
337/// The two not-serving refusals that bypass that function record themselves
338/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
339/// `required_capability_unprovided` is not tracked: the module itself is up,
340/// and the outage belongs to the missing provider.
341const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342    "reloading",
343    "supervisor_not_live",
344    "registration_not_active",
345    "no_forwarding_connection",
346    "relay_send_failed",
347];
348
349/// Real channel-0 control handler for subc itself.
350#[derive(Clone)]
351pub struct ControlHandler {
352    registry: Arc<Registry>,
353    activity: Option<crate::activity::ActivityLedger>,
354    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355    forwarding: Arc<ForwardingTable>,
356    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357    supervisor: SupervisorHandle,
358    subc_capabilities: Arc<[String]>,
359    /// Daemon-wide route.bind relay budget. Used as the fallback when the
360    /// target module has no per-module override in
361    /// `route_bind_relay_timeouts`.
362    route_bind_relay_timeout: Duration,
363    /// Per-module route.bind relay budget overrides, keyed by module id. When
364    /// `handle_route_open` resolves the deadline for a target module, a
365    /// per-module entry wins over the daemon-wide value above.
366    route_bind_relay_timeouts: BTreeMap<String, Duration>,
367    /// Per-target-module bind-relay breaker state. Shared with the forwarding
368    /// table, which is where a new module connection resets it.
369    route_bind_breakers: RouteBindBreakers,
370    /// Live relay admissions keyed by target module. Shared through the
371    /// forwarding table so cloned or separately built handlers enforce one cap.
372    route_bind_concurrency: RouteBindConcurrency,
373    /// Start and end of each module's not-serving period as seen by
374    /// `route.open`, so an outage gets one line at each edge instead of only
375    /// the per-refusal INFO lines. Taken from the forwarding table, so every
376    /// handler built over one table shares it.
377    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378    /// Consecutive relay timeouts that open a module's breaker.
379    route_bind_breaker_threshold: u32,
380    /// How long a breaker stays open before one probe is admitted.
381    route_bind_breaker_cooldown: Duration,
382    health_probe_timeout: Duration,
383    /// Central storage policy. When set, each registering module receives its
384    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
385    storage_config: Option<crate::daemon_config::StorageConfig>,
386    /// The machine id established at boot, served on every HELLO_ACK and on
387    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
388    /// changes the file, never this value. `None` serves no id.
389    machine_id: Option<crate::machine_id::MachineId>,
390    admission_facts_carrier_module_id: Option<String>,
391    admission_facts_targets: Option<Vec<String>>,
392    /// Scope records with their sync authorities and tombstones; see
393    /// `crate::scopes`. Shared by clones of this handler, so every connection
394    /// reads and writes one table.
395    scopes: Arc<RwLock<ScopeTable>>,
396    /// Wakes the expiry loop when an accepted scope change may have added a
397    /// deadline, so the loop can wait without ticking while no live scope has
398    /// one. A permit is stored when nothing waits, so no wake is lost.
399    scope_deadline_added: Arc<tokio::sync::Notify>,
400    /// The configured `scope_authority_owners`, kept so a rescan can report a
401    /// changed value as needing a daemon restart; rescan never applies it.
402    scope_authority_owners: Vec<String>,
403    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
404    privileged_operator_prompt: PrivilegedOperatorPrompt,
405    /// The local and peer address of each accepted TCP client connection, read
406    /// from the daemon's own socket, so a privileged request can name its caller.
407    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408    #[cfg(test)]
409    caller_lookup: Option<TestCallerLookup>,
410    /// The launch nonce each module connection presented at HELLO, which is how
411    /// a `scope.sync` is matched to the owner's current launch.
412    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413    rescan: Option<SupervisorRescanContext>,
414    connected_clients: ConnectedClients,
415    counters: DaemonCounters,
416    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
418    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
419    /// their usual severity (see [`log_requirement_events`]). A handler built
420    /// without one has a flag that is never set.
421    daemon_shutdown: DaemonShutdownFlag,
422    daemon_provenance: DaemonProvenanceFacts,
423    #[cfg(test)]
424    control_dispatch_delay: Option<Duration>,
425    #[cfg(test)]
426    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427    /// Holds `route.open` after the route.bind is queued to the module and
428    /// before it is marked enqueued, so a test can make the module's reply (or
429    /// a client teardown) land exactly inside that window.
430    #[cfg(test)]
431    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432    /// Holds a HELLO after its registry entry is reserved and before its
433    /// forwarding endpoint is installed, so a test can read the catalog and
434    /// open routes while the module is registered but not yet routable. A
435    /// blocking barrier because the HELLO handler is synchronous; the test
436    /// runs the HELLO on its own thread.
437    #[cfg(test)]
438    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443        f.debug_struct("ControlHandler")
444            .field("registry", &self.registry)
445            .field("forwarding", &self.forwarding)
446            .field("process_liveness", &self.process_liveness.is_some())
447            .field("supervisor", &self.supervisor)
448            .field("subc_capabilities", &self.subc_capabilities)
449            .finish()
450    }
451}
452
453struct RouteOpenRequest {
454    target: RouteTarget,
455    identity: BindIdentity,
456    consumer_identity: Option<ConsumerIdentity>,
457    consumer_capabilities: Option<Vec<String>>,
458    role_versions: Option<BTreeMap<String, String>>,
459    admission_facts: Option<serde_json::Value>,
460    scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465    Admission,
466    ProjectRoot,
467    EgressReservation,
468    BindSend,
469    BindAckWait,
470}
471
472/// Store elapsed time for each route-open phase, including attempts that end
473/// in an early refusal or backend error. Admission also includes rechecks and
474/// commit bookkeeping performed between the other timed phases.
475struct RouteOpenTimings {
476    module_id: String,
477    principal: String,
478    admission: Duration,
479    project_root: Duration,
480    egress_reservation: Duration,
481    bind_send: Duration,
482    bind_ack_wait: Duration,
483    phase: Option<RouteOpenPhase>,
484    phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488    fn new() -> Self {
489        Self {
490            module_id: String::new(),
491            // Refusals before attestation must not claim a verified principal.
492            principal: "unresolved".to_string(),
493            admission: Duration::ZERO,
494            project_root: Duration::ZERO,
495            egress_reservation: Duration::ZERO,
496            bind_send: Duration::ZERO,
497            bind_ack_wait: Duration::ZERO,
498            phase: Some(RouteOpenPhase::Admission),
499            phase_started_at: Instant::now(),
500        }
501    }
502
503    fn finish(&mut self) {
504        let elapsed = self.phase_started_at.elapsed();
505        match self.phase.take() {
506            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511            None => {}
512        }
513    }
514
515    fn enter(&mut self, phase: RouteOpenPhase) {
516        self.finish();
517        self.phase = Some(phase);
518        self.phase_started_at = Instant::now();
519    }
520}
521
522struct RouteBindReservationGuard {
523    forwarding: Arc<ForwardingTable>,
524    endpoint: ModuleEndpointId,
525    relay_corr: u64,
526    armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530    forwarding: Arc<ForwardingTable>,
531    endpoint: ModuleEndpointId,
532    corr: u64,
533    armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538        Self {
539            forwarding,
540            endpoint,
541            corr,
542            armed: true,
543        }
544    }
545
546    fn disarm(&mut self) {
547        self.armed = false;
548    }
549}
550
551impl Drop for ModuleControlRpcGuard {
552    fn drop(&mut self) {
553        if self.armed {
554            let _ = self
555                .forwarding
556                .cancel_module_control_rpc(self.endpoint, self.corr);
557        }
558    }
559}
560
561impl RouteBindReservationGuard {
562    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563        Self {
564            forwarding,
565            endpoint,
566            relay_corr,
567            armed: true,
568        }
569    }
570
571    fn release_and_disarm(&mut self) {
572        if !self.armed {
573            return;
574        }
575        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576            self.endpoint,
577            self.relay_corr,
578            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579        ) {
580            send_goodbye_target_best_effort(
581                &self.forwarding.counters(),
582                &target,
583                "canceled route.bind",
584            );
585        }
586        self.armed = false;
587    }
588
589    fn disarm(&mut self) {
590        self.armed = false;
591    }
592}
593
594impl Drop for RouteBindReservationGuard {
595    fn drop(&mut self) {
596        self.release_and_disarm();
597    }
598}
599
600/// Per-target-module circuit breaker around the `route.bind` relay.
601///
602/// The connection reader is serial per connection, so a module whose `on_bind`
603/// sits on the ack blocks every LATER frame on the connections that call it,
604/// including calls to unrelated modules. This does not make any module's bind
605/// fast; it stops the daemon paying the full budget again and again for a
606/// condition it has already observed.
607///
608/// State is keyed by TARGET MODULE and shared by every connection: a wedged
609/// module wedges everyone, so what one connection learned should protect the
610/// rest.
611///
612/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
613/// relay to that module has actually timed out, and is removed again when a
614/// relay is accepted or the module reconnects, so it cannot grow with traffic
615/// or with modules that behave.
616///
617/// # Why a `std` mutex here is not the head-of-line defect again
618///
619/// Acquisition never awaits. The critical section is a hash lookup plus a few
620/// integer updates, with no I/O and no `.await` inside it, so a reader task
621/// cannot be descheduled behind it the way it can behind
622/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
623/// primitive, held for the same kind of work, as the refusal counter this very
624/// path already increments.
625///
626/// It is also NOT on the data-plane splice path: only `route.open` and module
627/// registration touch it, so bound-route frames gain no state check and no
628/// contention.
629#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636    modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640    concurrency: RouteBindConcurrency,
641    module_id: String,
642}
643
644impl RouteBindConcurrency {
645    /// Admit without waiting. Waiting here would move the bind stall from the
646    /// module reply to a semaphore and restore reader head-of-line blocking.
647    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648        let mut modules = self
649            .modules
650            .lock()
651            .expect("route.bind concurrency mutex poisoned");
652        let in_flight = modules.entry(module_id.to_string()).or_default();
653        if *in_flight >= limit {
654            return Err(*in_flight);
655        }
656        *in_flight += 1;
657        Ok(RouteBindConcurrencyGuard {
658            concurrency: self.clone(),
659            module_id: module_id.to_string(),
660        })
661    }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665    fn drop(&mut self) {
666        let mut modules = self
667            .concurrency
668            .modules
669            .lock()
670            .expect("route.bind concurrency mutex poisoned");
671        let remove = {
672            let in_flight = modules
673                .get_mut(&self.module_id)
674                .expect("admitted route.bind has a concurrency entry");
675            *in_flight -= 1;
676            *in_flight == 0
677        };
678        if remove {
679            modules.remove(&self.module_id);
680        }
681    }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686    /// Relay timeouts observed with no accepted relay in between.
687    consecutive_timeouts: u32,
688    /// `Some` while the breaker is open: the instant the cooldown expires and
689    /// the next arrival may probe. `None` means closed.
690    cooldown_until: Option<Instant>,
691    /// A half-open probe has been admitted and has not settled yet. This is
692    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
693    /// that read the cooldown, so concurrent opens arriving at the moment the
694    /// cooldown expires cannot all decide that they are the probe.
695    probe_in_flight: Option<Arc<()>>,
696}
697
698/// What the breaker decided for one `route.open`, before any relay work.
699enum RouteBindAdmission<'a> {
700    Admitted {
701        guard: RouteBindBreakerGuard<'a>,
702        /// This open is the single half-open probe, so the transition is worth
703        /// one log line.
704        probe: bool,
705    },
706    Refused {
707        consecutive_timeouts: u32,
708        /// What is left of the cooldown. Zero when the refusal is because the
709        /// one probe is already in flight rather than because the cooldown has
710        /// not elapsed.
711        retry_in: Duration,
712        probe_in_flight: bool,
713    },
714}
715
716/// An outstanding admission, which must be told how its relay settled.
717///
718/// `Drop` settles it as inconclusive, so an early return between admission and
719/// the relay -- or the whole handler being cancelled when the client
720/// disconnects -- releases a half-open probe slot instead of leaving the
721/// breaker wedged half-open with no further probes.
722struct RouteBindBreakerGuard<'a> {
723    breakers: RouteBindBreakers,
724    module_id: &'a str,
725    probe_token: Option<Arc<()>>,
726    settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730    /// The module answered within the budget and took the bind. THE ONLY
731    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
732    /// breaker, which is a transition worth logging.
733    fn record_accepted(&mut self) -> bool {
734        self.settled = true;
735        self.breakers.record_accepted(self.module_id)
736    }
737
738    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
739    /// COUNTS TOWARD OPENING.
740    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741        self.settled = true;
742        self.breakers.record_timeout(
743            self.module_id,
744            self.probe_token.as_ref(),
745            threshold,
746            cooldown,
747        )
748    }
749
750    /// Everything else: the module REJECTED the bind, its connection went away
751    /// mid-relay, or the waiter was cancelled.
752    ///
753    /// None of these is evidence that a module is slow, and each already has
754    /// its own refusal with its own code. A module that rejects a bind in
755    /// microseconds is healthy and must never be convicted for it; a module
756    /// that died has said nothing about the module that replaces it. So these
757    /// neither increment nor reset the count -- they only release a probe slot.
758    fn record_inconclusive(&mut self) {
759        self.settled = true;
760        self.breakers
761            .record_inconclusive(self.module_id, self.probe_token.as_ref());
762    }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766    fn drop(&mut self) {
767        if !self.settled {
768            self.breakers
769                .record_inconclusive(self.module_id, self.probe_token.as_ref());
770        }
771    }
772}
773
774/// The breaker moved to open, reported so the caller can log it outside the
775/// lock. Opening is rare and load-bearing; the refusals that follow are
776/// frequent and are counted rather than logged.
777struct BreakerOpened {
778    consecutive_timeouts: u32,
779    /// True when a failed probe re-opened an already-open breaker, which reads
780    /// very differently in a log from a first opening.
781    reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786        self.modules
787            .lock()
788            .expect("route.bind breaker mutex poisoned")
789    }
790
791    /// Decide whether this `route.open` may attempt its relay. Takes the map
792    /// lock and nothing else, and never awaits.
793    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795            probe: probe_token.is_some(),
796            guard: RouteBindBreakerGuard {
797                breakers: self.clone(),
798                module_id,
799                probe_token,
800                settled: false,
801            },
802        };
803
804        let mut modules = self.lock();
805        let Some(state) = modules.get_mut(module_id) else {
806            return admitted(None);
807        };
808        let Some(cooldown_until) = state.cooldown_until else {
809            return admitted(None);
810        };
811        if state.probe_in_flight.is_some() {
812            return RouteBindAdmission::Refused {
813                consecutive_timeouts: state.consecutive_timeouts,
814                retry_in: Duration::ZERO,
815                probe_in_flight: true,
816            };
817        }
818        let now = Instant::now();
819        if now < cooldown_until {
820            return RouteBindAdmission::Refused {
821                consecutive_timeouts: state.consecutive_timeouts,
822                retry_in: cooldown_until - now,
823                probe_in_flight: false,
824            };
825        }
826        let token = Arc::new(());
827        state.probe_in_flight = Some(Arc::clone(&token));
828        admitted(Some(token))
829    }
830
831    fn record_accepted(&self, module_id: &str) -> bool {
832        self.lock()
833            .remove(module_id)
834            .is_some_and(|state| state.cooldown_until.is_some())
835    }
836
837    fn record_timeout(
838        &self,
839        module_id: &str,
840        probe_token: Option<&Arc<()>>,
841        threshold: u32,
842        cooldown: Duration,
843    ) -> Option<BreakerOpened> {
844        let mut modules = self.lock();
845        let state = modules.entry(module_id.to_string()).or_default();
846        let was_open = state.cooldown_until.is_some();
847        let was_probe = Self::owns_probe(state, probe_token);
848        if was_probe {
849            state.probe_in_flight = None;
850        }
851        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852        if state.consecutive_timeouts < threshold {
853            return None;
854        }
855        state.cooldown_until = Some(Instant::now() + cooldown);
856        Some(BreakerOpened {
857            consecutive_timeouts: state.consecutive_timeouts,
858            reopened_after_probe: was_open && was_probe,
859        })
860    }
861
862    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863        // A relay can finish after the breaker was reset or after it opened
864        // again and started a new probe. Only the guard whose token matches the
865        // active probe may release it, so a late relay never frees a newer probe.
866        state
867            .probe_in_flight
868            .as_ref()
869            .zip(token)
870            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871    }
872
873    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874        if let Some(state) = self.lock().get_mut(module_id) {
875            if Self::owns_probe(state, probe_token) {
876                state.probe_in_flight = None;
877            }
878        }
879    }
880
881    /// Discard what was learned about a module, because the process it was
882    /// learned about is gone. Returns the discarded count when it was non-zero.
883    ///
884    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
885    /// `module_id` is a configuration identity that outlives any particular
886    /// child; what the breaker observed was the process behind the module
887    /// connection of the moment. When a new connection registers under that id
888    /// the verdict's subject no longer exists, so the verdict is stale by
889    /// construction rather than merely likely to be wrong. Keeping it would
890    /// apply a dead process's record to a live one, which is the same defect
891    /// class this breaker exists to stop the daemon committing.
892    ///
893    /// A half-open probe in flight is discarded with the rest: it was a
894    /// question about the old process.
895    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896        self.lock()
897            .remove(module_id)
898            .map(|state| state.consecutive_timeouts)
899            .filter(|discarded| *discarded > 0)
900    }
901
902    /// Open breakers, for the `server.describe` counters object. `None` when
903    /// none is open, so the key stays absent rather than present-and-empty.
904    ///
905    /// This is the operator's answer to "is this module refusing instantly or
906    /// is it fine?", which look identical from a client that retries and then
907    /// succeeds.
908    fn open_snapshot(&self) -> Option<serde_json::Value> {
909        let now = Instant::now();
910        let modules = self.lock();
911        let open = modules
912            .iter()
913            .filter_map(|(module_id, state)| {
914                let cooldown_until = state.cooldown_until?;
915                Some((
916                    module_id.clone(),
917                    serde_json::json!({
918                        "consecutive_timeouts": state.consecutive_timeouts,
919                        "cooldown_remaining_ms":
920                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
921                        "probe_in_flight": state.probe_in_flight.is_some(),
922                    }),
923                ))
924            })
925            .collect::<serde_json::Map<String, serde_json::Value>>();
926        (!open.is_empty()).then_some(serde_json::Value::Object(open))
927    }
928}
929
930impl ControlHandler {
931    pub fn new(registry: Arc<Registry>) -> Self {
932        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933    }
934
935    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936        let counters = forwarding.counters();
937        // Taken from the forwarding table rather than created here, so that the
938        // breaker a `route.open` consults is the same one a module's
939        // registration resets, however many handlers are built over one table.
940        let route_bind_breakers = forwarding.route_bind_breakers();
941        let route_bind_concurrency = forwarding.route_bind_concurrency();
942        let route_outages = forwarding.route_outages();
943        Self {
944            registry,
945            activity: None,
946            activity_streams: Arc::new(Mutex::new(HashMap::new())),
947            forwarding,
948            process_liveness: None,
949            supervisor: SupervisorHandle::new(),
950            subc_capabilities: Arc::from([
951                CAP_MANIFEST_REGISTRATION.to_string(),
952                CAP_CHANNEL_LIFECYCLE.to_string(),
953                CAP_PING_PONG.to_string(),
954                CAP_SESSION_ATTACH.to_string(),
955                CAP_ADMISSION_FACTS_RELAY.to_string(),
956                CAP_SCOPES_V1.to_string(),
957                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
964                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
965                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
966                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
967                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
968            ]),
969            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
970            route_bind_relay_timeouts: BTreeMap::new(),
971            route_bind_breakers,
972            route_bind_concurrency,
973            route_outages,
974            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
975            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
976            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
977            storage_config: None,
978            machine_id: None,
979            admission_facts_carrier_module_id: None,
980            admission_facts_targets: None,
981            scopes: Arc::new(RwLock::new(ScopeTable::new(
982                crate::daemon_config::default_scope_authority_owners(),
983            ))),
984            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
985            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
986            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
987            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
988            #[cfg(test)]
989            caller_lookup: None,
990            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
991            rescan: None,
992            connected_clients: ConnectedClients::new(),
993            counters,
994            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
995            daemon_shutdown: DaemonShutdownFlag::default(),
996            daemon_provenance: DaemonProvenanceFacts::default(),
997            #[cfg(test)]
998            control_dispatch_delay: None,
999            #[cfg(test)]
1000            provenance_probe_override: None,
1001            #[cfg(test)]
1002            route_bind_relay_sent_hold: None,
1003            #[cfg(test)]
1004            hello_registry_to_forwarding_hold: None,
1005        }
1006    }
1007
1008    /// Set the central storage policy: registering modules then receive their
1009    /// resolved storage descriptor in HELLO_ACK.
1010    pub fn with_storage_config(
1011        mut self,
1012        storage_config: Option<crate::daemon_config::StorageConfig>,
1013    ) -> Self {
1014        self.storage_config = storage_config;
1015        self
1016    }
1017
1018    /// Open the ledger on its own writer thread; an unavailable sink does not
1019    /// prevent the daemon from serving routing or supervisor requests.
1020    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1021        self.activity = Some(crate::activity::ActivityLedger::open(
1022            run_dir.join("activity.jsonl"),
1023            incarnation,
1024        ));
1025        self.supervisor.set_activity(self.activity.clone());
1026        self
1027    }
1028
1029    /// Set the machine id served to every registering module (HELLO_ACK) and on
1030    /// `server.describe`.
1031    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1032        self.machine_id = machine_id;
1033        self
1034    }
1035
1036    /// Configure the exact reserved module and target ids permitted to relay
1037    /// opaque admission facts. Config-file loading validates this authority;
1038    /// this builder keeps the same policy available to embedded test daemons.
1039    pub fn with_admission_facts_config(
1040        mut self,
1041        carrier_module_id: Option<String>,
1042        targets: Option<Vec<String>>,
1043    ) -> Self {
1044        self.admission_facts_carrier_module_id = carrier_module_id;
1045        self.admission_facts_targets = targets;
1046        self
1047    }
1048
1049    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1050    /// Replaces the scope table with an empty one under the new list, so call it
1051    /// while building the handler, before any module can sync.
1052    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1053        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1054        self.scope_authority_owners = owners;
1055        self
1056    }
1057
1058    /// Set the startup policy before installing the rescan context or serving requests.
1059    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1060        self.privileged_operator_prompt = mode;
1061        self
1062    }
1063
1064    /// Override the route.bind relay timeout. Used by tests that assert the
1065    /// timeout path so they don't block on the production-safe default.
1066    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1067        self.route_bind_relay_timeout = timeout;
1068        self
1069    }
1070
1071    /// Install per-module route.bind relay budget overrides. A module id
1072    /// listed here wins over the daemon-wide default set via
1073    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1074    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1075    /// the same `Duration` the bind path will use.
1076    pub fn with_route_bind_relay_timeouts(
1077        mut self,
1078        timeouts: impl IntoIterator<Item = (String, Duration)>,
1079    ) -> Self {
1080        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1081        self
1082    }
1083
1084    /// Resolve the route.bind relay budget for a specific target module id.
1085    /// Per-module overrides win; the daemon-wide value (set via
1086    /// `with_route_bind_relay_timeout` or the built-in default) is the
1087    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1088    /// the same resolution `handle_route_open` will use.
1089    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1090        self.route_bind_relay_timeouts
1091            .get(module_id)
1092            .copied()
1093            .unwrap_or(self.route_bind_relay_timeout)
1094    }
1095
1096    /// Override the per-module bind-relay breaker policy.
1097    ///
1098    /// Used by tests, which cannot spend three production budgets opening a
1099    /// breaker or twenty seconds waiting for its cooldown. The production
1100    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1101    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1102    /// reasoning for the numbers.
1103    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1104        self.route_bind_breaker_threshold = threshold.max(1);
1105        self.route_bind_breaker_cooldown = cooldown;
1106        self
1107    }
1108
1109    #[cfg(test)]
1110    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1111        self.health_probe_timeout = timeout;
1112        self
1113    }
1114
1115    #[cfg(test)]
1116    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1117        self.control_dispatch_delay = Some(delay);
1118        self
1119    }
1120
1121    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1122    /// announce it is parked, once to be released. Build the barrier for two
1123    /// parties, the test being the other one.
1124    #[cfg(test)]
1125    pub(crate) fn with_route_bind_relay_sent_hold(
1126        mut self,
1127        hold: Arc<tokio::sync::Barrier>,
1128    ) -> Self {
1129        self.route_bind_relay_sent_hold = Some(hold);
1130        self
1131    }
1132
1133    /// A HELLO waits on `hold` twice between reserving its registry entry and
1134    /// installing its forwarding endpoint: once to announce it is parked, once
1135    /// to be released. Build the barrier for two parties, the test being the
1136    /// other one, and run the HELLO on a thread of its own.
1137    #[cfg(test)]
1138    pub(crate) fn with_hello_registry_to_forwarding_hold(
1139        mut self,
1140        hold: Arc<std::sync::Barrier>,
1141    ) -> Self {
1142        self.hello_registry_to_forwarding_hold = Some(hold);
1143        self
1144    }
1145
1146    #[cfg(test)]
1147    fn hold_between_registry_and_forwarding(&self) {
1148        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1149            hold.wait();
1150            hold.wait();
1151        }
1152    }
1153
1154    pub fn with_process_liveness(
1155        mut self,
1156        process_liveness: Arc<dyn ModuleProcessLiveness>,
1157    ) -> Self {
1158        self.process_liveness = Some(process_liveness);
1159        self
1160    }
1161
1162    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1163        self.supervisor = supervisor;
1164        self.supervisor.set_activity(self.activity.clone());
1165        self
1166    }
1167
1168    pub fn with_daemon_provenance(
1169        mut self,
1170        pid: u32,
1171        started_at_ms: u64,
1172        executable_path: Option<PathBuf>,
1173        build_git_sha: Option<String>,
1174        build_lock_digest: Option<String>,
1175    ) -> Self {
1176        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1177        let process_start_time = process_start_time(pid);
1178        self.daemon_provenance = DaemonProvenanceFacts {
1179            build: DaemonBuildProvenance {
1180                build_git_sha,
1181                build_lock_digest,
1182            },
1183            pid: Some(pid),
1184            started_at_ms: Some(started_at_ms),
1185            start_clock: None,
1186            executable_path,
1187            executable_identity,
1188            process_start_time,
1189            probe: ExecutableIdentityProbe::default(),
1190        };
1191        self
1192    }
1193
1194    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1195        self.daemon_provenance.start_clock = Some(clock);
1196        self
1197    }
1198
1199    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1200    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1201        self.scopes
1202            .write()
1203            .unwrap_or_else(|poisoned| poisoned.into_inner())
1204            .set_wall_clock(Arc::new(clock));
1205        self
1206    }
1207
1208    #[cfg(test)]
1209    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1210        self.provenance_probe_override = Some(result);
1211        self
1212    }
1213
1214    /// Install the configured module set and its reserved capability bindings.
1215    /// Bindings are configuration-scoped and may point at a provider that has not
1216    /// been installed yet, so this does not require the bound module to exist.
1217    pub fn with_capability_config(
1218        self,
1219        modules: impl IntoIterator<Item = (String, bool)>,
1220        reserved_capabilities: BTreeMap<String, String>,
1221    ) -> Self {
1222        self.capability_evaluator
1223            .configure(modules, reserved_capabilities);
1224        self
1225    }
1226
1227    /// Share the supervisor's daemon-shutdown flag with this handler, so
1228    /// requirement changes after the daemon has begun its announced shutdown
1229    /// are not reported as faults.
1230    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1231        self.daemon_shutdown = flag;
1232        self
1233    }
1234
1235    pub fn with_supervisor_rescan(
1236        mut self,
1237        supervisor: Supervisor,
1238        config_path: impl Into<PathBuf>,
1239        configured_port: Option<u16>,
1240    ) -> Self {
1241        self.rescan = Some(SupervisorRescanContext {
1242            supervisor,
1243            config_path: config_path.into(),
1244            configured_port,
1245            storage_config: self.storage_config.clone(),
1246            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1247            admission_facts_targets: self.admission_facts_targets.clone(),
1248            scope_authority_owners: self.scope_authority_owners.clone(),
1249            privileged_operator_prompt: self.privileged_operator_prompt,
1250            generation: Arc::default(),
1251        });
1252        self
1253    }
1254
1255    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1256        self.connected_clients = connected_clients;
1257        self
1258    }
1259
1260    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1261        Arc::clone(&self.forwarding)
1262    }
1263
1264    pub(crate) fn counters(&self) -> DaemonCounters {
1265        self.counters.clone()
1266    }
1267
1268    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1269    /// requirement event without depending on an operator polling a status command.
1270    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1271        tokio::spawn(async move {
1272            loop {
1273                self.capability_evaluator
1274                    .wait_for_change_or_deadline()
1275                    .await;
1276                self.refresh_capability_requirements();
1277            }
1278        });
1279    }
1280
1281    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1282    /// deadline passed during sleep is enforced on the first tick after wake.
1283    /// While no live scope carries a deadline the loop waits for an accepted
1284    /// scope change instead of ticking, so a daemon without deadlines never
1285    /// wakes for expiry.
1286    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1287        tokio::spawn(async move {
1288            loop {
1289                let has_deadlines = self
1290                    .scopes
1291                    .read()
1292                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1293                    .has_deadlines();
1294                if !has_deadlines {
1295                    self.scope_deadline_added.notified().await;
1296                    continue;
1297                }
1298                tokio::time::sleep(Duration::from_secs(1)).await;
1299                if let Err(error) = self.sweep_expired_scopes() {
1300                    warn!(%error, "scope expiry sweep failed");
1301                }
1302            }
1303        });
1304    }
1305
1306    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1307        let mut table = self
1308            .scopes
1309            .write()
1310            .unwrap_or_else(|poisoned| poisoned.into_inner());
1311        let swept = table.sweep_expired();
1312        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1313        drop(table);
1314        self.close_scope_drained_routes(drained);
1315        Ok(())
1316    }
1317
1318    fn publish_scope_changes(
1319        &self,
1320        changes: &[crate::scopes::ScopeTagChange],
1321        expired: &[crate::scopes::ScopeExpired],
1322    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1323        if changes.is_empty() {
1324            return Ok(Vec::new());
1325        }
1326        let drained = self
1327            .forwarding
1328            .publish_scope_changes(changes)
1329            .map_err(RouterError::Forwarding)?;
1330        let mut counts = HashMap::new();
1331        for route in &drained {
1332            *counts
1333                .entry((
1334                    route.scope.owner.as_str(),
1335                    route.scope.scope_ref.as_str(),
1336                    route.scope.tag.scope_epoch,
1337                ))
1338                .or_insert(0usize) += 1;
1339        }
1340        for scope in expired {
1341            let routes_closed = counts
1342                .get(&(
1343                    scope.owner.as_str(),
1344                    scope.scope_ref.as_str(),
1345                    scope.scope_epoch,
1346                ))
1347                .copied()
1348                .unwrap_or(0);
1349            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1350                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1351                routes_closed, "scope expired");
1352        }
1353        Ok(drained)
1354    }
1355
1356    fn runtime_capability_snapshot(
1357        &self,
1358    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1359        let runtime = self
1360            .supervisor
1361            .list()
1362            .into_iter()
1363            .map(|module| {
1364                let status = module.status().map_err(|err| {
1365                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1366                })?;
1367                Ok(RuntimeModule {
1368                    module_id: status.module_id,
1369                    state: status.state,
1370                    enabled: status.enabled,
1371                })
1372            })
1373            .collect::<Result<Vec<_>, RouterError>>()?;
1374        // Include every registration, published or not. The capability
1375        // evaluator counts a module as providing its capabilities as soon as it
1376        // has registered, even before `catalog.list` shows it, so a HELLO in
1377        // progress does not change other modules' required-capability
1378        // verdicts. See `Registry::list_registered_modules`.
1379        let registrations = self.registry.list_registered_modules().map_err(|err| {
1380            RouterError::backend(
1381                0,
1382                0,
1383                format!("failed to list capability registrations: {err}"),
1384            )
1385        })?;
1386        let registrations = registrations
1387            .into_iter()
1388            .map(|registration| RegisteredModule {
1389                module_id: registration.manifest.module_id,
1390                module_version: registration.manifest.module_version,
1391                capabilities: registration.manifest.capabilities,
1392            })
1393            .collect();
1394        Ok((runtime, registrations))
1395    }
1396
1397    /// The capability side effects of a module becoming the active registration
1398    /// for its id: cache its manifest (warning if its claims drifted), run the
1399    /// deny census when its declarations call for one, and recompute the
1400    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1401    /// candidate's does not, and the supervisor does it at promotion instead,
1402    /// through [`crate::supervise::SwapPromotionObserver`].
1403    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1404        let cached_registration = RegisteredModule {
1405            module_id: registration.manifest.module_id.clone(),
1406            module_version: registration.manifest.module_version.clone(),
1407            capabilities: registration.manifest.capabilities.clone(),
1408        };
1409        if self.capability_evaluator.record_hello(&cached_registration) {
1410            warn!(
1411                module_id = %cached_registration.module_id,
1412                "capability claims drifted from the cached manifest"
1413            );
1414        }
1415        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1416            self.enforce_capability_denies();
1417        }
1418        self.refresh_capability_requirements();
1419    }
1420
1421    /// Point the shared supervisor handle at this handler for swap promotions.
1422    /// Called wherever a handler is put behind the `Arc` the router serves, so
1423    /// it can be held weakly.
1424    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1425        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1426            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1427        self.supervisor.set_swap_promotion_observer(observer);
1428    }
1429
1430    pub fn refresh_capability_requirements(&self) {
1431        match self.runtime_capability_snapshot() {
1432            Ok((runtime, registrations)) => {
1433                log_requirement_events(
1434                    self.capability_evaluator
1435                        .evaluate_now(&runtime, &registrations),
1436                    &self.daemon_shutdown,
1437                );
1438            }
1439            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1440        }
1441    }
1442
1443    /// Reconcile only live, attested route bindings after a capability deny edge
1444    /// or target claim was added. This is deliberately a control-plane census:
1445    /// the opaque forwarding hot path must not grow a per-frame capability check.
1446    fn enforce_capability_denies(&self) {
1447        let registrations = match self.registry.list_registered_modules() {
1448            Ok(snapshot) => snapshot,
1449            Err(err) => {
1450                warn!(error = %err, "failed to read registrations for capability deny census");
1451                return;
1452            }
1453        };
1454        let manifests = registrations
1455            .into_iter()
1456            .map(|registration| {
1457                (
1458                    registration.manifest.module_id.clone(),
1459                    registration.manifest,
1460                )
1461            })
1462            .collect::<BTreeMap<_, _>>();
1463        let census = match self.forwarding.route_census(None) {
1464            Ok(census) => census,
1465            Err(err) => {
1466                warn!(error = %err, "failed to read route census for capability deny enforcement");
1467                return;
1468            }
1469        };
1470
1471        for (target_module_id, routes) in census {
1472            let Some(target_manifest) = manifests.get(&target_module_id) else {
1473                continue;
1474            };
1475            let mut closed_routes = Vec::new();
1476            let mut goodbyes = Vec::new();
1477            for route in routes {
1478                let Principal::Reserved {
1479                    module_id: opening_module_id,
1480                } = &route.principal
1481                else {
1482                    continue;
1483                };
1484                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1485                    continue;
1486                };
1487                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1488                    continue;
1489                };
1490
1491                match self.forwarding.release_client_route(
1492                    route.goodbye_target.connection_id,
1493                    route.goodbye_target.channel,
1494                    route.goodbye_target.epoch,
1495                ) {
1496                    Ok(RouteRelease::Removed(module_goodbye)) => {
1497                        warn!(
1498                            opening_module_id,
1499                            target_module_id,
1500                            capability,
1501                            "force-closing route because an attested capability deny edge now matches"
1502                        );
1503                        goodbyes.push(module_goodbye);
1504                        goodbyes.push(route.goodbye_target.clone());
1505                        closed_routes.push(route);
1506                    }
1507                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1508                    Err(err) => warn!(
1509                        opening_module_id,
1510                        target_module_id,
1511                        capability,
1512                        error = %err,
1513                        "failed to force-close capability-denied route"
1514                    ),
1515                }
1516            }
1517
1518            if closed_routes.is_empty() {
1519                continue;
1520            }
1521            send_route_control_pushes(
1522                &self.forwarding,
1523                closed_routes,
1524                ClientControlPush::RouteClosed {
1525                    module_id: target_module_id,
1526                    channels: Vec::new(),
1527                    reason: RouteCloseReason::CapabilityDenied,
1528                    drained: false,
1529                    abandoned: 0,
1530                    excluded_subscriptions: 0,
1531                    terminal: Some(false),
1532                },
1533            );
1534            // route.closed records the reason; the route GOODBYE ends the
1535            // client's handle and settles any calls still pending on it.
1536            self.emit_route_goodbyes(goodbyes);
1537        }
1538    }
1539
1540    /// Why a registered module is not accepting new route binds, or `None` when
1541    /// it is. This is the module's effective readiness: its declared readiness
1542    /// first, then every `need: required` capability it declares evaluating to
1543    /// `provided`. `route.open` and `catalog.list` both read it here so the
1544    /// catalog never reports a module routable that `route.open` would refuse.
1545    fn not_ready_reason(
1546        &self,
1547        registration: &crate::registry::ModuleRegistration,
1548    ) -> Option<NotReadyReason> {
1549        if !registration.ready {
1550            return Some(NotReadyReason {
1551                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1552                capability: None,
1553            });
1554        }
1555        self.first_unprovided_required_capability(registration)
1556            .map(|capability| NotReadyReason {
1557                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1558                capability: Some(capability),
1559            })
1560    }
1561
1562    /// The lexicographically first capability this registration declares
1563    /// `need: required` whose evaluator verdict is not `provided`.
1564    ///
1565    /// The verdicts are the capability evaluator's own; nothing here decides
1566    /// what "provided" means. The evaluator counts a capability provided as
1567    /// soon as a module claiming it has REGISTERED, not once that module is
1568    /// ready. That distinction is what keeps two modules that require each
1569    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1570    /// is ready", each would wait for the other to become ready first and
1571    /// neither ever would. Do not tighten it to readiness.
1572    ///
1573    /// A required capability with no verdict at all means this registration's
1574    /// HELLO or catalog.update landed after the last recompute; recompute once
1575    /// rather than let a missing verdict read as either answer. If it is still
1576    /// missing (the recompute itself failed) the capability counts as
1577    /// unprovided: the refusal is retryable, and routing a module whose
1578    /// required provider is unknown is the outcome this check exists to stop.
1579    fn first_unprovided_required_capability(
1580        &self,
1581        registration: &crate::registry::ModuleRegistration,
1582    ) -> Option<String> {
1583        let required = registration
1584            .manifest
1585            .capabilities
1586            .iter()
1587            .flat_map(|declarations| declarations.requires.iter())
1588            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1589            .map(|requirement| requirement.capability.as_str())
1590            .collect::<BTreeSet<_>>();
1591        if required.is_empty() {
1592            return None;
1593        }
1594        let module_id = registration.manifest.module_id.as_str();
1595        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1596        if required
1597            .iter()
1598            .any(|capability| verdict(capability).is_none())
1599        {
1600            self.refresh_capability_requirements();
1601        }
1602        required
1603            .into_iter()
1604            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1605            .map(str::to_string)
1606    }
1607
1608    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1609        self.capability_evaluator
1610            .statuses()
1611            .into_iter()
1612            .map(capability_requirement_status)
1613            .collect()
1614    }
1615
1616    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1617    /// registration-release watch. The signal is what the supervisor waits on
1618    /// before spawning a replacement, so it must only fire once forwarding
1619    /// teardown is also done (see [`Self::cleanup_connection`] /
1620    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1621    /// admitted but its forwarding endpoint could not be installed.
1622    fn deregister_connection(
1623        &self,
1624        connection_id: ConnectionId,
1625        reason: RegistrationEndReason,
1626    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1627        self.registry
1628            .deregister_connection_with_reason(connection_id, reason)
1629    }
1630
1631    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1632        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1633            return None;
1634        }
1635        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1636            parse_client_control_request(&frame.body)
1637        else {
1638            return None;
1639        };
1640        Some(target_module_id(&target).to_string())
1641    }
1642
1643    pub(crate) fn route_open_capacity_refusal(
1644        &self,
1645        ctx: &RouteCtx,
1646        frame: &Frame,
1647        target_module_id: &str,
1648        in_flight: usize,
1649        limit: usize,
1650    ) -> Result<Frame, RouterError> {
1651        // Sent before the request reaches `handle_route_open`, so the caller
1652        // fields come from the raw frame here.
1653        let caller = self.route_open_caller_from_frame(frame);
1654        self.route_open_admission_refusal_frame(
1655            ctx,
1656            &caller,
1657            frame,
1658            target_module_id,
1659            "open_admission_full",
1660            (in_flight, limit),
1661            format!(
1662                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1663            ),
1664        )
1665    }
1666
1667    fn route_open_target_capacity_refusal(
1668        &self,
1669        ctx: &RouteCtx,
1670        caller: &RouteOpenCaller,
1671        frame: &Frame,
1672        target_module_id: &str,
1673        in_flight: usize,
1674    ) -> Result<Frame, RouterError> {
1675        self.route_open_admission_refusal_frame(
1676            ctx,
1677            caller,
1678            frame,
1679            target_module_id,
1680            "target_binds_full",
1681            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1682            format!(
1683                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1684            ),
1685        )
1686    }
1687
1688    /// Admission pressure clears as existing binds settle, so its refusal must
1689    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1690    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1691    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1692    /// cannot currently reach its target; `module_timeout` would falsely claim
1693    /// that a wait expired. A new, cleaner code would be terminal to deployed
1694    /// clients, so it requires a client-tolerance rollout before daemon emission.
1695    #[allow(clippy::too_many_arguments)]
1696    fn route_open_admission_refusal_frame(
1697        &self,
1698        ctx: &RouteCtx,
1699        caller: &RouteOpenCaller,
1700        frame: &Frame,
1701        target_module_id: &str,
1702        reason: &'static str,
1703        (in_flight, limit): (usize, usize),
1704        message: impl Into<String>,
1705    ) -> Result<Frame, RouterError> {
1706        let code = error_codes::TARGET_UNAVAILABLE;
1707        self.counters.increment_route_open_refused(code);
1708        info!(
1709            target: "control",
1710            code,
1711            reason,
1712            module_id = ?target_module_id,
1713            connection_id = ctx.connection_id.get(),
1714            principal = %caller.principal,
1715            scope = %caller.scope,
1716            harness = %caller.harness,
1717            in_flight,
1718            limit,
1719            "route.open refused"
1720        );
1721        control_error_frame(frame, code, message.into())
1722    }
1723
1724    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1725    ///
1726    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1727    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1728    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1729    #[cfg(test)]
1730    pub fn handle_control(
1731        &self,
1732        connection_id: ConnectionId,
1733        frame: Frame,
1734    ) -> Result<Vec<Frame>, RouterError> {
1735        match frame.header.ty {
1736            FrameType::Ping => Ok(vec![pong(&frame)?]),
1737            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1738            FrameType::Goodbye => self.handle_goodbye(connection_id),
1739            ty => Ok(vec![control_error_frame(
1740                &frame,
1741                "unsupported_control_frame",
1742                format!("unsupported channel-0 frame {ty:?}"),
1743            )?]),
1744        }
1745    }
1746
1747    pub async fn handle_control_frame(
1748        &self,
1749        ctx: &RouteCtx,
1750        frame: Frame,
1751    ) -> Result<Vec<Frame>, RouterError> {
1752        self.handle_control_frame_timed(ctx, frame, None).await
1753    }
1754
1755    pub(crate) fn record_caller_endpoints(
1756        &self,
1757        connection: ConnectionId,
1758        endpoints: Option<(SocketAddr, SocketAddr)>,
1759    ) {
1760        let mut callers = self
1761            .caller_endpoints
1762            .lock()
1763            .unwrap_or_else(|p| p.into_inner());
1764        callers.remove(&connection);
1765        if let Some(endpoints) = endpoints {
1766            callers.insert(connection, endpoints);
1767        }
1768    }
1769
1770    #[cfg(test)]
1771    pub(crate) fn with_caller_lookup_for_test(
1772        mut self,
1773        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1774    ) -> Self {
1775        self.caller_lookup = Some(Arc::new(lookup));
1776        self
1777    }
1778
1779    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1780        let endpoints = self
1781            .caller_endpoints
1782            .lock()
1783            .unwrap_or_else(|p| p.into_inner())
1784            .get(&connection)
1785            .copied();
1786        #[cfg(test)]
1787        let lookup = self.caller_lookup.clone();
1788        tokio::task::spawn_blocking(move || {
1789            #[cfg(test)]
1790            if let Some(lookup) = lookup {
1791                return lookup(endpoints);
1792            }
1793            endpoints
1794                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1795                .unwrap_or_default()
1796        })
1797        .await
1798        .unwrap_or_default()
1799    }
1800
1801    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1802        frame.header.channel == 0
1803            && frame.header.ty == FrameType::Request
1804            && self
1805                .forwarding
1806                .module_endpoint_for_connection(ctx.connection_id)
1807                .is_ok_and(|endpoint| endpoint.is_none())
1808            && parse_client_control_request(&frame.body)
1809                .ok()
1810                .is_some_and(|request| privileged_operation(&request).is_some())
1811    }
1812
1813    pub(crate) async fn handle_control_frame_timed(
1814        &self,
1815        ctx: &RouteCtx,
1816        frame: Frame,
1817        dispatch_started_at: Option<StdInstant>,
1818    ) -> Result<Vec<Frame>, RouterError> {
1819        match frame.header.ty {
1820            FrameType::Ping => Ok(vec![pong(&frame)?]),
1821            FrameType::Hello => {
1822                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1823            }
1824            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1825            FrameType::Cancel => {
1826                // A Cancel on channel 0 names a waiting operator.confirm or a
1827                // subscription. Successful cancellation has no ordinary reply.
1828                if self
1829                    .forwarding
1830                    .operator_confirms()
1831                    .cancel(ctx.connection_id, frame.header.corr)
1832                    || self
1833                        .supervisor
1834                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1835                    || self.cancel_activity_subscription(ctx, &frame)?
1836                {
1837                    Ok(Vec::new())
1838                } else {
1839                    Ok(vec![control_error_frame(
1840                        &frame,
1841                        "unknown_subscription",
1842                        "no subscription has this correlation id",
1843                    )?])
1844                }
1845            }
1846            FrameType::Request => {
1847                // This additive operation is not part of the existing exhaustive
1848                // module-control enum. Probe the op before decoding that enum.
1849                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1850                if op
1851                    .as_ref()
1852                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1853                {
1854                    return self.handle_activity_record(ctx, frame).await;
1855                }
1856                if op.as_ref().is_some_and(|probe| {
1857                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1858                }) {
1859                    return self.handle_activity_read(ctx, frame).await;
1860                }
1861                if op
1862                    .as_ref()
1863                    .is_some_and(|probe| probe.op == "operator.confirm")
1864                {
1865                    return self.handle_operator_confirm(ctx, frame);
1866                }
1867                if op
1868                    .as_ref()
1869                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1870                {
1871                    return self.handle_scope_served_by(ctx.connection_id, frame);
1872                }
1873                if self
1874                    .forwarding
1875                    .module_endpoint_for_connection(ctx.connection_id)
1876                    .map_err(RouterError::Forwarding)?
1877                    .is_some()
1878                {
1879                    if !is_known_module_request_op(&frame.body) {
1880                        return Ok(vec![control_error_frame(
1881                            &frame,
1882                            "unsupported_control_frame",
1883                            "module-originated channel-0 REQUEST is not supported",
1884                        )?]);
1885                    }
1886                    let request = match parse_module_control_request_from_module(&frame.body) {
1887                        Ok(request) => request,
1888                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1889                            return Ok(vec![control_error_frame(
1890                                &frame,
1891                                "unsupported_control_frame",
1892                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1893                            )?])
1894                        }
1895                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1896                            return Ok(vec![control_error_frame(
1897                                &frame,
1898                                "invalid_control_body",
1899                                format!("malformed module control body: {err}"),
1900                            )?])
1901                        }
1902                    };
1903                    let op = module_control_request_op(&request);
1904                    let corr = frame.header.corr;
1905                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1906                    let result =
1907                        self.handle_module_control_request(ctx.connection_id, frame, request);
1908                    log_slow_control_dispatch(
1909                        dispatch_started_at,
1910                        op,
1911                        ctx.connection_id,
1912                        corr,
1913                        None,
1914                    );
1915                    return result;
1916                }
1917
1918                if is_known_module_request_op(&frame.body) {
1919                    return Ok(vec![control_error_frame(
1920                        &frame,
1921                        "not_registered",
1922                        "catalog.update requires an active module registration owned by this connection",
1923                    )?]);
1924                }
1925
1926                let request = match parse_client_control_request(&frame.body) {
1927                    Ok(request) => request,
1928                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1929                        return Ok(vec![control_error_frame(
1930                            &frame,
1931                            "unknown_control_op",
1932                            format!("unknown client control op: {err}"),
1933                        )?])
1934                    }
1935                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1936                        return Ok(vec![control_error_frame(
1937                            &frame,
1938                            "invalid_control_body",
1939                            format!("malformed client control body: {err}"),
1940                        )?])
1941                    }
1942                };
1943                let op = client_control_request_op(&request);
1944                let corr = frame.header.corr;
1945                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1946                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1947                #[cfg(test)]
1948                if let Some(delay) = self.control_dispatch_delay {
1949                    tokio::time::sleep(delay).await;
1950                }
1951                let result = self
1952                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1953                    .await;
1954                if let Some(timings) = route_open_timings.as_mut() {
1955                    timings.finish();
1956                }
1957                log_slow_control_dispatch(
1958                    dispatch_started_at,
1959                    op,
1960                    ctx.connection_id,
1961                    corr,
1962                    route_open_timings.as_ref(),
1963                );
1964                result
1965            }
1966            FrameType::Push => {
1967                let Some(endpoint) = self
1968                    .forwarding
1969                    .module_endpoint_for_connection(ctx.connection_id)
1970                    .map_err(RouterError::Forwarding)?
1971                else {
1972                    return Ok(vec![control_error_frame(
1973                        &frame,
1974                        "unsupported_control_frame",
1975                        "client-originated channel-0 PUSH is not supported",
1976                    )?]);
1977                };
1978                self.handle_status_update(endpoint, frame)
1979            }
1980            FrameType::Response | FrameType::Error
1981                if self
1982                    .forwarding
1983                    .module_endpoint_for_connection(ctx.connection_id)
1984                    .map_err(RouterError::Forwarding)?
1985                    .is_some() =>
1986            {
1987                self.handle_module_relay_response(ctx.connection_id, frame)
1988            }
1989            ty => Ok(vec![control_error_frame(
1990                &frame,
1991                "unsupported_control_frame",
1992                format!("unsupported channel-0 frame {ty:?}"),
1993            )?]),
1994        }
1995    }
1996
1997    pub fn cleanup_connection(
1998        &self,
1999        connection_id: ConnectionId,
2000    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2001        self.caller_endpoints
2002            .lock()
2003            .unwrap_or_else(|p| p.into_inner())
2004            .remove(&connection_id);
2005        self.forwarding
2006            .operator_confirms()
2007            .connection_closed(connection_id);
2008        self.cleanup_connection_with_end_reason(
2009            connection_id,
2010            RegistrationEndReason::ConnectionClosed,
2011        )
2012    }
2013
2014    fn cleanup_connection_with_end_reason(
2015        &self,
2016        connection_id: ConnectionId,
2017        requested_reason: RegistrationEndReason,
2018    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2019        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2020            self.registry
2021                .get_module_by_connection(connection_id)?
2022                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2023                .and_then(|module| module.registration_end_reason().ok().flatten())
2024                .unwrap_or(requested_reason)
2025        } else {
2026            requested_reason
2027        };
2028        let crash_closed = self
2029            .registry
2030            .get_module_by_connection(connection_id)?
2031            .and_then(|registration| {
2032                self.forwarding
2033                    .module_endpoint_for_connection(connection_id)
2034                    .ok()
2035                    .flatten()
2036                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2037                    .map(|routes| (registration.manifest.module_id, routes))
2038            });
2039        let crash_closed = crash_closed.map(|(module_id, routes)| {
2040            let terminal = match self.supervisor.get(&module_id) {
2041                None => false,
2042                Some(module) => match module.will_recover_after_connection_loss() {
2043                    Ok(will_recover) => !will_recover,
2044                    Err(err) => {
2045                        warn!(
2046                            %module_id,
2047                            error = %err,
2048                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2049                        );
2050                        false
2051                    }
2052                },
2053            };
2054            // The forwarding table gates all providers at the start of daemon
2055            // shutdown, before their connections are closed. An ordinary
2056            // module disconnect still reports crash if that gate is not set.
2057            let (reason, drained) = match self
2058                .forwarding
2059                .daemon_drain_result_for_connection(connection_id)
2060            {
2061                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2062                Ok(None) => (RouteCloseReason::Crash, false),
2063                Err(err) => {
2064                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2065                    (RouteCloseReason::Crash, false)
2066                }
2067            };
2068            (module_id, routes, reason, drained, terminal)
2069        });
2070        let registrations = self.deregister_connection(connection_id, end_reason);
2071        let cleanup = if crash_closed.is_some() {
2072            self.forwarding.cleanup_connection_counted(connection_id)
2073        } else {
2074            // A module connection's teardown needs the count of abandoned
2075            // route.bind relays for its route.closed notice below. Any other
2076            // connection, such as a client's, sends no such notice and needs
2077            // only its routes released, so it uses the route-only wrapper and
2078            // reports zero.
2079            self.forwarding
2080                .cleanup_connection(connection_id)
2081                .map(|released| crate::forwarding::ConnectionCleanup {
2082                    released,
2083                    abandoned_relays: 0,
2084                })
2085        };
2086        // The route.closed push waits for forwarding teardown because only
2087        // teardown knows how many pending route.bind relays it aborted. It still
2088        // goes out before the GOODBYEs for the released routes, and its targets
2089        // were captured above, before teardown removed those routes.
2090        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2091            let abandoned = cleanup
2092                .as_ref()
2093                .map_or(0, |cleanup| cleanup.abandoned_relays);
2094            send_route_control_pushes(
2095                &self.forwarding,
2096                routes,
2097                ClientControlPush::RouteClosed {
2098                    module_id,
2099                    channels: Vec::new(),
2100                    reason,
2101                    drained,
2102                    abandoned,
2103                    excluded_subscriptions: 0,
2104                    terminal: Some(terminal),
2105                },
2106            );
2107        }
2108        if let Ok(cleanup) = cleanup {
2109            self.emit_route_goodbyes(cleanup.released);
2110        }
2111        // Signal the registration-release watch only now that BOTH registry and
2112        // forwarding teardown are done, so a supervisor waiting to spawn a
2113        // replacement never observes release while old routes still exist.
2114        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2115            crate::supervise::notify_registration_release();
2116            self.capability_evaluator.wake_deadline_loop();
2117            self.refresh_capability_requirements();
2118        }
2119        self.supervisor.remove_spawn_subscribers(connection_id);
2120        self.activity_streams
2121            .lock()
2122            .unwrap_or_else(|p| p.into_inner())
2123            .retain(|(id, _), task| {
2124                if *id == connection_id {
2125                    task.abort();
2126                    false
2127                } else {
2128                    true
2129                }
2130            });
2131        // Sync authority dies with its connection, so the owner's next
2132        // connection can take it; the owner's scopes stay as they are.
2133        self.hello_launch_nonces
2134            .lock()
2135            .unwrap_or_else(|poisoned| poisoned.into_inner())
2136            .forget(connection_id);
2137        self.scopes
2138            .write()
2139            .unwrap_or_else(|poisoned| poisoned.into_inner())
2140            .release_connection(connection_id);
2141        registrations
2142    }
2143
2144    pub(crate) fn handle_route_goodbye(
2145        &self,
2146        connection_id: ConnectionId,
2147        route_channel: u16,
2148        route_epoch: u32,
2149    ) -> Result<bool, RouterError> {
2150        debug!(
2151            connection_id = connection_id.get(),
2152            route_channel, route_epoch, "handling route GOODBYE"
2153        );
2154        let RouteRelease::Removed(released_route) = self
2155            .forwarding
2156            .release_client_route(connection_id, route_channel, route_epoch)
2157            .map_err(RouterError::Forwarding)?
2158        else {
2159            return Ok(false);
2160        };
2161        self.emit_route_goodbyes(vec![released_route]);
2162        Ok(true)
2163    }
2164
2165    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2166        for released in released_routes {
2167            let frame = match Frame::build_with_version(
2168                released.negotiated_ver,
2169                FrameType::Goodbye,
2170                control_flags(),
2171                released.channel,
2172                released.epoch,
2173                0,
2174                Vec::new(),
2175            ) {
2176                Ok(frame) => frame,
2177                Err(err) => {
2178                    warn!(
2179                        route_channel = released.channel,
2180                        error = %err,
2181                        "failed to build route GOODBYE frame"
2182                    );
2183                    continue;
2184                }
2185            };
2186            if !released.close_on_delivery_failure() {
2187                crate::forwarding::send_module_route_goodbye(
2188                    &self.counters,
2189                    &released.sink,
2190                    frame,
2191                    released.module_id.as_deref(),
2192                    "client route released",
2193                );
2194                continue;
2195            }
2196            if let Err(err) = released.sink.try_send(frame) {
2197                warn!(
2198                    target_connection_id = released.connection_id.get(),
2199                    route_channel = released.channel,
2200                    error = %err,
2201                    "route GOODBYE was not delivered to client; closing target connection"
2202                );
2203                if self
2204                    .forwarding
2205                    .escalate_client_delivery_failure(
2206                        released.connection_id,
2207                        released.channel,
2208                        released.epoch,
2209                        CloseReason::new(
2210                            "route_goodbye_delivery_failed",
2211                            format!(
2212                                "failed to enqueue route GOODBYE for channel {}: {err}",
2213                                released.channel
2214                            ),
2215                        ),
2216                        crate::forwarding::UndeliveredFrame {
2217                            module_id: released.module_id.as_deref(),
2218                            sink: &released.sink,
2219                        },
2220                    )
2221                    .unwrap_or(false)
2222                {
2223                    self.counters.increment_goodbye_relay_client_failed();
2224                }
2225            }
2226        }
2227    }
2228
2229    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2230    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2231    /// own commit failed after the module had already accepted). Without this, a
2232    /// module that accepts late keeps a binding subc has torn down, so a later
2233    /// frame on that module channel could misdeliver if the channel is reused.
2234    ///
2235    /// Never closes the shared module connection on failure: a dropped notification
2236    /// only wastes a bounded amount of warm module-side state, which the module's
2237    /// own idle reaper reclaims. Only call this once the route.bind relay was
2238    /// actually enqueued to the module — if the relay send itself failed, the
2239    /// module never created a binding and there is nothing to tear down.
2240    fn send_abandoned_route_bind_goodbye(
2241        &self,
2242        module_sink: &crate::FrameSink,
2243        negotiated_ver: u8,
2244        module_channel: u16,
2245        module_epoch: u32,
2246    ) {
2247        let frame = match Frame::build_with_version(
2248            negotiated_ver,
2249            FrameType::Goodbye,
2250            control_flags(),
2251            module_channel,
2252            module_epoch,
2253            0,
2254            Vec::new(),
2255        ) {
2256            Ok(frame) => frame,
2257            Err(err) => {
2258                warn!(
2259                    route_channel = module_channel,
2260                    error = %err,
2261                    "failed to build GOODBYE for abandoned route.bind"
2262                );
2263                return;
2264            }
2265        };
2266        crate::forwarding::send_module_route_goodbye(
2267            &self.counters,
2268            module_sink,
2269            frame,
2270            None,
2271            "abandoned route.bind",
2272        );
2273    }
2274
2275    fn handle_hello(
2276        &self,
2277        connection_id: ConnectionId,
2278        sink: Option<crate::FrameSink>,
2279        frame: Frame,
2280    ) -> Result<Vec<Frame>, RouterError> {
2281        debug!(
2282            connection_id = connection_id.get(),
2283            corr = frame.header.corr,
2284            "handling HELLO"
2285        );
2286        // A module connection has one identity for its entire lifetime. A second
2287        // registration would leave the old registry owner behind while replacing
2288        // its forwarding endpoint and launch nonce.
2289        if self
2290            .registry
2291            .get_module_by_connection(connection_id)
2292            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2293            .is_some()
2294        {
2295            return Ok(vec![control_error_frame(
2296                &frame,
2297                "invalid_hello",
2298                "connection is already registered as a module",
2299            )?]);
2300        }
2301        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2302            Ok(value) => value,
2303            Err(err) => {
2304                return Ok(vec![control_error_frame(
2305                    &frame,
2306                    "invalid_hello",
2307                    format!("malformed HELLO body: {err}"),
2308                )?])
2309            }
2310        };
2311        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2312            return Ok(vec![control_error_frame(
2313                &frame,
2314                "invalid_capability_grammar",
2315                err.to_string(),
2316            )?]);
2317        }
2318        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2319            return Ok(vec![control_error_frame(
2320                &frame,
2321                "invalid_manifest",
2322                err.to_string(),
2323            )?]);
2324        }
2325        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2326            return Ok(vec![control_error_frame(
2327                &frame,
2328                "invalid_event_declaration",
2329                err.to_string(),
2330            )?]);
2331        }
2332        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2333            return Ok(vec![control_error_frame(
2334                &frame,
2335                "invalid_manifest",
2336                err.to_string(),
2337            )?]);
2338        }
2339        if let Some(provenance) = hello_value
2340            .get("manifest")
2341            .and_then(|manifest| manifest.get("provenance"))
2342        {
2343            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2344                return Ok(vec![control_error_frame(
2345                    &frame,
2346                    "invalid_manifest",
2347                    format!("malformed manifest provenance: {err}"),
2348                )?]);
2349            }
2350        }
2351        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2352            let module_id = hello_value["manifest"]["module_id"]
2353                .as_str()
2354                .unwrap_or("<unknown>");
2355            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2356        }
2357        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2358            Ok(hello) => hello,
2359            Err(err) => {
2360                return Ok(vec![control_error_frame(
2361                    &frame,
2362                    "invalid_hello",
2363                    format!("malformed HELLO body: {err}"),
2364                )?])
2365            }
2366        };
2367
2368        if hello.protocol_ver != hello.manifest.protocol_ver {
2369            return Ok(vec![control_error_frame(
2370                &frame,
2371                "invalid_manifest",
2372                format!(
2373                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2374                    hello.protocol_ver, hello.manifest.protocol_ver
2375                ),
2376            )?]);
2377        }
2378
2379        if hello.manifest.module_id.trim().is_empty() {
2380            return Ok(vec![control_error_frame(
2381                &frame,
2382                "invalid_manifest",
2383                "manifest module_id must not be empty",
2384            )?]);
2385        }
2386
2387        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2388            Ok(negotiated_ver) => negotiated_ver,
2389            Err(message) => {
2390                return Ok(vec![control_error_frame(
2391                    &frame,
2392                    "version_unsupported",
2393                    message,
2394                )?])
2395            }
2396        };
2397
2398        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2399        // swap is open for this id, the only HELLO admitted as a second process
2400        // is the one carrying the candidate's launch nonce (the swap token), and
2401        // it registers into the candidate slot rather than being refused as a
2402        // duplicate. Run after the reserved gate, a reserved module's candidate
2403        // would be refused `reserved_module` for presenting a nonce that gate
2404        // does not know. See `SupervisorHandle::swap_hello_admission`.
2405        let swap_admission = self
2406            .supervisor
2407            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2408        if swap_admission == SwapHelloAdmission::Refused {
2409            warn!(
2410                module_id = %hello.manifest.module_id,
2411                connection_id = connection_id.get(),
2412                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2413            );
2414            return Ok(vec![control_error_frame(
2415                &frame,
2416                "swap_token_invalid",
2417                format!(
2418                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2419                    hello.manifest.module_id
2420                ),
2421            )?]);
2422        }
2423        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2424
2425        // A swap candidate proves its identity with the candidate nonce above.
2426        // Other registrations receive this catalog fact only when reserved-module
2427        // policy required and accepted a nonce; unreserved registrations require
2428        // no nonce check.
2429        let nonce_verified = if swap_candidate {
2430            true
2431        } else {
2432            match self.supervisor.reserved_hello_verification(
2433                &hello.manifest.module_id,
2434                hello.launch_nonce.as_deref(),
2435            ) {
2436                Ok(verified) => verified,
2437                Err(rejection) => {
2438                    let message = match rejection {
2439                        ReservedHelloRejection::Exact { module_id } => format!(
2440                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2441                        ),
2442                        ReservedHelloRejection::Prefix {
2443                            prefix,
2444                            owner_module_id,
2445                        } => format!(
2446                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2447                            hello.manifest.module_id
2448                        ),
2449                    };
2450                    return Ok(vec![control_error_frame(
2451                        &frame,
2452                        "reserved_module",
2453                        message,
2454                    )?]);
2455                }
2456            }
2457        };
2458
2459        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2460            &hello.manifest.module_id,
2461            hello.manifest.capabilities.as_ref(),
2462        );
2463        if let Some(refusal) = reserved_capability_refusals.first() {
2464            let capability = refusal.capability.clone();
2465            let bound_module = refusal.claimants[0].clone();
2466            log_duplicate_claim_events(reserved_capability_refusals);
2467            return Ok(vec![control_error_frame(
2468                &frame,
2469                "reserved_capability",
2470                format!(
2471                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2472                    capability, bound_module, hello.manifest.module_id
2473                ),
2474            )?]);
2475        }
2476
2477        // A connection that already opened client routes must not also register as
2478        // a module: cleanup would then release only one side and leak the other.
2479        if self
2480            .forwarding
2481            .connection_has_client_routes(connection_id)
2482            .map_err(RouterError::Forwarding)?
2483        {
2484            return Ok(vec![control_error_frame(
2485                &frame,
2486                "invalid_hello",
2487                "connection has open client routes and cannot also register as a module",
2488            )?]);
2489        }
2490
2491        // Kept for scope sync authority, which goes only to the connection that
2492        // presented the module's current launch nonce. Recorded before the
2493        // registration is attempted: a connection whose registration then fails
2494        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2495        self.hello_launch_nonces
2496            .lock()
2497            .unwrap_or_else(|poisoned| poisoned.into_inner())
2498            .record(connection_id, hello.launch_nonce.as_deref());
2499        let control_ops = effective_module_control_ops(hello.control_ops);
2500        // Built before anything is registered so an encoding failure leaves no
2501        // registry or forwarding state behind.
2502        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2503        if swap_candidate {
2504            return self.register_swap_candidate(
2505                connection_id,
2506                sink,
2507                &frame,
2508                hello.manifest,
2509                negotiated_ver,
2510                control_ops,
2511                hello_ack,
2512            );
2513        }
2514        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2515        // is refused, but `catalog.list` does not show the module and the
2516        // catalog generation does not move until it is published below, once
2517        // its forwarding endpoint exists. Listing it now would advertise a
2518        // module that `route.open` still refuses for having no forwarding
2519        // connection.
2520        let registration = match self
2521            .registry
2522            .register_unpublished_verified_with_control_ops(
2523                hello.manifest,
2524                negotiated_ver,
2525                connection_id,
2526                control_ops,
2527                nonce_verified,
2528            ) {
2529            Ok(registration) => registration,
2530            Err(RegistryError::DuplicateModuleId { module_id }) => {
2531                return Ok(vec![control_error_frame(
2532                    &frame,
2533                    "duplicate_module_id",
2534                    format!(
2535                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2536                    ),
2537                )?])
2538            }
2539            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2540                return Ok(vec![control_error_frame(
2541                    &frame,
2542                    "invalid_module_id",
2543                    err.to_string(),
2544                )?])
2545            }
2546            Err(err) => {
2547                return Ok(vec![control_error_frame(
2548                    &frame,
2549                    "registry_error",
2550                    err.to_string(),
2551                )?])
2552            }
2553        };
2554        #[cfg(test)]
2555        self.hold_between_registry_and_forwarding();
2556
2557        let reply = if let Some(sink) = sink {
2558            // The forwarding table's module store is also the daemon-to-module
2559            // control-RPC lane, so every HELLO gets a live endpoint even when the
2560            // manifest has no routable provider role. Non-routable modules still
2561            // cannot receive route.bind in production: `handle_route_open` checks
2562            // the registry manifest with `target_has_required_role` before the
2563            // only production call to `begin_route_bind_relay_for` below that
2564            // route.open path. The remaining direct relay callers are unit tests
2565            // and benchmark harnesses that construct forwarding state explicitly.
2566            //
2567            // The HELLO_ACK is queued by the forwarding table itself, before the
2568            // endpoint becomes visible, and is NOT returned as a reply. A module
2569            // reads HELLO_ACK first and exits on anything else; a reply is only
2570            // written after this handler returns, by which time a route.open on
2571            // another connection could already have queued a route.bind request
2572            // for this module ahead of it.
2573            let concurrency = manifest_concurrency(&registration.manifest);
2574            if let Err(err) = self.forwarding.register_module_connection_acked(
2575                connection_id,
2576                registration.manifest.module_id.clone(),
2577                negotiated_ver,
2578                concurrency,
2579                sink,
2580                hello_ack,
2581            ) {
2582                // Forwarding registration failed, so there is no forwarding
2583                // state to tear down. Remove the registry entry, which was never
2584                // published and so never listed or counted in the catalog
2585                // generation, and signal the release watch directly.
2586                if matches!(
2587                    self.deregister_connection(
2588                        connection_id,
2589                        RegistrationEndReason::RegistrationFailed,
2590                    ),
2591                    Ok(r) if !r.is_empty()
2592                ) {
2593                    crate::supervise::notify_registration_release();
2594                }
2595                return Ok(vec![control_error_frame(
2596                    &frame,
2597                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2598                        "invalid_hello"
2599                    } else {
2600                        forwarding_error_code(&err)
2601                    },
2602                    err.to_string(),
2603                )?]);
2604            }
2605            Vec::new()
2606        } else {
2607            // No sink means no forwarding endpoint, so nothing can be routed
2608            // ahead of the ack; it goes out as the reply.
2609            vec![hello_ack]
2610        };
2611
2612        // The module is routable now (or, without a sink, as routable as it
2613        // will ever be), so the catalog may list it. Publishing bumps the
2614        // catalog generation in the same registry critical section that makes
2615        // the module listed.
2616        if !self
2617            .registry
2618            .publish_registration(connection_id)
2619            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2620        {
2621            debug!(
2622                module_id = %registration.manifest.module_id,
2623                connection_id = connection_id.get(),
2624                "registration was removed before it could be published"
2625            );
2626        }
2627
2628        // Exposure over assumption: Concurrency's serde default is pinned to the
2629        // pre-field behavior (ModuleManaged), so a management surface that is
2630        // genuinely Serial and just never declared it inherits concurrent
2631        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2632        // turns "no module has been bitten yet" into the checkable claim "no
2633        // module is exposed" -- one read of the boot log instead of a fleet
2634        // audit. Detected from the raw HELLO bytes because the serde default
2635        // deliberately erases the absent/declared distinction from the type.
2636        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2637            info!(
2638                module_id = %registration.manifest.module_id,
2639                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2640            );
2641        }
2642
2643        self.apply_registration_capabilities(&registration);
2644
2645        self.supervisor
2646            .record_activity_registration(&registration.manifest.module_id);
2647
2648        info!(
2649            module_id = %registration.manifest.module_id,
2650            module_version = %registration.manifest.module_version,
2651            negotiated_ver,
2652            routable_provider = manifest_provides_routable_role(&registration.manifest),
2653            connection_id = connection_id.get(),
2654            "module registered"
2655        );
2656
2657        Ok(reply)
2658    }
2659
2660    /// Register a HELLO the swap gate admitted into the candidate slot of the
2661    /// registry and of forwarding, where it is reachable over its own
2662    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2663    /// nothing routes to it until the supervisor cuts over.
2664    ///
2665    /// Registry first (unpublished), then forwarding, then publication, the
2666    /// same order as an ordinary HELLO; a forwarding failure removes the
2667    /// registry entry again. The swap supervisor treats a published, ready
2668    /// candidate as registered and promotes it, and promotion needs the
2669    /// candidate's forwarding endpoint, so the candidate is published only
2670    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2671    /// recompute capability verdicts or run the capability deny census
2672    /// (`enforce_capability_denies`): both are about the module in the active
2673    /// slot, which `route.open` can target. A candidate sits in the candidate
2674    /// slot, so it neither provides capabilities to other modules nor can be
2675    /// routed to until promotion moves it into the active slot.
2676    #[allow(clippy::too_many_arguments)]
2677    fn register_swap_candidate(
2678        &self,
2679        connection_id: ConnectionId,
2680        sink: Option<crate::FrameSink>,
2681        frame: &Frame,
2682        manifest: ModuleManifest,
2683        negotiated_ver: u8,
2684        control_ops: Vec<String>,
2685        hello_ack: Frame,
2686    ) -> Result<Vec<Frame>, RouterError> {
2687        let module_id = manifest.module_id.clone();
2688        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2689            manifest,
2690            negotiated_ver,
2691            connection_id,
2692            control_ops,
2693        ) {
2694            Ok(registration) => registration,
2695            Err(RegistryError::DuplicateModuleId { module_id }) => {
2696                return Ok(vec![control_error_frame(
2697                    frame,
2698                    "duplicate_module_id",
2699                    format!(
2700                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2701                    ),
2702                )?])
2703            }
2704            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2705                return Ok(vec![control_error_frame(
2706                    frame,
2707                    "invalid_module_id",
2708                    err.to_string(),
2709                )?])
2710            }
2711            Err(err) => {
2712                return Ok(vec![control_error_frame(
2713                    frame,
2714                    "registry_error",
2715                    err.to_string(),
2716                )?])
2717            }
2718        };
2719        #[cfg(test)]
2720        self.hold_between_registry_and_forwarding();
2721        let reply = if let Some(sink) = sink {
2722            // Same ordering as an ordinary HELLO: the forwarding table queues
2723            // the HELLO_ACK before the candidate endpoint is inserted, because
2724            // a module exits if its first frame after HELLO is anything else.
2725            let concurrency = manifest_concurrency(&registration.manifest);
2726            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2727                connection_id,
2728                module_id.clone(),
2729                negotiated_ver,
2730                concurrency,
2731                sink,
2732                hello_ack,
2733            ) {
2734                if matches!(
2735                    self.deregister_connection(
2736                        connection_id,
2737                        RegistrationEndReason::RegistrationFailed,
2738                    ),
2739                    Ok(r) if !r.is_empty()
2740                ) {
2741                    crate::supervise::notify_registration_release();
2742                }
2743                return Ok(vec![control_error_frame(
2744                    frame,
2745                    forwarding_error_code(&err),
2746                    err.to_string(),
2747                )?]);
2748            }
2749            Vec::new()
2750        } else {
2751            vec![hello_ack]
2752        };
2753        if !self
2754            .registry
2755            .publish_registration(connection_id)
2756            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2757        {
2758            debug!(
2759                module_id = %module_id,
2760                connection_id = connection_id.get(),
2761                "swap candidate registration was removed before it could be published"
2762            );
2763        }
2764        self.supervisor.mark_swap_candidate_admitted(&module_id);
2765        info!(
2766            module_id = %module_id,
2767            module_version = %registration.manifest.module_version,
2768            negotiated_ver,
2769            ready = registration.ready,
2770            connection_id = connection_id.get(),
2771            "swap candidate registered; not routable until cutover"
2772        );
2773        Ok(reply)
2774    }
2775
2776    fn build_hello_ack(
2777        &self,
2778        frame: &Frame,
2779        negotiated_ver: u8,
2780        module_id: &str,
2781    ) -> Result<Frame, RouterError> {
2782        let ack = ModuleHelloAckBody {
2783            negotiated_ver,
2784            subc_ops: module_subc_ops(),
2785            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2786            storage: self
2787                .storage_config
2788                .as_ref()
2789                .map(|cfg| cfg.descriptor_for(module_id)),
2790            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2791        };
2792        let body = serde_json::to_vec(&ack).map_err(|err| {
2793            RouterError::backend(
2794                0,
2795                frame.header.corr,
2796                format!("failed to encode HELLO_ACK: {err}"),
2797            )
2798        })?;
2799
2800        Frame::build_with_version(
2801            negotiated_ver,
2802            FrameType::HelloAck,
2803            control_flags(),
2804            0,
2805            0,
2806            frame.header.corr,
2807            body,
2808        )
2809        .map_err(RouterError::FrameBuild)
2810    }
2811
2812    async fn handle_client_control_request(
2813        &self,
2814        ctx: &RouteCtx,
2815        frame: Frame,
2816        request: ClientControlRequest,
2817        route_open_timings: &mut Option<RouteOpenTimings>,
2818    ) -> Result<Vec<Frame>, RouterError> {
2819        if let Some(operation) = privileged_operation(&request) {
2820            let caller = self.resolve_caller(ctx.connection_id).await;
2821            let op = client_control_request_op(&request);
2822            let text = prompt_text(operation, &caller);
2823            match self.privileged_operator_prompt {
2824                PrivilegedOperatorPrompt::Log => {
2825                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2826                        target = ?operation, caller = %caller.description(), summary = %text,
2827                        "privileged request would prompt the operator");
2828                }
2829                PrivilegedOperatorPrompt::Enforce => {
2830                    let key =
2831                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2832                    let outcome = self
2833                        .forwarding
2834                        .operator_confirms()
2835                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2836                        .await;
2837                    if outcome.code != "confirmed" {
2838                        return Ok(vec![outcome.frame(&frame)]);
2839                    }
2840                }
2841            }
2842        }
2843        match request {
2844            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2845            ClientControlRequest::CatalogList { module_id } => {
2846                self.handle_catalog_list(frame, module_id)
2847            }
2848            ClientControlRequest::RouteOpen {
2849                target,
2850                identity,
2851                consumer_identity,
2852                consumer_capabilities,
2853                role_versions,
2854                admission_facts,
2855                scope,
2856            } => {
2857                self.handle_route_open(
2858                    ctx,
2859                    frame,
2860                    RouteOpenRequest {
2861                        target,
2862                        identity,
2863                        consumer_identity,
2864                        consumer_capabilities,
2865                        role_versions,
2866                        admission_facts,
2867                        scope,
2868                    },
2869                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2870                )
2871                .await
2872            }
2873            ClientControlRequest::RoutePoll {
2874                route_channel,
2875                route_epoch,
2876                kind,
2877            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2878            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2879            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2880                self.handle_supervisor_spawn_snapshot(frame)
2881            }
2882            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2883                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2884            }
2885            ClientControlRequest::SupervisorRestart {
2886                module_id,
2887                drain_timeout_ms,
2888            } => {
2889                self.log_supervisor_request_received(
2890                    ctx,
2891                    frame.header.corr,
2892                    ops::SUPERVISOR_RESTART,
2893                    Some(&module_id),
2894                    None,
2895                )?;
2896                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2897                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2898                    .await
2899            }
2900            ClientControlRequest::SupervisorSwap {
2901                module_id,
2902                ready_timeout_ms,
2903            } => {
2904                self.log_supervisor_request_received(
2905                    ctx,
2906                    frame.header.corr,
2907                    ops::SUPERVISOR_SWAP,
2908                    Some(&module_id),
2909                    None,
2910                )?;
2911                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2912                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2913                    .await
2914            }
2915            ClientControlRequest::SupervisorReload { module_id } => {
2916                self.log_supervisor_request_received(
2917                    ctx,
2918                    frame.header.corr,
2919                    ops::SUPERVISOR_RELOAD,
2920                    Some(&module_id),
2921                    None,
2922                )?;
2923                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2924                self.handle_supervisor_reload(frame, module_id, requester)
2925                    .await
2926            }
2927            ClientControlRequest::SupervisorRescan { preview } => {
2928                if !preview {
2929                    self.log_supervisor_request_received(
2930                        ctx,
2931                        frame.header.corr,
2932                        ops::SUPERVISOR_RESCAN,
2933                        None,
2934                        None,
2935                    )?;
2936                }
2937                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2938                self.handle_supervisor_rescan(
2939                    frame,
2940                    if preview {
2941                        RescanMode::Preview
2942                    } else {
2943                        RescanMode::Execute
2944                    },
2945                    requester,
2946                )
2947                .await
2948            }
2949            ClientControlRequest::SupervisorRescanChecked {
2950                preview,
2951                candidate,
2952                expect,
2953            } => {
2954                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2955                    .await
2956            }
2957            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2958                self.log_supervisor_request_received(
2959                    ctx,
2960                    frame.header.corr,
2961                    ops::SUPERVISOR_RELEASE_RESERVED,
2962                    Some(&module_id),
2963                    None,
2964                )?;
2965                self.handle_supervisor_release_reserved(frame, module_id)
2966                    .await
2967            }
2968            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2969                self.log_supervisor_request_received(
2970                    ctx,
2971                    frame.header.corr,
2972                    ops::SUPERVISOR_SET_ENABLED,
2973                    Some(&module_id),
2974                    Some(enabled),
2975                )?;
2976                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2977                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2978                    .await
2979            }
2980            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2981                self.handle_supervisor_health_probe(frame, module_id).await
2982            }
2983            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2984            ClientControlRequest::SupervisorRoutes { module_id } => {
2985                self.handle_supervisor_routes(frame, module_id)
2986            }
2987            ClientControlRequest::SupervisorProvenance { module_id } => {
2988                self.handle_supervisor_provenance(frame, module_id).await
2989            }
2990            ClientControlRequest::SupervisorStderrTail {
2991                module_id,
2992                max_lines,
2993                max_bytes,
2994            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2995            ClientControlRequest::SupervisorTerminals { module_id } => {
2996                self.handle_supervisor_terminals(frame, module_id).await
2997            }
2998        }
2999    }
3000
3001    fn log_supervisor_request_received(
3002        &self,
3003        ctx: &RouteCtx,
3004        corr: u64,
3005        op: &'static str,
3006        module_id: Option<&str>,
3007        enabled: Option<bool>,
3008    ) -> Result<(), RouterError> {
3009        let caller = self.supervisor_requester(ctx, corr)?;
3010        let caller = principal_label(&caller);
3011        // `Option` fields are recorded only when present, so a request with no
3012        // module id (rescan) or no enabled flag simply omits that field.
3013        info!(
3014            target: "control",
3015            op,
3016            module_id,
3017            enabled,
3018            connection_id = ctx.connection_id.get(),
3019            caller = %caller,
3020            "supervisor request received"
3021        );
3022        Ok(())
3023    }
3024
3025    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3026        Ok(self
3027            .registry
3028            .get_module_by_connection(ctx.connection_id)
3029            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3030            .map(|registration| Principal::Reserved {
3031                module_id: registration.manifest.module_id,
3032            })
3033            .unwrap_or(Principal::Direct))
3034    }
3035
3036    fn handle_module_control_request(
3037        &self,
3038        connection_id: ConnectionId,
3039        frame: Frame,
3040        request: ModuleControlRequestFromModule,
3041    ) -> Result<Vec<Frame>, RouterError> {
3042        match request {
3043            ModuleControlRequestFromModule::CatalogUpdate {
3044                provides,
3045                capabilities,
3046                ready,
3047            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3048            ModuleControlRequestFromModule::LiveRoots {} => {
3049                let registered = self
3050                    .registry
3051                    .get_module_by_connection(connection_id)
3052                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3053                let Some(registration) = registered else {
3054                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3055                };
3056                let response = self
3057                    .forwarding
3058                    .live_roots(&registration.manifest.module_id)
3059                    .map_err(RouterError::Forwarding)?;
3060                Ok(vec![control_response_body_frame(
3061                    &frame,
3062                    &response,
3063                    "ModuleControlResponseToModule::LiveRoots",
3064                )?])
3065            }
3066            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3067                self.handle_scope_sync(connection_id, frame, generation, scopes)
3068            }
3069            ModuleControlRequestFromModule::ScopeApply {
3070                generation,
3071                upsert,
3072                end,
3073            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3074            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3075                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3076            }
3077        }
3078    }
3079
3080    fn handle_scope_served_by(
3081        &self,
3082        connection_id: ConnectionId,
3083        frame: Frame,
3084    ) -> Result<Vec<Frame>, RouterError> {
3085        let registered = self
3086            .registry
3087            .get_module_by_connection(connection_id)
3088            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3089        let Some(registered) = registered else {
3090            return Ok(vec![control_error_frame(
3091                &frame,
3092                "not_registered",
3093                "scope.served_by requires an active module registration owned by this connection",
3094            )?]);
3095        };
3096        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3097        {
3098            Ok(request) => request,
3099            Err(err) => {
3100                return Ok(vec![control_error_frame(
3101                    &frame,
3102                    "invalid_control_body",
3103                    format!("malformed scope.served_by body: {err}"),
3104                )?])
3105            }
3106        };
3107        let answer = self
3108            .scopes
3109            .read()
3110            .unwrap_or_else(|poisoned| poisoned.into_inner())
3111            .served_by(
3112                &registered.manifest.module_id,
3113                &request.scope_ref,
3114                request.scope_epoch,
3115                &request.module_id,
3116            );
3117        let reply = subc_control::ScopeServedByReply::new(
3118            answer,
3119            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3120        );
3121        Ok(vec![control_response_body_frame(
3122            &frame,
3123            &reply,
3124            "ScopeServedByReply",
3125        )?])
3126    }
3127
3128    fn handle_operator_confirm(
3129        &self,
3130        ctx: &RouteCtx,
3131        frame: Frame,
3132    ) -> Result<Vec<Frame>, RouterError> {
3133        use crate::operator_confirm::{audit, Outcome};
3134        let registration = self
3135            .registry
3136            .get_module_by_connection(ctx.connection_id)
3137            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3138        let Some(registration) = registration else {
3139            let outcome = Outcome::refusal("not_registered");
3140            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3141            return Ok(vec![outcome.frame(&frame)]);
3142        };
3143        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3144            Ok(request) => request,
3145            Err(_) => {
3146                let outcome = Outcome::refusal("invalid_control_body");
3147                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3148                return Ok(vec![outcome.frame(&frame)]);
3149            }
3150        };
3151        let module_id = registration.manifest.module_id;
3152        // Read the launch nonce (under its own lock) before taking the forwarding
3153        // table's lock below: holding forwarding while waiting on another daemon
3154        // lock risks a lock-order deadlock with paths that take them the other way.
3155        let nonce = self
3156            .hello_launch_nonces
3157            .lock()
3158            .unwrap_or_else(|p| p.into_inner())
3159            .nonce(ctx.connection_id)
3160            .map(str::to_owned);
3161        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3162            self.supervisor
3163                .spawned_consumer_authorized(&module_id, nonce)
3164        });
3165        let confirms = self.forwarding.operator_confirms();
3166        self.forwarding
3167            .with_operator_route(
3168                ctx.connection_id,
3169                request.route_channel,
3170                request.route_epoch,
3171                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3172            )
3173            .map_err(RouterError::Forwarding)
3174    }
3175
3176    /// Resolve folders before taking the scope-table lock: a slow filesystem
3177    /// must not hold up admissions for unrelated scoped routes.
3178    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3179        let mut scopes = ScopeRecords::from(scopes);
3180        for (index, record) in scopes.records.iter_mut().enumerate() {
3181            if let Some(refusal) = Self::canonicalize_workdir(record) {
3182                scopes.refusals.insert(index, refusal);
3183            }
3184        }
3185        scopes
3186    }
3187
3188    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3189        let workdir = record.workdir.as_mut()?;
3190        let path = std::path::Path::new(workdir);
3191        if !path.is_absolute() {
3192            return Some((
3193                error_codes::INVALID_WORKDIR,
3194                "workdir must be absolute".to_string(),
3195            ));
3196        }
3197        match ProjectRootId::from_path_allowing_missing(path) {
3198            Ok(root) => match root.as_path().to_str() {
3199                Some(canonical) => {
3200                    *workdir = canonical.to_string();
3201                    None
3202                }
3203                None => Some((
3204                    error_codes::INVALID_WORKDIR,
3205                    "canonical workdir is not UTF-8".to_string(),
3206                )),
3207            },
3208            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3209        }
3210    }
3211
3212    /// `scope.sync`: the owner is the module registered on this connection.
3213    /// A connection with no registration (every client connection, `direct`
3214    /// included) is refused `not_registered` before the table is consulted.
3215    fn handle_scope_sync(
3216        &self,
3217        connection_id: ConnectionId,
3218        frame: Frame,
3219        generation: u64,
3220        scopes: Vec<ScopeRecord>,
3221    ) -> Result<Vec<Frame>, RouterError> {
3222        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3223    }
3224
3225    fn handle_scope_change(
3226        &self,
3227        connection_id: ConnectionId,
3228        frame: Frame,
3229        generation: u64,
3230        scopes: Vec<ScopeRecord>,
3231        end: Option<Vec<ScopeEnd>>,
3232    ) -> Result<Vec<Frame>, RouterError> {
3233        let is_apply = end.is_some();
3234        let Some(registration) = self
3235            .registry
3236            .get_module_by_connection(connection_id)
3237            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3238        else {
3239            return Ok(vec![control_error_frame(
3240                &frame,
3241                "not_registered",
3242                "scope.sync and scope.apply require an active module registration owned by this connection",
3243            )?]);
3244        };
3245        let owner = registration.manifest.module_id;
3246        let scopes = Self::prepare_scope_records(scopes);
3247        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3248        let is_current_launch = |connection: ConnectionId| {
3249            self.hello_launch_nonces
3250                .lock()
3251                .unwrap_or_else(|poisoned| poisoned.into_inner())
3252                .presented(connection, current_nonce.as_deref())
3253        };
3254        // Lock order is the scope table, then the forwarding table: the new
3255        // tags are published, and the routes the change closes are selected,
3256        // while the scope table is still write-locked, so no admission can read
3257        // a record whose tag is not yet published.
3258        let mut table = self
3259            .scopes
3260            .write()
3261            .unwrap_or_else(|poisoned| poisoned.into_inner());
3262        let outcome = match end {
3263            Some(end) => table.apply(
3264                &owner,
3265                connection_id,
3266                is_current_launch,
3267                generation,
3268                scopes,
3269                end,
3270            ),
3271            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3272        };
3273        let drained = match &outcome {
3274            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3275            Err(_) => Vec::new(),
3276        };
3277        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3278        drop(table);
3279        if has_deadlines {
3280            self.scope_deadline_added.notify_one();
3281        }
3282        match outcome {
3283            Ok(applied) => {
3284                let counts = ScopeOutcomeCounts::of(&applied.results);
3285                info!(
3286                    owner = %owner,
3287                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3288                    generation,
3289                    records = applied.results.len(),
3290                    created = counts.created,
3291                    replaced = counts.replaced,
3292                    updated = counts.updated,
3293                    unchanged = counts.unchanged,
3294                    refused = counts.refused,
3295                    ended = applied.ended.len(),
3296                    tag_changes = applied.tag_changes.len(),
3297                    routes_closed = drained.len(),
3298                    "scope change accepted"
3299                );
3300                // An accepted scope change can still refuse individual records, and the
3301                // owner is the only party that sees the reply. Name them here so
3302                // an operator can tell a refused session from a missing one
3303                // without the owner's logs. Capped so a call that refuses
3304                // thousands cannot flood the log; the count above is complete.
3305                for refused in applied
3306                    .results
3307                    .iter()
3308                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3309                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3310                {
3311                    warn!(
3312                        owner = %owner,
3313                        generation,
3314                        scope_ref = %refused.scope_ref,
3315                        scope_epoch = refused.scope_epoch,
3316                        code = refused.code.as_deref().unwrap_or(""),
3317                        "scope record refused"
3318                    );
3319                }
3320                self.close_scope_drained_routes(drained);
3321                let response = if is_apply {
3322                    ModuleControlResponseToModule::ScopeApply {
3323                        generation,
3324                        results: applied.results,
3325                        end_results: applied.end_results,
3326                        ended: applied.ended,
3327                    }
3328                } else {
3329                    ModuleControlResponseToModule::ScopeSync {
3330                        generation,
3331                        results: applied.results,
3332                        ended: applied.ended,
3333                    }
3334                };
3335                Ok(vec![control_response_body_frame(
3336                    &frame,
3337                    &response,
3338                    "scope change response",
3339                )?])
3340            }
3341            Err(refusal) => {
3342                info!(
3343                    owner = %owner,
3344                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3345                    generation,
3346                    code = refusal.code,
3347                    "scope change refused"
3348                );
3349                Ok(vec![control_error_frame(
3350                    &frame,
3351                    refusal.code,
3352                    refusal.message,
3353                )?])
3354            }
3355        }
3356    }
3357
3358    /// Tell both ends of each route a scope change closed. The module gets a
3359    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3360    /// the client's route handle. The client also gets `route.closed` with the
3361    /// scope reason, one push per module and reason, so it can tell a revoked
3362    /// route from an ordinary close and not reopen it.
3363    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3364        if drained.is_empty() {
3365            return;
3366        }
3367        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3368            BTreeMap::new();
3369        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3370        for route in drained {
3371            warn!(
3372                module_id = %route.module_id,
3373                reason = ?route.reason,
3374                client_connection_id = route.client.connection_id.get(),
3375                route_channel = route.client.channel,
3376                "closing route because its scope changed"
3377            );
3378            pushes
3379                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3380                .or_insert_with(|| (route.reason, Vec::new()))
3381                .1
3382                .push(EndpointRoute {
3383                    goodbye_target: route.client.clone(),
3384                    principal: Principal::Unverified,
3385                    bound_at: Instant::now(),
3386                    draining: false,
3387                    drain_reason: None,
3388                });
3389            goodbyes.push(route.module);
3390            goodbyes.push(route.client);
3391        }
3392        for ((module_id, _), (reason, routes)) in pushes {
3393            send_route_control_pushes(
3394                &self.forwarding,
3395                routes,
3396                ClientControlPush::RouteClosed {
3397                    module_id,
3398                    channels: Vec::new(),
3399                    reason,
3400                    drained: false,
3401                    abandoned: 0,
3402                    excluded_subscriptions: 0,
3403                    terminal: Some(false),
3404                },
3405            );
3406        }
3407        self.emit_route_goodbyes(goodbyes);
3408    }
3409
3410    /// `scope.describe`: any registered module may read any scope, because a
3411    /// provider must read the scope a route it serves is stamped with.
3412    fn handle_scope_describe(
3413        &self,
3414        connection_id: ConnectionId,
3415        frame: Frame,
3416        owner: Principal,
3417        scope_ref: String,
3418    ) -> Result<Vec<Frame>, RouterError> {
3419        let registered = self
3420            .registry
3421            .get_module_by_connection(connection_id)
3422            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3423        let Some(registered) = registered else {
3424            return Ok(vec![control_error_frame(
3425                &frame,
3426                "not_registered",
3427                "scope.describe requires an active module registration owned by this connection",
3428            )?]);
3429        };
3430        let mut description = self
3431            .scopes
3432            .read()
3433            .unwrap_or_else(|poisoned| poisoned.into_inner())
3434            .describe(&owner, &scope_ref);
3435        // Describe readers also decode strict ScopeAttributes. Attribution is
3436        // safe to omit unless the requesting module declares it understands it.
3437        if !registered
3438            .manifest
3439            .capabilities
3440            .as_ref()
3441            .is_some_and(|caps| {
3442                caps.provides
3443                    .iter()
3444                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3445            })
3446        {
3447            if let Some(stamp) = description.stamp.as_mut() {
3448                stamp.attributes.on_behalf_of_agent = None;
3449            }
3450        }
3451        if !registered
3452            .manifest
3453            .capabilities
3454            .as_ref()
3455            .is_some_and(|caps| {
3456                caps.provides
3457                    .iter()
3458                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3459            })
3460        {
3461            if let Some(stamp) = description.stamp.as_mut() {
3462                stamp.attributes.flow_version = None;
3463                stamp.attributes.flow_code_hash = None;
3464                stamp.attributes.flow_install_id = None;
3465            }
3466        }
3467        let owner_configured = match &owner {
3468            // Ask whether the owner is configured (`is_configured`), not
3469            // whether it is on the roster (`get(..).is_some()`): a supervised
3470            // module's process can register and describe a scope before the
3471            // supervisor has put it on the roster.
3472            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3473            _ => false,
3474        };
3475        let response = ModuleControlResponseToModule::ScopeDescribe {
3476            status: description.status,
3477            scope_epoch: description.scope_epoch,
3478            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3479            owner_synced: description.owner_synced,
3480            owner_configured,
3481            scope: description.stamp,
3482        };
3483        Ok(vec![control_response_body_frame(
3484            &frame,
3485            &response,
3486            "ModuleControlResponseToModule::ScopeDescribe",
3487        )?])
3488    }
3489
3490    fn handle_catalog_update(
3491        &self,
3492        connection_id: ConnectionId,
3493        frame: Frame,
3494        provides: Vec<ProviderRole>,
3495        capabilities: Option<CapabilityDeclarations>,
3496        ready: Option<bool>,
3497    ) -> Result<Vec<Frame>, RouterError> {
3498        self.refresh_capability_requirements();
3499        let Some(registration) = self
3500            .registry
3501            .get_module_by_connection(connection_id)
3502            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3503        else {
3504            return Ok(vec![control_error_frame(
3505                &frame,
3506                "not_registered",
3507                "catalog.update requires an active module registration owned by this connection",
3508            )?]);
3509        };
3510
3511        if let Some(message) =
3512            catalog_update_frozen_field_message(&registration.manifest, &provides)
3513        {
3514            return Ok(vec![control_error_frame(
3515                &frame,
3516                "catalog_update_frozen_field",
3517                message,
3518            )?]);
3519        }
3520
3521        let mut candidate = registration.manifest.clone();
3522        candidate.provides = provides.clone();
3523        candidate.capabilities = capabilities
3524            .clone()
3525            .or_else(|| registration.manifest.capabilities.clone());
3526        if let Some(rich_ui) = &candidate.rich_ui {
3527            if let Err(err) = rich_ui.validate() {
3528                return Ok(vec![control_error_frame(
3529                    &frame,
3530                    "invalid_manifest",
3531                    err.to_string(),
3532                )?]);
3533            }
3534        }
3535        if let Err(err) = candidate.validate_capability_grammar() {
3536            return Ok(vec![control_error_frame(
3537                &frame,
3538                "invalid_capability_grammar",
3539                err.to_string(),
3540            )?]);
3541        }
3542
3543        // Updates must honor the same reserved owner as initial registration;
3544        // otherwise an empty HELLO could acquire the claim after admission.
3545        let mut conflicts = self
3546            .capability_evaluator
3547            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3548        if let Some(conflict) = conflicts.first() {
3549            let message = format!(
3550                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3551                conflict.capability, conflict.claimants[0], candidate.module_id
3552            );
3553            for conflict in &mut conflicts {
3554                conflict.source = DuplicateClaimSource::CatalogUpdate;
3555            }
3556            log_duplicate_claim_events(conflicts);
3557            return Ok(vec![control_error_frame(
3558                &frame,
3559                "reserved_capability",
3560                message,
3561            )?]);
3562        }
3563
3564        let updated = self
3565            .registry
3566            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3567            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3568        if updated.is_none() {
3569            return Ok(vec![control_error_frame(
3570                &frame,
3571                "not_registered",
3572                "catalog.update requires an active module registration owned by this connection",
3573            )?]);
3574        }
3575        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3576            log_duplicate_claim_events(
3577                self.capability_evaluator
3578                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3579            );
3580        }
3581        if capability_census_trigger(
3582            registration.manifest.capabilities.as_ref(),
3583            updated
3584                .as_ref()
3585                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3586        ) {
3587            self.enforce_capability_denies();
3588        }
3589        self.refresh_capability_requirements();
3590
3591        let response = ModuleControlResponseToModule::CatalogUpdate {};
3592        control_response_body_frame(
3593            &frame,
3594            &response,
3595            "ModuleControlResponseToModule::CatalogUpdate",
3596        )
3597        .map(|frame| vec![frame])
3598    }
3599
3600    pub(crate) fn with_activity_ledger_config(
3601        mut self,
3602        run_dir: &Path,
3603        incarnation: String,
3604        config: crate::activity::ActivityConfig,
3605    ) -> Self {
3606        self.activity = Some(crate::activity::ActivityLedger::open_config(
3607            run_dir.join("activity.jsonl"),
3608            incarnation,
3609            config,
3610        ));
3611        self.supervisor.set_activity(self.activity.clone());
3612        self
3613    }
3614
3615    pub(crate) async fn configure_activity(
3616        &self,
3617        config: crate::activity::ActivityConfig,
3618    ) -> Result<(), &'static str> {
3619        match &self.activity {
3620            Some(ledger) => ledger.configure(config).await,
3621            None => Ok(()),
3622        }
3623    }
3624
3625    fn cancel_activity_subscription(
3626        &self,
3627        ctx: &RouteCtx,
3628        frame: &Frame,
3629    ) -> Result<bool, RouterError> {
3630        let task = self
3631            .activity_streams
3632            .lock()
3633            .unwrap_or_else(|p| p.into_inner())
3634            .remove(&(ctx.connection_id, frame.header.corr));
3635        let Some(task) = task else {
3636            return Ok(false);
3637        };
3638        task.abort();
3639        let end = Frame::build_with_version(
3640            response_version(frame),
3641            FrameType::StreamEnd,
3642            control_flags(),
3643            0,
3644            0,
3645            frame.header.corr,
3646            Vec::new(),
3647        )
3648        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3649        let sink = ctx.egress.clone();
3650        tokio::spawn(async move {
3651            let _ = sink.send(end).await;
3652        });
3653        Ok(true)
3654    }
3655
3656    async fn handle_activity_read(
3657        &self,
3658        ctx: &RouteCtx,
3659        frame: Frame,
3660    ) -> Result<Vec<Frame>, RouterError> {
3661        use subc_control::{
3662            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3663            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3664        };
3665        #[derive(Deserialize)]
3666        struct Identity {
3667            #[serde(default)]
3668            consumer_identity: Option<ConsumerIdentity>,
3669        }
3670        // A registered module connection already identifies its caller. Ignore
3671        // consumer_identity there so a module cannot borrow a mirror's identity.
3672        // Only separate consumer connections use the Direct/nonce trust boundary.
3673        let principal = match self
3674            .forwarding
3675            .module_id_for_connection(ctx.connection_id)
3676            .map_err(RouterError::Forwarding)?
3677        {
3678            Some(module_id) => Ok(Principal::Reserved { module_id }),
3679            None => {
3680                let identity: Identity = match serde_json::from_slice(&frame.body) {
3681                    Ok(identity) => identity,
3682                    Err(_) => {
3683                        return Ok(vec![control_error_frame(
3684                            &frame,
3685                            ACTIVITY_MALFORMED,
3686                            "malformed activity read",
3687                        )?])
3688                    }
3689                };
3690                self.route_open_principal(&frame, identity.consumer_identity)?
3691            }
3692        };
3693        let admitted = matches!(principal, Ok(Principal::Direct))
3694            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3695        if !admitted {
3696            return Ok(vec![control_error_frame(
3697                &frame,
3698                ACTIVITY_READ_FORBIDDEN,
3699                "activity reads require an operator or trusted mirror principal",
3700            )?]);
3701        }
3702        let Some(ledger) = &self.activity else {
3703            return Ok(vec![control_error_frame(
3704                &frame,
3705                ACTIVITY_UNAVAILABLE,
3706                "activity ledger unavailable",
3707            )?]);
3708        };
3709        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3710            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3711        {
3712            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3713                Ok(request) => ledger.query(request).await,
3714                Err(_) => Err(ACTIVITY_MALFORMED),
3715            };
3716            return match result {
3717                Ok(reply) => Ok(vec![control_response_body_frame(
3718                    &frame,
3719                    &reply,
3720                    ops::ACTIVITY_QUERY,
3721                )?]),
3722                Err(code) => Ok(vec![control_error_frame(
3723                    &frame,
3724                    code,
3725                    "activity query refused",
3726                )?]),
3727            };
3728        }
3729        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3730            Ok(request) => request,
3731            Err(_) => {
3732                return Ok(vec![control_error_frame(
3733                    &frame,
3734                    ACTIVITY_MALFORMED,
3735                    "malformed activity subscription",
3736                )?])
3737            }
3738        };
3739        let mut subscription = match ledger.subscribe(request.since).await {
3740            Ok(subscription) => subscription,
3741            Err(code) => {
3742                return Ok(vec![control_error_frame(
3743                    &frame,
3744                    code,
3745                    "activity subscription refused",
3746                )?])
3747            }
3748        };
3749        let key = (ctx.connection_id, frame.header.corr);
3750        let sink = ctx.egress.clone();
3751        let mut streams = self
3752            .activity_streams
3753            .lock()
3754            .unwrap_or_else(|p| p.into_inner());
3755        streams.retain(|_, task| !task.is_finished());
3756        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3757            return Ok(vec![control_error_frame(
3758                &frame,
3759                "duplicate_subscription",
3760                "activity subscription correlation id already in use",
3761            )?]);
3762        }
3763        let task = tokio::spawn(async move {
3764            loop {
3765                let entry = if let Some(entry) = subscription.replay.pop_front() {
3766                    Some(entry)
3767                } else {
3768                    subscription.receiver.recv().await
3769                };
3770                let Some(entry) = entry else {
3771                    break;
3772                };
3773                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3774                    return;
3775                };
3776                let Ok(next) = Frame::build_with_version(
3777                    response_version(&frame),
3778                    FrameType::StreamData,
3779                    control_flags(),
3780                    0,
3781                    0,
3782                    frame.header.corr,
3783                    body,
3784                ) else {
3785                    return;
3786                };
3787                if sink.send(next).await.is_err() {
3788                    return;
3789                }
3790            }
3791            // Dropping a full subscriber's sender drains its accepted entries
3792            // before the terminal error, preserving their order on the wire.
3793            if let Some(code) = subscription.terminal_code() {
3794                if let Ok(end) =
3795                    control_error_frame(&frame, code, "activity subscriber fell behind")
3796                {
3797                    let _ = sink.send(end).await;
3798                }
3799            }
3800        });
3801        streams.insert(key, task.abort_handle());
3802        Ok(Vec::new())
3803    }
3804
3805    async fn handle_activity_record(
3806        &self,
3807        ctx: &RouteCtx,
3808        frame: Frame,
3809    ) -> Result<Vec<Frame>, RouterError> {
3810        use subc_control::{
3811            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3812            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3813        };
3814        let result = async {
3815            let module = self
3816                .forwarding
3817                .module_id_for_connection(ctx.connection_id)
3818                .ok()
3819                .flatten()
3820                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3821            let request: ActivityRecordRequest =
3822                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3823            crate::activity::validate(&module, &request)?;
3824            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3825            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3826            let at_ms = std::time::SystemTime::now()
3827                .duration_since(std::time::UNIX_EPOCH)
3828                .unwrap_or_default()
3829                .as_millis() as u64;
3830            ledger.record(module, request, actor, at_ms).await
3831        }
3832        .await;
3833        match result {
3834            Ok(entry) => Ok(vec![control_response_body_frame(
3835                &frame,
3836                &ActivityRecordResponse { seq: entry.seq },
3837                "activity.record",
3838            )?]),
3839            Err(code) => Ok(vec![control_error_frame(
3840                &frame,
3841                code,
3842                "activity recording refused",
3843            )?]),
3844        }
3845    }
3846
3847    fn activity_actor(
3848        &self,
3849        connection: ConnectionId,
3850        module: &str,
3851        on: Option<&subc_control::ActivityOn>,
3852    ) -> Result<subc_control::ActivityActor, &'static str> {
3853        use crate::forwarding::{DataRoute, DataRouteState};
3854        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3855        let principal = |id: &str| Principal::Reserved {
3856            module_id: id.to_owned(),
3857        };
3858        let mut actor = ActivityActor {
3859            principal: principal(module),
3860            scope_owner: None,
3861            agent_id: None,
3862            run_id: None,
3863            flow_id: None,
3864            project_root: None,
3865            workdir: None,
3866            project_id: None,
3867        };
3868        let scope = match on {
3869            None => return Ok(actor),
3870            Some(ActivityOn::Scope(scope)) => {
3871                let selector = subc_protocol::scope::ScopeSelector::new(
3872                    principal(module),
3873                    scope.scope_ref.clone(),
3874                    Some(scope.epoch),
3875                );
3876                self.scopes
3877                    .read()
3878                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3879                    .attribute(&principal(module), module, &selector, true)
3880                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3881            }
3882            Some(ActivityOn::Route(route)) => {
3883                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3884                    .forwarding
3885                    .lookup_data_route(connection, route.channel, route.epoch)
3886                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3887                else {
3888                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3889                };
3890                if binding.module_id != module {
3891                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3892                }
3893                actor.principal = binding.principal.clone();
3894                actor.project_root = binding
3895                    .project_root
3896                    .as_ref()
3897                    .map(|root| root.as_path().to_path_buf());
3898                let Some(bound) = &binding.scope else {
3899                    return Ok(actor);
3900                };
3901                let owner = principal(&bound.owner);
3902                let selector = subc_protocol::scope::ScopeSelector::new(
3903                    owner.clone(),
3904                    bound.scope_ref.clone(),
3905                    Some(bound.tag.scope_epoch),
3906                );
3907                let admitted = self
3908                    .scopes
3909                    .read()
3910                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3911                    .attribute(&owner, module, &selector, true)
3912                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3913                if admitted.tag != bound.tag {
3914                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3915                }
3916                admitted
3917            }
3918        };
3919        actor.scope_owner = Some(scope.stamp.owner);
3920        actor.agent_id = scope.stamp.attributes.agent_id;
3921        actor.run_id = scope.stamp.attributes.run_id;
3922        actor.flow_id = scope.stamp.attributes.flow_id;
3923        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3924        actor.project_id = scope.stamp.project_id;
3925        Ok(actor)
3926    }
3927
3928    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3929        self.refresh_capability_requirements();
3930        // A bare connection count is ambiguous between many clients holding a
3931        // route each and one client accumulating hundreds, so publish the
3932        // concentration alongside it. Route state is best-effort here: a
3933        // diagnostic endpoint must still answer if the forwarding lock is
3934        // contended.
3935        let mut counters = self.counters.snapshot();
3936        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3937            obj.insert(
3938                "activity_unparseable_lines".into(),
3939                ledger.read_errors().into(),
3940            );
3941            obj.insert(
3942                "activity_lifecycle_append_failures".into(),
3943                ledger.lifecycle_failures().into(),
3944            );
3945            obj.insert(
3946                "activity_started_digest_skips".into(),
3947                ledger.lifecycle_digest_skips().into(),
3948            );
3949        }
3950        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3951            self.forwarding.client_route_concentration(),
3952            counters.as_object_mut(),
3953        ) {
3954            obj.insert(
3955                "client_connections_with_routes".into(),
3956                connections_with_routes.into(),
3957            );
3958            obj.insert("max_routes_on_one_connection".into(), max.into());
3959        }
3960        // A module that is being fast-refused and a module that is fine look
3961        // identical from a client that retries and succeeds, so name the open
3962        // breakers here. This rides the existing free-form counters object
3963        // rather than a new wire field, so no sibling that deserializes
3964        // `ServerDescribe` has to be rebuilt to keep reading it.
3965        if let (Some(open_breakers), Some(obj)) = (
3966            self.route_bind_breakers.open_snapshot(),
3967            counters.as_object_mut(),
3968        ) {
3969            obj.insert("route_bind_breakers_open".into(), open_breakers);
3970        }
3971        let response = ClientControlResponse::ServerDescribe {
3972            protocol_ver: PROTOCOL_VERSION,
3973            subc_ops: subc_ops(),
3974            capabilities: self.describe_capabilities(),
3975            connected_clients: self.connected_clients.count(),
3976            counters: Some(counters),
3977            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3978            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3979            capability_requirements: self.capability_requirement_statuses(),
3980            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3981            config_path: self
3982                .rescan
3983                .as_ref()
3984                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3985        };
3986        Ok(vec![control_response_body_frame(
3987            &frame,
3988            &response,
3989            "ClientControlResponse::ServerDescribe",
3990        )?])
3991    }
3992
3993    fn handle_catalog_list(
3994        &self,
3995        frame: Frame,
3996        module_id: Option<String>,
3997    ) -> Result<Vec<Frame>, RouterError> {
3998        let (generation, modules) = self
3999            .registry
4000            .list_modules_with_nonce_verification()
4001            .map_err(|err| {
4002                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4003            })?;
4004        let entries = modules
4005            .into_iter()
4006            .filter(|(registration, _)| {
4007                module_id
4008                    .as_deref()
4009                    .map(|wanted| registration.manifest.module_id == wanted)
4010                    .unwrap_or(true)
4011            })
4012            .map(|(registration, nonce_verified)| {
4013                let not_ready = self.not_ready_reason(&registration);
4014                let roles = registration.manifest.provides;
4015                CatalogEntry::new(
4016                    registration.manifest.module_id,
4017                    roles,
4018                    registration.control_ops,
4019                )
4020                .with_ready(not_ready.is_none())
4021                .with_not_ready(not_ready)
4022                .with_module_version(Some(registration.manifest.module_version))
4023                .with_capabilities(registration.manifest.capabilities)
4024                .with_self_signals(registration.manifest.self_signals)
4025                .with_events(registration.manifest.events)
4026                .with_operation_delivery(registration.manifest.operation_delivery)
4027                .with_operation_payload(registration.manifest.operation_payload)
4028                .with_rich_ui(registration.manifest.rich_ui)
4029                .with_role_declarations(registration.manifest.role_declarations)
4030                .with_nonce_verified(nonce_verified)
4031            })
4032            .collect();
4033        let response = ClientControlResponse::CatalogList {
4034            generation,
4035            modules: entries,
4036            subc_ops: subc_ops(),
4037        };
4038        Ok(vec![control_response_body_frame(
4039            &frame,
4040            &response,
4041            "ClientControlResponse::CatalogList",
4042        )?])
4043    }
4044
4045    /// The principal a route.open's `consumer_identity` attests: `Direct`
4046    /// when it presents none, `Reserved` when its launch nonce verifies, and
4047    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4048    /// can run before any refusal check without changing which check refuses.
4049    fn attest_route_open_consumer(
4050        &self,
4051        consumer_identity: Option<&ConsumerIdentity>,
4052    ) -> Option<Principal> {
4053        let Some(consumer_identity) = consumer_identity else {
4054            return Some(Principal::Direct);
4055        };
4056        self.supervisor
4057            .spawned_consumer_authorized(
4058                &consumer_identity.module_id,
4059                &consumer_identity.launch_nonce,
4060            )
4061            .then(|| Principal::Reserved {
4062                module_id: consumer_identity.module_id.clone(),
4063            })
4064    }
4065
4066    /// The caller fields for a refusal that only has the raw request frame,
4067    /// such as the connection-level admission refusal sent before the request
4068    /// reaches [`Self::handle_route_open`].
4069    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4070        let Ok(ClientControlRequest::RouteOpen {
4071            identity,
4072            consumer_identity,
4073            scope,
4074            ..
4075        }) = parse_client_control_request(&frame.body)
4076        else {
4077            return RouteOpenCaller::unknown();
4078        };
4079        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4080        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4081    }
4082
4083    fn route_open_principal(
4084        &self,
4085        frame: &Frame,
4086        consumer_identity: Option<ConsumerIdentity>,
4087    ) -> Result<Result<Principal, Frame>, RouterError> {
4088        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4089        self.route_open_principal_attested(frame, consumer_identity, attested)
4090    }
4091
4092    /// [`Self::route_open_principal`] with `attested` already taken by
4093    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4094    /// open checks the launch nonce once.
4095    fn route_open_principal_attested(
4096        &self,
4097        frame: &Frame,
4098        consumer_identity: Option<ConsumerIdentity>,
4099        attested: Option<Principal>,
4100    ) -> Result<Result<Principal, Frame>, RouterError> {
4101        if let Some(principal) = attested {
4102            return Ok(Ok(principal));
4103        }
4104        let Some(consumer_identity) = consumer_identity else {
4105            return Ok(Ok(Principal::Direct));
4106        };
4107
4108        Ok(Err(control_error_frame(
4109            frame,
4110            "bad_consumer_identity",
4111            format!(
4112                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4113                consumer_identity.module_id
4114            ),
4115        )?))
4116    }
4117
4118    /// Ordinary `route.open` refusals go through here; admission and breaker
4119    /// refusals log separately with their capacity or breaker state. The daemon can
4120    /// attest which code it sent: without the event, a client's "the daemon
4121    /// refused me" and the daemon's own view could only be reconciled by
4122    /// argument. Malformed input (`invalid_project_root`) does not come here;
4123    /// rejecting a request that was never a valid open is not a refusal of one.
4124    #[allow(clippy::too_many_arguments)]
4125    fn route_open_refusal_frame(
4126        &self,
4127        ctx: &RouteCtx,
4128        caller: &RouteOpenCaller,
4129        frame: &Frame,
4130        module_id: &str,
4131        reason: &'static str,
4132        code: &'static str,
4133        message: impl Into<String>,
4134    ) -> Result<Frame, RouterError> {
4135        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4136        control_error_frame(frame, code, message.into())
4137    }
4138
4139    /// Refuse a `route.open` because the target module's bind-relay breaker is
4140    /// open, without attempting the relay.
4141    ///
4142    /// The wire code is `module_timeout`, which is the truth (the module has
4143    /// not been answering binds) and which both SDKs already classify as
4144    /// retryable with capped backoff. Reusing it is what keeps this change out
4145    /// of both SDKs; the daemon-side distinction lives in the counter key
4146    /// instead.
4147    ///
4148    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4149    /// While a breaker is open this fires on every open to that module, and the
4150    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4151    /// produced 261 lines about a single module inside 3000 lines of daemon
4152    /// log. The rare transitions are logged at warn/info instead and the volume
4153    /// is carried by the counter, so the evidence survives without the flood.
4154    /// The debug line keeps a per-refusal record reachable for whoever turns
4155    /// the level up.
4156    #[allow(clippy::too_many_arguments)]
4157    fn route_open_breaker_refusal_frame(
4158        &self,
4159        ctx: &RouteCtx,
4160        caller: &RouteOpenCaller,
4161        frame: &Frame,
4162        module_id: &str,
4163        consecutive_timeouts: u32,
4164        retry_in: Duration,
4165        probe_in_flight: bool,
4166    ) -> Result<Frame, RouterError> {
4167        self.counters
4168            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4169        debug!(
4170            target: "control",
4171            code = "module_timeout",
4172            module_id = ?module_id,
4173            connection_id = ctx.connection_id.get(),
4174            principal = %caller.principal,
4175            scope = %caller.scope,
4176            harness = %caller.harness,
4177            consecutive_timeouts,
4178            retry_in_ms = retry_in.as_millis() as u64,
4179            probe_in_flight,
4180            "route.open refused by open bind-relay breaker"
4181        );
4182        // Say what a caller can act on. An open bind-relay breaker means the
4183        // module timed out accepting several new routes in a row. The module
4184        // is still running and its established routes keep working; only new
4185        // route.open requests are refused until the cooldown ends and one
4186        // test route (the probe) gets through. A message that only counts
4187        // failed relays reads as "the module is down" to a worker that sees it.
4188        let detail = if probe_in_flight {
4189            "one test route is already being tried; retry once it settles".to_string()
4190        } else {
4191            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4192        };
4193        control_error_frame(
4194            frame,
4195            "module_timeout",
4196            format!(
4197                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4198                 timed out in a row); {detail}; its established routes are unaffected"
4199            ),
4200        )
4201    }
4202
4203    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4204    /// requester's bytes (an unknown target is whatever the client sent) and
4205    /// is Debug-formatted so control characters land in the log escaped
4206    /// rather than as terminal sequences for whoever tails it.
4207    ///
4208    /// `reason` names the check that refused, because one wire code has
4209    /// several senders: after a module registers, `target_unavailable` can
4210    /// come from a missing role, an inactive registration, a supervisor that
4211    /// has not marked the process live, a missing forwarding connection, or a
4212    /// failed relay, and a log that records only the code cannot say which of
4213    /// them fired. It is a static, daemon-chosen label per branch, so it is
4214    /// safe to print plainly and stays a closed set.
4215    fn observe_route_open_refusal(
4216        &self,
4217        ctx: &RouteCtx,
4218        caller: &RouteOpenCaller,
4219        module_id: &str,
4220        reason: &'static str,
4221        code: &'static str,
4222    ) {
4223        self.counters.increment_route_open_refused(code);
4224        info!(
4225            target: "control",
4226            code,
4227            reason,
4228            module_id = ?module_id,
4229            connection_id = ctx.connection_id.get(),
4230            principal = %caller.principal,
4231            scope = %caller.scope,
4232            harness = %caller.harness,
4233            "route.open refused"
4234        );
4235        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4236            self.route_outages.record_not_serving(module_id, reason);
4237        }
4238    }
4239
4240    /// Record an ACCEPTED route.open.
4241    ///
4242    /// Refusals have been logged and counted since the attestation work; accepts
4243    /// were invisible, so the daemon knew every principal it stamped and wrote
4244    /// none of them down. The party that attests the identity was the only party
4245    /// not recording it, which left a credential vault unable to name the sender
4246    /// of a call that reached it (claustrum #43) and left the launch-nonce
4247    /// concurrency question unanswerable from the outside.
4248    ///
4249    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4250    /// `code`/`module_id`/`connection_id` returns both directions of the same
4251    /// decision rather than two shapes a reader has to join by hand.
4252    ///
4253    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4254    /// and the difference carries information rather than being an
4255    /// inconsistency. This line is only reachable after a successful bind to a
4256    /// REGISTERED module, so the value has already passed HELLO validation
4257    /// including the path-hazard refusal and cannot contain control bytes. A
4258    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4259    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4260    ///
4261    /// Bare is also what every other daemon line already emits (`module
4262    /// registered`, `configured module supervised`). Shipping `?module_id` here
4263    /// made this instrument the only one in the file whose ids did not answer
4264    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4265    /// line whose whole purpose is being grepped beside its sibling.
4266    ///
4267    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4268    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4269    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4270    /// one are recorded identically. A test written against that harness passes
4271    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4272    /// green assertion that cannot fail. The same limit applies to the escaping
4273    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4274    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4275    /// Fencing either needs the real formatter, not the capture layer.
4276    ///
4277    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4278    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4279    /// to record. The ephemeral port would decay within minutes and answer only a
4280    /// live question. The identity question is instead answered by counting
4281    /// distinct live connections presenting one module's `consumer_identity` --
4282    /// "is anyone else holding this secret" rather than "is this the right
4283    /// process".
4284    fn observe_route_open_accept(
4285        &self,
4286        ctx: &RouteCtx,
4287        module_id: &str,
4288        principal: &str,
4289        scope: &str,
4290    ) {
4291        self.route_outages.record_accepted(module_id);
4292        self.counters.increment_route_open_accepted(principal);
4293        info!(
4294            target: "control",
4295            principal,
4296            module_id,
4297            connection_id = ctx.connection_id.get(),
4298            scope = %scope,
4299            "route.open accepted"
4300        );
4301    }
4302
4303    fn supervised_absent_route_open_refusal_frame(
4304        &self,
4305        ctx: &RouteCtx,
4306        caller: &RouteOpenCaller,
4307        frame: &Frame,
4308        module_id: &str,
4309        code: &'static str,
4310        status: &crate::supervise::ModuleStatus,
4311    ) -> Result<Frame, RouterError> {
4312        self.counters.increment_route_open_refused(code);
4313        info!(
4314            target: "control",
4315            code,
4316            reason = "supervised_not_registered",
4317            module_id = ?module_id,
4318            connection_id = ctx.connection_id.get(),
4319            principal = %caller.principal,
4320            scope = %caller.scope,
4321            harness = %caller.harness,
4322            state = %status.state,
4323            enabled = status.enabled,
4324            live = status.live,
4325            "route.open refused"
4326        );
4327        // A supervised module whose process has not registered is not
4328        // serving, whatever the reason; the supervisor knows this id, so it is
4329        // safe to track.
4330        self.route_outages
4331            .record_not_serving(module_id, "supervised_not_registered");
4332        let message = if let Some(pid) = status.waiting_for_previous_pid {
4333            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4334                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4335            })
4336        } else {
4337            format!(
4338                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4339                status.state, status.enabled, status.live
4340            )
4341        };
4342        control_error_frame(frame, code, message)
4343    }
4344
4345    async fn handle_route_open(
4346        &self,
4347        ctx: &RouteCtx,
4348        frame: Frame,
4349        request: RouteOpenRequest,
4350        timings: &mut RouteOpenTimings,
4351    ) -> Result<Vec<Frame>, RouterError> {
4352        let RouteOpenRequest {
4353            target,
4354            mut identity,
4355            consumer_identity,
4356            consumer_capabilities,
4357            role_versions,
4358            admission_facts,
4359            scope,
4360        } = request;
4361        let target_module_id = target_module_id(&target).to_string();
4362        timings.module_id.clone_from(&target_module_id);
4363        if self
4364            .registry
4365            .get_module_by_connection(ctx.connection_id)
4366            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4367            .is_some()
4368        {
4369            return Ok(vec![control_error_frame(
4370                &frame,
4371                "invalid_request",
4372                "module connections cannot open client routes",
4373            )?]);
4374        }
4375        debug!(
4376            connection_id = ctx.connection_id.get(),
4377            corr = frame.header.corr,
4378            module_id = %target_module_id,
4379            "handling route.open"
4380        );
4381
4382        // The caller's attested principal, requested scope and harness, for
4383        // every `route.open refused` line below. The launch nonce is checked
4384        // here, once, and the answer is reused where the principal is
4385        // enforced further down, so the order of refusals is unchanged.
4386        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4387        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4388
4389        // A malformed declaration is refused first, before anything about the
4390        // target is looked up: the same body would be refused against any
4391        // module, so the caller learns nothing by retrying or waiting. An empty
4392        // map declares nothing and travels as no field at all, so a provider
4393        // only ever sees a missing field or a non-empty one.
4394        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4395        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4396            self.observe_route_open_refusal(
4397                ctx,
4398                &caller,
4399                &target_module_id,
4400                "invalid_role_versions",
4401                error_codes::INVALID_REQUEST,
4402            );
4403            return Ok(vec![control_error_body_frame(
4404                &frame,
4405                ErrorBody {
4406                    code: error_codes::INVALID_REQUEST.to_string(),
4407                    message: error.to_string(),
4408                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4409                },
4410            )?]);
4411        }
4412
4413        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4414            if status.waiting_for_previous_pid.is_some() {
4415                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4416                    ctx,
4417                    &caller,
4418                    &frame,
4419                    &target_module_id,
4420                    "module_warming",
4421                    &status,
4422                )?]);
4423            }
4424        }
4425
4426        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4427        // opposite. Below, a caller learns whether a module is unregistered,
4428        // supervised-but-down (with state/enabled/live), or registered without the
4429        // requested role. Elsewhere that is an enumeration leak: a probe learning
4430        // the shape of a fleet it cannot otherwise see.
4431        //
4432        // It is not one here, and the reason is the ACCESS MODEL rather than
4433        // anything about these errors. Reaching route.open requires the
4434        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4435        // connection file, so any caller who completes it already runs as this
4436        // user -- and can read subc.jsonc for the module list and `ck module
4437        // status` for live state. The reply discloses nothing the caller cannot
4438        // read more easily from disk, while the precision is load-bearing:
4439        // `unknown_module` is retryable and a missing role is not.
4440        //
4441        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4442        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4443        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4444        let Some(registration) = self
4445            .registry
4446            .get_module(&target_module_id)
4447            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4448        else {
4449            if let Some((status, warming)) =
4450                self.supervisor_status(&target_module_id, frame.header.corr)?
4451            {
4452                // BEFORE the two availability codes below, because for a module
4453                // that speaks no subc wire both of them are false comfort: they
4454                // say "not right now" and are retried, and this module will
4455                // never register no matter how long the caller waits. The
4456                // absence here is the declaration being honoured, not a module
4457                // that is late.
4458                if status.protocol == ModuleProtocol::None {
4459                    return Ok(vec![self.route_open_refusal_frame(
4460                        ctx,
4461                        &caller,
4462                        &frame,
4463                        &target_module_id,
4464                        "protocol_none",
4465                        error_codes::MODULE_NO_PROTOCOL,
4466                        format!(
4467                            "module_id '{target_module_id}' is declared protocol: none; \
4468                             it speaks no subc wire and serves no routes"
4469                        ),
4470                    )?]);
4471                }
4472                let code = if warming {
4473                    "module_warming"
4474                } else {
4475                    "target_unavailable"
4476                };
4477                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4478                    ctx,
4479                    &caller,
4480                    &frame,
4481                    &target_module_id,
4482                    code,
4483                    &status,
4484                )?]);
4485            }
4486            if let Some(removed_ago_ms) =
4487                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4488            {
4489                return Ok(vec![self.route_open_refusal_frame(
4490                    ctx,
4491                    &caller,
4492                    &frame,
4493                    &target_module_id,
4494                    "removed",
4495                    error_codes::MODULE_REMOVED,
4496                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4497                )?]);
4498            }
4499            return Ok(vec![self.route_open_refusal_frame(
4500                ctx,
4501                &caller,
4502                &frame,
4503                &target_module_id,
4504                "not_registered",
4505                error_codes::UNKNOWN_MODULE,
4506                format!("module_id '{target_module_id}' is not registered"),
4507            )?]);
4508        };
4509
4510        // Best-effort only: registry readiness and forwarding reservation use
4511        // different locks, so a module can flip readiness between this read and
4512        // the relay. Modules must still tolerate an `on_bind` while not ready.
4513        if !registration.ready {
4514            self.counters
4515                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4516            info!(
4517                target: "control",
4518                code = error_codes::MODULE_WARMING,
4519                module_id = ?target_module_id,
4520                connection_id = ctx.connection_id.get(),
4521                principal = %caller.principal,
4522                scope = %caller.scope,
4523                harness = %caller.harness,
4524                reason = "declared_not_ready",
4525                "route.open refused"
4526            );
4527            // The module is registered but says it cannot take work, which is
4528            // an outage from the caller's side even though its process is up.
4529            self.route_outages
4530                .record_not_serving(&target_module_id, "declared_not_ready");
4531            return Ok(vec![control_error_body_frame(
4532                &frame,
4533                ErrorBody {
4534                    code: error_codes::MODULE_WARMING.to_string(),
4535                    message: format!(
4536                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4537                    ),
4538                    detail: Some(serde_json::json!({
4539                        "reason": "declared_not_ready"
4540                    })),
4541                },
4542            )?]);
4543        }
4544
4545        // Effective readiness, second half: a module that declares a capability
4546        // `need: required` is not routable while that capability has no
4547        // registered provider. It is enforced HERE, as a retryable routing
4548        // refusal, and deliberately not as spawn ordering or a boot block. The
4549        // module is still started and registered and can make its own calls;
4550        // spawn ordering is a promise that cannot be kept once a provider
4551        // crashes at runtime, and refusing to boot would stop the whole
4552        // machine, including the tools needed to fix its configuration.
4553        //
4554        // "Provided" is the evaluator's verdict, which counts a provider as
4555        // soon as it has REGISTERED, not once it is ready. Two modules that
4556        // require each other's capabilities are therefore both routable once
4557        // both register; counting readiness instead would deadlock them.
4558        //
4559        // Only new opens are refused. Routes already bound when a provider
4560        // goes away stay bound: nothing here tears them down, and the module
4561        // answers them as it can. Like the readiness read above this is
4562        // best-effort against a provider registering or leaving concurrently.
4563        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4564            self.counters
4565                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4566            info!(
4567                target: "control",
4568                code = error_codes::MODULE_WARMING,
4569                module_id = ?target_module_id,
4570                connection_id = ctx.connection_id.get(),
4571                principal = %caller.principal,
4572                scope = %caller.scope,
4573                harness = %caller.harness,
4574                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4575                capability = %capability,
4576                "route.open refused"
4577            );
4578            return Ok(vec![control_error_body_frame(
4579                &frame,
4580                ErrorBody {
4581                    code: error_codes::MODULE_WARMING.to_string(),
4582                    message: format!(
4583                        "module_id '{target_module_id}' requires capability '{capability}', \
4584                         which no registered module provides; retry"
4585                    ),
4586                    detail: Some(serde_json::json!({
4587                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4588                        "capability": capability,
4589                    })),
4590                },
4591            )?]);
4592        }
4593
4594        if !target_has_required_role(&target, &registration.manifest.provides) {
4595            return Ok(vec![self.route_open_refusal_frame(
4596                ctx,
4597                &caller,
4598                &frame,
4599                &target_module_id,
4600                "role_not_provided",
4601                "target_unavailable",
4602                format!("module_id '{target_module_id}' does not provide the requested target"),
4603            )?]);
4604        }
4605
4606        if registration.state != ChannelState::Active {
4607            return Ok(vec![self.route_open_refusal_frame(
4608                ctx,
4609                &caller,
4610                &frame,
4611                &target_module_id,
4612                "registration_not_active",
4613                "target_unavailable",
4614                format!("module_id '{target_module_id}' is not active"),
4615            )?]);
4616        }
4617
4618        if self
4619            .forwarding
4620            .module_is_draining(&target_module_id)
4621            .map_err(RouterError::Forwarding)?
4622        {
4623            return Ok(vec![self.route_open_refusal_frame(
4624                ctx,
4625                &caller,
4626                &frame,
4627                &target_module_id,
4628                "reloading",
4629                "module_reloading",
4630                format!("module_id '{target_module_id}' is reloading"),
4631            )?]);
4632        }
4633
4634        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4635            process_liveness.process_live(&target_module_id) == Some(false)
4636        }) {
4637            // A module the supervisor is restarting or reloading can still hold
4638            // a registration: the old process before its connection closes, or
4639            // a new one that registered while the supervisor was draining. The
4640            // forwarding table does not see that as draining, but the consumer
4641            // should still be told to retry soon, exactly as for the drain
4642            // above, rather than that the target is unavailable.
4643            if process_liveness.process_replacing(&target_module_id) {
4644                return Ok(vec![self.route_open_refusal_frame(
4645                    ctx,
4646                    &caller,
4647                    &frame,
4648                    &target_module_id,
4649                    "reloading",
4650                    "module_reloading",
4651                    format!("module_id '{target_module_id}' is reloading"),
4652                )?]);
4653            }
4654            return Ok(vec![self.route_open_refusal_frame(
4655                ctx,
4656                &caller,
4657                &frame,
4658                &target_module_id,
4659                "supervisor_not_live",
4660                "target_unavailable",
4661                format!("module_id '{target_module_id}' is not live"),
4662            )?]);
4663        }
4664
4665        if !self
4666            .forwarding
4667            .has_live_module_connection(&target_module_id)
4668            .map_err(RouterError::Forwarding)?
4669        {
4670            return Ok(vec![self.route_open_refusal_frame(
4671                ctx,
4672                &caller,
4673                &frame,
4674                &target_module_id,
4675                "no_forwarding_connection",
4676                "target_unavailable",
4677                format!("module_id '{target_module_id}' has no live forwarding connection"),
4678            )?]);
4679        }
4680
4681        if let Some(error) =
4682            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4683        {
4684            self.observe_route_open_refusal(
4685                ctx,
4686                &caller,
4687                &target_module_id,
4688                "op_not_allowed",
4689                "op_not_allowed",
4690            );
4691            return Ok(vec![error]);
4692        }
4693
4694        let principal =
4695            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4696                Ok(principal) => principal,
4697                Err(error) => {
4698                    self.observe_route_open_refusal(
4699                        ctx,
4700                        &caller,
4701                        &target_module_id,
4702                        "bad_consumer_identity",
4703                        "bad_consumer_identity",
4704                    );
4705                    return Ok(vec![error]);
4706                }
4707            };
4708        timings.principal = principal_label(&principal);
4709
4710        // This is attested, control-plane policy for supervised module origins.
4711        // Keep it before route reservation and out of the opaque forwarding hot
4712        // path: data frames must never acquire a per-frame capability check.
4713        if let Principal::Reserved {
4714            module_id: opening_module_id,
4715        } = &principal
4716        {
4717            if let Some(opening_registration) = self
4718                .registry
4719                .get_module(opening_module_id)
4720                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4721            {
4722                if let Some(capability) =
4723                    denied_capability(&opening_registration.manifest, &registration.manifest)
4724                {
4725                    warn!(
4726                        opening_module_id,
4727                        target_module_id,
4728                        capability,
4729                        "refusing route.open because an attested capability deny edge matches"
4730                    );
4731                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4732                        &frame,
4733                        &target_module_id,
4734                        "capability_deny_edge",
4735                        "capability_forbidden",
4736                        format!(
4737                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4738                        ),
4739                    )?]);
4740                }
4741            }
4742        }
4743
4744        if admission_facts.is_some() {
4745            let carrier_matches = matches!(
4746                &principal,
4747                Principal::Reserved { module_id }
4748                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4749            );
4750            if !carrier_matches {
4751                return Ok(vec![self.route_open_refusal_frame(
4752                    ctx,
4753                    &caller,
4754                    &frame,
4755                    &target_module_id,
4756                    "admission_facts_carrier_not_permitted",
4757                    "admission_facts_not_permitted",
4758                    "admission facts may only be carried by the configured reserved module",
4759                )?]);
4760            }
4761
4762            let target_allowed = self
4763                .admission_facts_targets
4764                .as_ref()
4765                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4766            if !target_allowed {
4767                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4768                    &frame,
4769                    &target_module_id,
4770                    "admission_facts_target_not_listed",
4771                    "admission_facts_target_not_allowed",
4772                    format!(
4773                        "admission facts are not permitted for target module_id '{target_module_id}'"
4774                    ),
4775                )?]);
4776            }
4777
4778            // Keep the value opaque to subc. The downstream admission validator owns
4779            // schema and semantic checks; this daemon only enforces carrier authority
4780            // and the configured destination allowlist.
4781        }
4782
4783        // Scope admission, on the attested principal above and never on the
4784        // request body. The tag read here travels with the pending bind and is
4785        // compared with the published one at commit, so a sync between here
4786        // and the module's ack refuses the open instead of binding a stamp
4787        // that is no longer true.
4788        let (bound_scope, mut scope_stamp) = match &scope {
4789            None => (None, None),
4790            Some(selector) => {
4791                let owner_configured = match &selector.owner {
4792                    // A reserved owner counts as configured from before its
4793                    // process is spawned (see `SupervisorHandle::is_configured`).
4794                    // So an owner that has not synced its scopes yet is refused
4795                    // as retryable (`scope_not_synced`), not as one that will
4796                    // never sync.
4797                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4798                    _ => false,
4799                };
4800                let mut scopes = self
4801                    .scopes
4802                    .write()
4803                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4804                let admitted = scopes
4805                    .admit(&principal, &target_module_id, selector, owner_configured)
4806                    .and_then(|admission| {
4807                        crate::scopes::check_target_flow_support(
4808                            &admission.stamp,
4809                            &target_module_id,
4810                            registration.manifest.capabilities.as_ref(),
4811                        )?;
4812                        crate::scopes::check_target_agent_run_support(
4813                            &admission.stamp,
4814                            &target_module_id,
4815                            registration.manifest.capabilities.as_ref(),
4816                        )?;
4817                        let bound = scopes.reserve_bind(
4818                            &admission,
4819                            &principal,
4820                            ctx.connection_id,
4821                            selector.hold_presence,
4822                        );
4823                        Ok((admission, bound))
4824                    });
4825                match admitted {
4826                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4827                    Err(refusal) => {
4828                        return Ok(vec![self.route_open_refusal_frame(
4829                            ctx,
4830                            &caller,
4831                            &frame,
4832                            &target_module_id,
4833                            refusal.code,
4834                            refusal.code,
4835                            refusal.message,
4836                        )?]);
4837                    }
4838                }
4839            }
4840        };
4841        // From here on a refusal names the scope by the stamp admission
4842        // issued, exactly as the `route.open accepted` line does.
4843        if let Some(stamp) = &scope_stamp {
4844            caller.scope = route_open_scope_label(stamp);
4845        }
4846
4847        // Bind admits a root that no longer exists on disk, because refusing here
4848        // closes the only exit from a paused run: cancel needs a bound route, and a
4849        // renamed or reclaimed directory makes that route unopenable forever. The
4850        // run itself is intact and still addressable by its recorded identity.
4851        //
4852        // This does NOT relax the rule the strict constructor protects. That rule is
4853        // that no root is ever aliased into NEW durable state -- a missing component
4854        // can reappear as a symlink elsewhere, which would move the identity and
4855        // split a session's history across two of them. The engine now refuses the
4856        // two operations that create such state (send and import) at admission,
4857        // which is a narrower way to hold the same invariant: reads and terminations
4858        // are admitted, writes are not. That refusal had to ship before this line
4859        // changed, or there is an interval where a send commits under a provisional
4860        // identity -- the exact failure the original policy existed to prevent.
4861        //
4862        // Resolution follows realpath rather than lexical cleanup: the longest
4863        // existing ancestor is canonicalized and the missing tail re-appended, so a
4864        // live root is unchanged and a vanished leaf keeps the identity it was
4865        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4866        // same caller the moment the directory vanished, which strands the run more
4867        // quietly than refusing it.
4868        timings.enter(RouteOpenPhase::ProjectRoot);
4869        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4870            Ok(project_root) => project_root,
4871            Err(err) => {
4872                return Ok(vec![control_error_frame(
4873                    &frame,
4874                    "invalid_project_root",
4875                    err.to_string(),
4876                )?])
4877            }
4878        };
4879        timings.enter(RouteOpenPhase::Admission);
4880        identity.project_root = project_root.as_path().to_path_buf();
4881
4882        // Last gate before any relay work, and deliberately after the cheap
4883        // registry and availability checks above: those name a more precise
4884        // condition (unknown, removed, reloading) and a caller is better served
4885        // by the precise code than by this one.
4886        //
4887        // Everything below this point costs an egress permit, a reserved handle
4888        // pair and, if the module does not answer, the whole relay budget. The
4889        // reader no longer waits for that budget, so cap each target explicitly;
4890        // serial dispatch used to provide the accidental cap of one relay per
4891        // connection. Admission is a mutex-protected count and never waits.
4892        let _concurrency_guard = match self
4893            .route_bind_concurrency
4894            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4895        {
4896            Ok(guard) => guard,
4897            Err(in_flight) => {
4898                return Ok(vec![self.route_open_target_capacity_refusal(
4899                    ctx,
4900                    &caller,
4901                    &frame,
4902                    &target_module_id,
4903                    in_flight,
4904                )?]);
4905            }
4906        };
4907
4908        // A module that has already burned the whole budget `threshold` times
4909        // in a row does not get to charge it again until a probe says it recovered.
4910        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4911            RouteBindAdmission::Admitted { guard, probe } => {
4912                if probe {
4913                    info!(
4914                        module_id = %target_module_id,
4915                        connection_id = ctx.connection_id.get(),
4916                        "route.bind breaker half-open: admitting one probe"
4917                    );
4918                }
4919                guard
4920            }
4921            RouteBindAdmission::Refused {
4922                consecutive_timeouts,
4923                retry_in,
4924                probe_in_flight,
4925            } => {
4926                return Ok(vec![self.route_open_breaker_refusal_frame(
4927                    ctx,
4928                    &caller,
4929                    &frame,
4930                    &target_module_id,
4931                    consecutive_timeouts,
4932                    retry_in,
4933                    probe_in_flight,
4934                )?]);
4935            }
4936        };
4937
4938        // Resolve the per-module budget here so the wait matches the operator's
4939        // intent for this specific target. A per-module override in
4940        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4941        // daemons) wins over the daemon-wide default.
4942        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4943        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4944        timings.enter(RouteOpenPhase::EgressReservation);
4945        let pending = match self
4946            .forwarding
4947            .begin_route_bind_relay_for(
4948                ctx.connection_id,
4949                ctx.egress.clone(),
4950                response_version(&frame),
4951                frame.header.corr,
4952                &target_module_id,
4953                principal.clone(),
4954                bound_scope,
4955                Some(project_root),
4956                relay_deadline,
4957            )
4958            .await
4959        {
4960            Ok(pending) => pending,
4961            Err(err) => {
4962                return Ok(vec![self.route_open_refusal_frame(
4963                    ctx,
4964                    &caller,
4965                    &frame,
4966                    &target_module_id,
4967                    "relay_reservation_failed",
4968                    forwarding_error_code(&err),
4969                    err.to_string(),
4970                )?])
4971            }
4972        };
4973        timings.enter(RouteOpenPhase::Admission);
4974        let crate::forwarding::PendingRouteBindRelay {
4975            endpoint,
4976            module_sink,
4977            negotiated_ver,
4978            client_channel,
4979            client_epoch,
4980            module_channel,
4981            module_epoch,
4982            corr: relay_corr,
4983            receiver,
4984        } = pending;
4985        let mut reservation =
4986            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4987
4988        // Reserving egress can wait while a module reconnects or a swap cuts
4989        // over. Check the relay's captured connection, not the earlier by-id
4990        // lookup: the original module's flow/run capabilities cannot authorize
4991        // its replacement. The captured sink stays bound to that connection.
4992        let relay_registration = self
4993            .registry
4994            .get_module_by_connection(endpoint.connection_id)
4995            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4996        let relay_capabilities = relay_registration
4997            .as_ref()
4998            .and_then(|registration| registration.manifest.capabilities.as_ref());
4999        if let Some(selector) = &scope {
5000            // Egress reservation may wait across a scope update. Recheck the
5001            // proof against the current record before sending the captured bind.
5002            if let Err(refusal) = self
5003                .scopes
5004                .read()
5005                .unwrap_or_else(|poisoned| poisoned.into_inner())
5006                .check_bind_proof(&principal, selector)
5007            {
5008                reservation.release_and_disarm();
5009                return Ok(vec![self.route_open_refusal_frame(
5010                    ctx,
5011                    &caller,
5012                    &frame,
5013                    &target_module_id,
5014                    refusal.code,
5015                    refusal.code,
5016                    refusal.message,
5017                )?]);
5018            }
5019        }
5020        if let Some(stamp) = scope_stamp
5021            .as_ref()
5022            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5023        {
5024            if let Err(refusal) = crate::scopes::check_target_flow_support(
5025                stamp,
5026                &target_module_id,
5027                relay_capabilities,
5028            )
5029            .and_then(|()| {
5030                crate::scopes::check_target_agent_run_support(
5031                    stamp,
5032                    &target_module_id,
5033                    relay_capabilities,
5034                )
5035            }) {
5036                reservation.release_and_disarm();
5037                return Ok(vec![self.route_open_refusal_frame(
5038                    ctx,
5039                    &caller,
5040                    &frame,
5041                    &target_module_id,
5042                    refusal.code,
5043                    refusal.code,
5044                    refusal.message,
5045                )?]);
5046            }
5047        }
5048
5049        // Older readers may reject unknown fields. Select optional fields from
5050        // the captured endpoint's declaration, including on unscoped binds.
5051        if !relay_capabilities.is_some_and(|caps| {
5052            caps.provides
5053                .iter()
5054                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5055        }) {
5056            if let Some(stamp) = scope_stamp.as_mut() {
5057                stamp.workdir = None;
5058                stamp.project_id = None;
5059            }
5060        }
5061        // A worker run has no agent_id; on_behalf_of_agent is its only agent
5062        // attribution. The run gate above has already required the target to
5063        // declare `worker-runs/v1`, which promises to read both
5064        // `on_behalf_of_agent` and `call_mode`.
5065        let worker_run = scope_stamp
5066            .as_ref()
5067            .is_some_and(crate::scopes::is_worker_run)
5068            && relay_capabilities.is_some_and(|caps| {
5069                caps.provides
5070                    .iter()
5071                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5072            });
5073        if !worker_run
5074            && !relay_capabilities.is_some_and(|caps| {
5075                caps.provides
5076                    .iter()
5077                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5078            })
5079        {
5080            if let Some(stamp) = scope_stamp.as_mut() {
5081                stamp.attributes.on_behalf_of_agent = None;
5082            }
5083        }
5084        if !relay_capabilities.is_some_and(|caps| {
5085            caps.provides
5086                .iter()
5087                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5088        }) {
5089            if let Some(stamp) = scope_stamp.as_mut() {
5090                stamp.attributes.flow_version = None;
5091                stamp.attributes.flow_code_hash = None;
5092                stamp.attributes.flow_install_id = None;
5093            }
5094        }
5095        let call_mode = if worker_run
5096            || relay_capabilities
5097                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5098        {
5099            Some(derive_call_mode(scope_stamp.as_ref()))
5100        } else {
5101            if let Some(stamp) = scope_stamp.as_mut() {
5102                stamp.expires_at_ms = None;
5103            }
5104            None
5105        };
5106
5107        debug!(
5108            connection_id = ctx.connection_id.get(),
5109            client_channel,
5110            client_epoch,
5111            module_channel,
5112            module_epoch,
5113            "reserved route handle pair"
5114        );
5115        // Rendered BEFORE the move into the relay, because the accept arm below
5116        // is where it is logged and the principal is gone by then.
5117        let route_principal_label = principal_label(&principal);
5118        let relay = ModuleControlRequest::RouteBind {
5119            route_channel: module_channel,
5120            epoch: module_epoch,
5121            target,
5122            identity,
5123            principal: Some(principal),
5124            consumer_capabilities,
5125            role_versions,
5126            admission_facts,
5127            scope: scope_stamp,
5128            call_mode,
5129        };
5130        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5131            RouterError::backend(
5132                0,
5133                frame.header.corr,
5134                format!("failed to encode route.bind request: {err}"),
5135            )
5136        })?;
5137        let relay_frame = Frame::build_with_version(
5138            negotiated_ver,
5139            FrameType::Request,
5140            control_flags(),
5141            0,
5142            0,
5143            relay_corr,
5144            relay_body,
5145        )
5146        .map_err(RouterError::FrameBuild)?;
5147
5148        timings.enter(RouteOpenPhase::BindSend);
5149        let sent = module_sink.send(relay_frame).await;
5150        timings.enter(RouteOpenPhase::Admission);
5151        if let Err(err) = sent {
5152            reservation.release_and_disarm();
5153            return Ok(vec![self.route_open_refusal_frame(
5154                ctx,
5155                &caller,
5156                &frame,
5157                &target_module_id,
5158                "relay_send_failed",
5159                "target_unavailable",
5160                err.to_string(),
5161            )?]);
5162        }
5163
5164        #[cfg(test)]
5165        if let Some(hold) = &self.route_bind_relay_sent_hold {
5166            // First wait: tell the test the bind is queued. Second wait: stay
5167            // here until the test has driven whatever it wants to happen
5168            // between queueing the bind and marking it enqueued.
5169            hold.wait().await;
5170            hold.wait().await;
5171        }
5172
5173        // A missing relay is not by itself an abandoned bind: the module may
5174        // already have acked it and the route may be committed. Only a relay
5175        // dropped without any module reply needs this GOODBYE; one a reply
5176        // settled has already had its GOODBYE decided by the reply path.
5177        if self
5178            .forwarding
5179            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5180            .map_err(RouterError::Forwarding)?
5181            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5182        {
5183            self.send_abandoned_route_bind_goodbye(
5184                &module_sink,
5185                negotiated_ver,
5186                module_channel,
5187                module_epoch,
5188            );
5189        }
5190
5191        timings.enter(RouteOpenPhase::BindAckWait);
5192        let outcome = timeout_at(relay_deadline, receiver).await;
5193        timings.enter(RouteOpenPhase::Admission);
5194        match outcome {
5195            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5196                reservation.disarm();
5197                if breaker.record_accepted() {
5198                    info!(
5199                        module_id = %target_module_id,
5200                        "route.bind breaker closed: the probe was accepted"
5201                    );
5202                }
5203                self.observe_route_open_accept(
5204                    ctx,
5205                    &target_module_id,
5206                    &route_principal_label,
5207                    &caller.scope,
5208                );
5209                Ok(Vec::new())
5210            }
5211            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5212                reservation.release_and_disarm();
5213                // A module that says no in microseconds is healthy. Rejection
5214                // is a different condition with its own refusal and must not
5215                // move the breaker.
5216                breaker.record_inconclusive();
5217                // The daemon's own commit re-check refused the bind because the
5218                // scope ended or changed after admission. The module accepted;
5219                // counting it as a module rejection would blame the module.
5220                let scope_code = match body.code.as_str() {
5221                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5222                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5223                    _ => None,
5224                };
5225                if let Some(code) = scope_code {
5226                    self.observe_route_open_refusal(
5227                        ctx,
5228                        &caller,
5229                        &target_module_id,
5230                        "scope_changed_before_commit",
5231                        code,
5232                    );
5233                    return Ok(vec![control_error_body_frame(&frame, body)?]);
5234                }
5235                self.counters
5236                    .increment_route_open_refused("module_rejected");
5237                info!(
5238                    target: "control",
5239                    code = "module_rejected",
5240                    module_code = ?body.code,
5241                    module_id = ?target_module_id,
5242                    connection_id = ctx.connection_id.get(),
5243                    principal = %caller.principal,
5244                    scope = %caller.scope,
5245                    harness = %caller.harness,
5246                    "route.open refused"
5247                );
5248                Ok(vec![control_error_body_frame(&frame, body)?])
5249            }
5250            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5251                reservation.release_and_disarm();
5252                breaker.record_inconclusive();
5253                // Fires when the module's connection closes while a relayed
5254                // bind is pending -- typically a caller racing a module restart
5255                // whose bind was relayed BEFORE the drain mark went up. Logged
5256                // because the caller sees only its own error and the fleet has
5257                // already spent one diagnosis round unable to tell this arm
5258                // from a relay timeout without daemon-side evidence.
5259                tracing::warn!(
5260                    module_id = %target_module_id,
5261                    "route.bind relay abandoned: {message}"
5262                );
5263                Ok(vec![self.route_open_refusal_frame(
5264                    ctx,
5265                    &caller,
5266                    &frame,
5267                    &target_module_id,
5268                    "relay_abandoned",
5269                    "target_unavailable",
5270                    message,
5271                )?])
5272            }
5273            Ok(Err(_)) => {
5274                reservation.release_and_disarm();
5275                breaker.record_inconclusive();
5276                Ok(vec![self.route_open_refusal_frame(
5277                    ctx,
5278                    &caller,
5279                    &frame,
5280                    &target_module_id,
5281                    "relay_waiter_canceled",
5282                    "target_unavailable",
5283                    "route.bind relay waiter was canceled before the module responded",
5284                )?])
5285            }
5286            Err(_) => {
5287                reservation.release_and_disarm();
5288                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5289                // answer at all is the one condition a fast refusal can
5290                // usefully stand in for; every other arm already answered.
5291                if let Some(opened) = breaker.record_timeout(
5292                    self.route_bind_breaker_threshold,
5293                    self.route_bind_breaker_cooldown,
5294                ) {
5295                    warn!(
5296                        module_id = %target_module_id,
5297                        consecutive_timeouts = opened.consecutive_timeouts,
5298                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5299                        reopened_after_probe = opened.reopened_after_probe,
5300                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5301                    );
5302                }
5303                // The generous budget just burned to no answer: the module is
5304                // registered and its connection is up, but its bind handler sat
5305                // on the ack for the full budget (warm-on-bind, cold configure,
5306                // or a wedged handler). Every earlier unavailability shape
5307                // fast-refuses BEFORE the relay, so this arm firing means the
5308                // slowness is module-side -- log it so the per-module timeline
5309                // is reconstructable without client audit rows.
5310                tracing::warn!(
5311                    module_id = %target_module_id,
5312                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5313                    "route.bind relay timed out: module did not ack within budget"
5314                );
5315                Ok(vec![self.route_open_refusal_frame(
5316                    ctx,
5317                    &caller,
5318                    &frame,
5319                    &target_module_id,
5320                    "relay_timed_out",
5321                    "module_timeout",
5322                    format!(
5323                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5324                        route_bind_relay_timeout
5325                    ),
5326                )?])
5327            }
5328        }
5329    }
5330
5331    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5332        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5333            snapshot: self.supervisor.spawn_snapshot(),
5334        };
5335        Ok(vec![control_response_body_frame(
5336            &frame,
5337            &response,
5338            "ClientControlResponse::SupervisorSpawnSnapshot",
5339        )?])
5340    }
5341
5342    fn handle_supervisor_spawn_subscribe(
5343        &self,
5344        ctx: &RouteCtx,
5345        frame: Frame,
5346        since: Option<SpawnCursor>,
5347    ) -> Result<Vec<Frame>, RouterError> {
5348        match self.supervisor.subscribe_spawns(
5349            ctx.connection_id,
5350            frame.header.corr,
5351            response_version(&frame),
5352            since,
5353            ctx.egress.clone(),
5354        ) {
5355            Ok(()) => Ok(Vec::new()),
5356            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5357                Ok(vec![control_error_body_frame(
5358                    &frame,
5359                    ErrorBody {
5360                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5361                        message: "spawn cursor belongs to a different daemon incarnation"
5362                            .to_string(),
5363                        detail: Some(serde_json::json!({
5364                            "current_daemon_incarnation": current
5365                        })),
5366                    },
5367                )?])
5368            }
5369            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5370                &frame,
5371                ErrorBody {
5372                    code: "spawn_cursor_too_old".to_string(),
5373                    message: "spawn cursor predates the retained event ring".to_string(),
5374                    detail: Some(serde_json::json!({
5375                        "oldest_retained_cursor": oldest
5376                    })),
5377                },
5378            )?]),
5379            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5380                0,
5381                frame.header.corr,
5382                format!("failed to open supervisor spawn subscription: {error}"),
5383            )),
5384        }
5385    }
5386
5387    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5388        let generation = self
5389            .registry
5390            .generation()
5391            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5392        let mut modules = Vec::new();
5393        for module in self.supervisor.list() {
5394            let status = module.status_for_control("list").map_err(|err| {
5395                RouterError::backend(
5396                    0,
5397                    frame.header.corr,
5398                    format!("failed to read supervisor status: {err}"),
5399                )
5400            })?;
5401            let (configured, _) = module.configuration().map_err(|err| {
5402                RouterError::backend(
5403                    0,
5404                    frame.header.corr,
5405                    format!("failed to read module configuration: {err}"),
5406                )
5407            })?;
5408            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5409                RouterError::backend(
5410                    0,
5411                    frame.header.corr,
5412                    format!("failed to read retry time: {err}"),
5413                )
5414            })?;
5415            // Status and configuration snapshots release their locks before the image probe awaits.
5416            let image = module.running_image_agreement().await;
5417            // Read per request so the figure is current when the operator asks;
5418            // the daemon samples nothing in between.
5419            let resources = Some(module.child_resource_usage());
5420            let mut verdict =
5421                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5422            verdict.spec = Some(launch_spec_agreement(
5423                &configured,
5424                status.process_alive,
5425                status.launch_spec_digest.as_ref(),
5426            ));
5427            let pending_reload = Some(verdict);
5428            let configured_enabled = self
5429                .capability_evaluator
5430                .configured_enabled(&status.module_id);
5431            // Keep the retired policy field on the wire for one release so
5432            // existing status consumers still receive the platform policy.
5433            modules.push(
5434                SupervisorEntry::new(
5435                    status.module_id,
5436                    status.state.to_string(),
5437                    status.enabled,
5438                    status.live,
5439                    status.health.status,
5440                )
5441                .with_configured_enabled(configured_enabled)
5442                .with_next_retry_at_ms(next_retry_at_ms)
5443                .with_launch_nonce_env(Some(!cfg!(unix)))
5444                .with_launch_nonce_source(status.launch_nonce_source)
5445                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5446                .with_protocol(status.protocol)
5447                .with_pending_reload(pending_reload)
5448                .with_last_probe_ms(status.health.last_probe_ms)
5449                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5450                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5451                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5452                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5453                .with_restart_count(Some(status.restart_count))
5454                .with_max_restarts(Some(status.max_restarts))
5455                .with_lifetime_restarts(Some(status.lifetime_restarts))
5456                .with_spawn_generation(Some(status.spawn_generation))
5457                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5458                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5459                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5460                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5461                .with_resources(resources),
5462            );
5463        }
5464        let response = ClientControlResponse::SupervisorList {
5465            generation,
5466            modules,
5467        };
5468        Ok(vec![control_response_body_frame(
5469            &frame,
5470            &response,
5471            "ClientControlResponse::SupervisorList",
5472        )?])
5473    }
5474
5475    fn handle_supervisor_stderr_tail(
5476        &self,
5477        frame: Frame,
5478        module_id: String,
5479        max_lines: Option<u32>,
5480        max_bytes: Option<u32>,
5481    ) -> Result<Vec<Frame>, RouterError> {
5482        let Some(module) = self.supervisor.get(&module_id) else {
5483            return Ok(vec![control_error_frame(
5484                &frame,
5485                "unknown_module",
5486                format!("module_id '{module_id}' is not supervised"),
5487            )?]);
5488        };
5489
5490        let snapshot = module.stderr_tail(
5491            max_lines.map(|value| value as usize),
5492            max_bytes.map(|value| value as usize),
5493        );
5494
5495        let response = ClientControlResponse::SupervisorStderrTail {
5496            module_id,
5497            tail: StderrTail {
5498                capture: match snapshot.capture {
5499                    CaptureState::Captured => StderrCaptureState::Captured,
5500                    CaptureState::Incomplete { reason } => {
5501                        StderrCaptureState::Incomplete { reason }
5502                    }
5503                    CaptureState::NotCaptured { reason } => {
5504                        StderrCaptureState::NotCaptured { reason }
5505                    }
5506                },
5507                entries: snapshot
5508                    .entries
5509                    .into_iter()
5510                    .map(|entry| match entry {
5511                        TailEntry::Line {
5512                            text,
5513                            truncated,
5514                            at_ms,
5515                        } => StderrTailEntry::Line {
5516                            text,
5517                            truncated,
5518                            at_ms,
5519                        },
5520                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5521                    })
5522                    .collect(),
5523                dropped_lines: snapshot.dropped_lines,
5524            },
5525        };
5526        Ok(vec![control_response_body_frame(
5527            &frame,
5528            &response,
5529            "ClientControlResponse::SupervisorStderrTail",
5530        )?])
5531    }
5532
5533    async fn handle_supervisor_terminals(
5534        &self,
5535        frame: Frame,
5536        module_id: String,
5537    ) -> Result<Vec<Frame>, RouterError> {
5538        let Some(module) = self.supervisor.get(&module_id) else {
5539            return Ok(vec![control_error_frame(
5540                &frame,
5541                "unknown_module",
5542                format!("module_id '{module_id}' is not supervised"),
5543            )?]);
5544        };
5545
5546        // The journal read runs on a blocking thread: it can be megabytes of
5547        // file I/O and must not occupy a runtime worker.
5548        let terminals = module
5549            .read_durable_terminal_history()
5550            .await
5551            .map_err(|error| {
5552                RouterError::backend(
5553                    0,
5554                    frame.header.corr,
5555                    format!("failed to read terminal history: {error}"),
5556                )
5557            })?;
5558        let response = ClientControlResponse::SupervisorTerminals {
5559            module_id,
5560            terminals,
5561        };
5562        Ok(vec![control_response_body_frame(
5563            &frame,
5564            &response,
5565            "ClientControlResponse::SupervisorTerminals",
5566        )?])
5567    }
5568
5569    fn handle_supervisor_routes(
5570        &self,
5571        frame: Frame,
5572        module_id: Option<String>,
5573    ) -> Result<Vec<Frame>, RouterError> {
5574        let modules = self
5575            .forwarding
5576            .route_census(module_id.as_deref())
5577            .map_err(RouterError::Forwarding)?
5578            .into_iter()
5579            .map(|(module_id, routes)| SupervisorRouteModule {
5580                module_id,
5581                routes: routes
5582                    .into_iter()
5583                    .map(|route| SupervisorRoute {
5584                        consumer: match route.principal {
5585                            Principal::Reserved { module_id } => {
5586                                SupervisorRouteConsumer::Reserved { module_id }
5587                            }
5588                            Principal::Direct | Principal::Unverified => {
5589                                SupervisorRouteConsumer::Direct {
5590                                    connection_id: route.goodbye_target.connection_id.get(),
5591                                }
5592                            }
5593                        },
5594                        age_ms: Instant::now()
5595                            .saturating_duration_since(route.bound_at)
5596                            .as_millis()
5597                            .try_into()
5598                            .unwrap_or(u64::MAX),
5599                        draining: route.draining,
5600                        drain_reason: route.drain_reason,
5601                    })
5602                    .collect(),
5603            })
5604            .collect();
5605        let response = ClientControlResponse::SupervisorRoutes { modules };
5606        Ok(vec![control_response_body_frame(
5607            &frame,
5608            &response,
5609            "ClientControlResponse::SupervisorRoutes",
5610        )?])
5611    }
5612
5613    async fn handle_supervisor_provenance(
5614        &self,
5615        frame: Frame,
5616        module_id: Option<String>,
5617    ) -> Result<Vec<Frame>, RouterError> {
5618        let mut selected = if let Some(module_id) = module_id {
5619            let Some(module) = self.supervisor.get(&module_id) else {
5620                return Ok(vec![control_error_frame(
5621                    &frame,
5622                    "unknown_module",
5623                    format!("module_id '{module_id}' is not supervised"),
5624                )?]);
5625            };
5626            vec![module]
5627        } else {
5628            self.supervisor.list()
5629        };
5630
5631        let mut modules = Vec::with_capacity(selected.len());
5632        for module in selected.drain(..) {
5633            let (status, observed_image) = module
5634                .status_and_running_image_agreement()
5635                .await
5636                .map_err(|err| {
5637                    RouterError::backend(
5638                        0,
5639                        frame.header.corr,
5640                        format!("failed to read supervisor status: {err}"),
5641                    )
5642                })?;
5643            let module_declared = self
5644                .registry
5645                .get_module(&status.module_id)
5646                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5647                .and_then(|registration| registration.manifest.provenance)
5648                .map(|build| ModuleDeclaredProvenance::Reported { build })
5649                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5650            #[cfg(test)]
5651            let running_image = match &self.provenance_probe_override {
5652                Some(result) => result.clone(),
5653                None => observed_image,
5654            };
5655            #[cfg(not(test))]
5656            let running_image = observed_image;
5657            modules.push(SupervisorModuleProvenance {
5658                module_id: status.module_id,
5659                module_declared,
5660                daemon_observed: SupervisorObservedProcess {
5661                    launch_nonce_source: status.launch_nonce_source,
5662                    pid: status.pid,
5663                    spawned_at_ms: status.spawned_at_ms,
5664                    spawned_from: status.spawned_from,
5665                    running_image,
5666                },
5667            });
5668        }
5669        let daemon = SupervisorDaemonProvenance {
5670            daemon_build: self.daemon_provenance.build.clone(),
5671            daemon_observed: DaemonObservedProcess {
5672                pid: self.daemon_provenance.pid,
5673                started_at_ms: self
5674                    .daemon_provenance
5675                    .start_clock
5676                    .map(|clock| clock.started_at_ms())
5677                    .or(self.daemon_provenance.started_at_ms),
5678                running_image: self
5679                    .daemon_provenance
5680                    .probe
5681                    .observe(
5682                        self.daemon_provenance.pid,
5683                        self.daemon_provenance.executable_path.as_deref(),
5684                        self.daemon_provenance.executable_identity.clone(),
5685                        self.daemon_provenance.process_start_time,
5686                    )
5687                    .await,
5688            },
5689        };
5690        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5691        Ok(vec![control_response_body_frame(
5692            &frame,
5693            &response,
5694            "ClientControlResponse::SupervisorProvenance",
5695        )?])
5696    }
5697
5698    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5699        self.refresh_capability_requirements();
5700        let generation = self
5701            .registry
5702            .generation()
5703            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5704        let modules = self
5705            .supervisor
5706            .list()
5707            .into_iter()
5708            .map(|module| {
5709                let status = module.status_for_control("health").map_err(|err| {
5710                    RouterError::backend(
5711                        0,
5712                        frame.header.corr,
5713                        format!("failed to read supervisor health: {err}"),
5714                    )
5715                })?;
5716                let module_id = status.module_id;
5717                let capability_detail = self
5718                    .capability_evaluator
5719                    .required_problem_detail(&module_id);
5720                Ok(SupervisorHealthEntry {
5721                    module_id,
5722                    status: status.health.status,
5723                    detail: append_capability_problem_detail(
5724                        status.health.detail,
5725                        capability_detail,
5726                    ),
5727                    metrics: status.health.metrics,
5728                    consecutive_failures: status.health.consecutive_failures,
5729                    late_answer_count: status.health.late_answer_count,
5730                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5731                    last_action: status.health.last_action,
5732                    last_action_ms: status.health.last_action_ms,
5733                    last_probe_ms: status.health.last_probe_ms,
5734                })
5735            })
5736            .collect::<Result<Vec<_>, RouterError>>()?;
5737        let response = ClientControlResponse::SupervisorHealth {
5738            generation,
5739            modules,
5740        };
5741        Ok(vec![control_response_body_frame(
5742            &frame,
5743            &response,
5744            "ClientControlResponse::SupervisorHealth",
5745        )?])
5746    }
5747
5748    async fn handle_supervisor_restart(
5749        &self,
5750        frame: Frame,
5751        module_id: String,
5752        drain_timeout_ms: Option<u64>,
5753        requester: Principal,
5754    ) -> Result<Vec<Frame>, RouterError> {
5755        let operation_lock = self.supervisor.operation_lock();
5756        let _operation_guard = operation_lock.lock().await;
5757        let Some(module) = self.supervisor.get(&module_id) else {
5758            return Ok(vec![control_error_frame(
5759                &frame,
5760                "unknown_module",
5761                format!("module_id '{module_id}' is not supervised"),
5762            )?]);
5763        };
5764
5765        self.route_outages.mark_operator_action(&module_id);
5766        if let Err(err) = module
5767            .restart_requested(drain_timeout_ms, Some(requester))
5768            .await
5769        {
5770            self.route_outages
5771                .operator_action_ended_unrefused(&module_id);
5772            let (code, message) = match err {
5773                crate::supervise::SuperviseError::Disabled { .. } => {
5774                    ("module_disabled", err.to_string())
5775                }
5776                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5777                    ("swap_in_progress", err.to_string())
5778                }
5779                _ => (
5780                    "target_unavailable",
5781                    format!("failed to restart module_id '{module_id}': {err}"),
5782                ),
5783            };
5784            return Ok(vec![control_error_frame(&frame, code, message)?]);
5785        }
5786
5787        let response = ClientControlResponse::SupervisorAck {
5788            module_id,
5789            applied: true,
5790        };
5791        Ok(vec![control_response_body_frame(
5792            &frame,
5793            &response,
5794            "ClientControlResponse::SupervisorAck",
5795        )?])
5796    }
5797
5798    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5799    /// when the old process has finished draining: a caller whose own lane
5800    /// rides the old process must get its reply before that drain waits on it.
5801    async fn handle_supervisor_swap(
5802        &self,
5803        frame: Frame,
5804        module_id: String,
5805        ready_timeout_ms: Option<u64>,
5806        requester: Principal,
5807    ) -> Result<Vec<Frame>, RouterError> {
5808        // The daemon-wide operation lock is held only to resolve the handle,
5809        // not across the swap. The swap can take its whole readiness budget,
5810        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5811        // holding it here would park an operator's stop behind the swap it is
5812        // meant to abort. A rescan or stop that reaches the module during the
5813        // swap is served by the swap itself (see `supervise_swap`).
5814        let module = {
5815            let operation_lock = self.supervisor.operation_lock();
5816            let _operation_guard = operation_lock.lock().await;
5817            self.supervisor.get(&module_id)
5818        };
5819        let Some(module) = module else {
5820            return Ok(vec![control_error_frame(
5821                &frame,
5822                "unknown_module",
5823                format!("module_id '{module_id}' is not supervised"),
5824            )?]);
5825        };
5826
5827        self.route_outages.mark_operator_action(&module_id);
5828        if let Err(err) = module
5829            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5830            .await
5831        {
5832            self.route_outages
5833                .operator_action_ended_unrefused(&module_id);
5834            use crate::supervise::SuperviseError;
5835            let message = err.to_string();
5836            let error = match err {
5837                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5838                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5839                    code: "swap_refused".to_string(),
5840                    message,
5841                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5842                },
5843                SuperviseError::SwapFailed {
5844                    arm,
5845                    candidate_exit,
5846                    ..
5847                } => ErrorBody {
5848                    code: "swap_failed".to_string(),
5849                    message,
5850                    detail: Some(serde_json::json!({
5851                        "arm": arm.as_str(),
5852                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5853                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5854                    })),
5855                },
5856                _ => ErrorBody::new(
5857                    "target_unavailable",
5858                    format!("failed to swap module_id '{module_id}': {message}"),
5859                ),
5860            };
5861            return Ok(vec![control_error_body_frame(&frame, error)?]);
5862        }
5863        // A completed swap kept the incumbent serving until cutover, so it
5864        // usually opened no outage; a mark left behind would make the next,
5865        // unrelated outage read as requested.
5866        self.route_outages
5867            .operator_action_ended_unrefused(&module_id);
5868
5869        let response = ClientControlResponse::SupervisorAck {
5870            module_id,
5871            applied: true,
5872        };
5873        Ok(vec![control_response_body_frame(
5874            &frame,
5875            &response,
5876            "ClientControlResponse::SupervisorAck",
5877        )?])
5878    }
5879
5880    async fn handle_supervisor_reload(
5881        &self,
5882        frame: Frame,
5883        module_id: String,
5884        requester: Principal,
5885    ) -> Result<Vec<Frame>, RouterError> {
5886        let operation_lock = self.supervisor.operation_lock();
5887        let _operation_guard = operation_lock.lock().await;
5888        let Some(module) = self.supervisor.get(&module_id) else {
5889            return Ok(vec![control_error_frame(
5890                &frame,
5891                "unknown_module",
5892                format!("module_id '{module_id}' is not supervised"),
5893            )?]);
5894        };
5895
5896        self.route_outages.mark_operator_action(&module_id);
5897        if let Err(err) = module.reload_requested(Some(requester)).await {
5898            self.route_outages
5899                .operator_action_ended_unrefused(&module_id);
5900            let (code, message) = match err {
5901                crate::supervise::SuperviseError::Disabled { .. } => {
5902                    ("module_disabled", err.to_string())
5903                }
5904                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5905                    ("swap_in_progress", err.to_string())
5906                }
5907                _ => (
5908                    "reload_failed",
5909                    format!("failed to reload module_id '{module_id}': {err}"),
5910                ),
5911            };
5912            return Ok(vec![control_error_frame(&frame, code, message)?]);
5913        }
5914
5915        let response = ClientControlResponse::SupervisorAck {
5916            module_id,
5917            applied: true,
5918        };
5919        Ok(vec![control_response_body_frame(
5920            &frame,
5921            &response,
5922            "ClientControlResponse::SupervisorAck",
5923        )?])
5924    }
5925
5926    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5927    /// the configuration comes from and whether to apply it; every mode then
5928    /// runs the same comparison, so a preview describes the apply it previews.
5929    async fn handle_supervisor_rescan(
5930        &self,
5931        frame: Frame,
5932        mode: RescanMode,
5933        requester: Principal,
5934    ) -> Result<Vec<Frame>, RouterError> {
5935        let preview = mode.previews();
5936        let Some(context) = self.rescan.clone() else {
5937            return Ok(vec![control_error_frame(
5938                &frame,
5939                "rescan_unavailable",
5940                "the daemon was not started with a reloadable config path".to_string(),
5941            )?]);
5942        };
5943
5944        let operation_lock = self.supervisor.operation_lock();
5945        let _operation_guard = operation_lock.lock().await;
5946        // Loaded under the operation lock, so no other rescan can change the
5947        // staged configuration between a checked apply's comparison and the
5948        // apply itself.
5949        let loaded = match self.load_rescan_config(&mode, &context) {
5950            Ok(config) => config,
5951            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5952        };
5953        // `load` reports a missing file as Ok(None), which is correct at boot
5954        // (no config, nothing to supervise) and catastrophic here: rescan treats
5955        // "not in the config" as "remove it", so an absent file would read as an
5956        // empty module list and retire the entire running fleet. An editor
5957        // writing via write-new-then-rename, or a half-finished edit, is enough
5958        // to open that window. Refuse instead: a config that cannot be read
5959        // carries no instruction to remove anything.
5960        let Some((config, activity_config)) = loaded else {
5961            return Ok(vec![control_error_frame(
5962                &frame,
5963                "invalid_daemon_config",
5964                format!(
5965                    "daemon config not found at {}; refusing to rescan (an absent config would \
5966                     retire every supervised module)",
5967                    context.config_path.display()
5968                ),
5969            )?]);
5970        };
5971        let (
5972            configured_port,
5973            storage_config,
5974            admission_facts_carrier_module_id,
5975            admission_facts_targets,
5976            scope_authority_owners,
5977            privileged_operator_prompt,
5978            modules,
5979            reserved_capabilities,
5980        ) = (
5981            config.port,
5982            config.storage,
5983            config.admission_facts_carrier_module_id,
5984            config.admission_facts_targets,
5985            config.scope_authority_owners,
5986            config.privileged_operator_prompt,
5987            config.modules,
5988            config.reserved_capabilities,
5989        );
5990
5991        // Collect the sections rescan cannot apply, so the REPLY carries them.
5992        //
5993        // The warning below has always been correct and has always gone only to
5994        // the journal -- addressed to whoever reads logs, while the person who
5995        // just edited the config is looking at the CLI. Naming each section
5996        // individually rather than setting a flag: "something outside modules
5997        // changed" sends the operator back to diffing their own file, which is
5998        // the work this is meant to save.
5999        let mut restart_required = Vec::new();
6000        for section in RestartRequiredSection::ALL {
6001            let changed = match section {
6002                RestartRequiredSection::Port => configured_port != context.configured_port,
6003                RestartRequiredSection::Storage => storage_config != context.storage_config,
6004                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6005                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6006                }
6007                RestartRequiredSection::AdmissionFactsTargets => {
6008                    admission_facts_targets != context.admission_facts_targets
6009                }
6010                RestartRequiredSection::ScopeAuthorityOwners => {
6011                    scope_authority_owners != context.scope_authority_owners
6012                }
6013                RestartRequiredSection::PrivilegedOperatorPrompt => {
6014                    privileged_operator_prompt != context.privileged_operator_prompt
6015                }
6016            };
6017            if changed {
6018                restart_required.push(section.label().to_string());
6019            }
6020        }
6021        if !restart_required.is_empty() {
6022            warn!(
6023                config_path = %context.config_path.display(),
6024                sections = %restart_required.join(", "),
6025                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6026            );
6027        }
6028
6029        for configured in &modules {
6030            if let Err(err) = validate_spec(&configured.module_spec()) {
6031                return Ok(vec![control_error_frame(
6032                    &frame,
6033                    "invalid_daemon_config",
6034                    format!("supervisor rescan rejected daemon config: {err}"),
6035                )?]);
6036            }
6037        }
6038
6039        let configured_capabilities = modules
6040            .iter()
6041            .map(|module| (module.module_id.clone(), module.enabled))
6042            .collect::<Vec<_>>();
6043        let preview_capability_warnings = if preview {
6044            let (_, registrations) = self.runtime_capability_snapshot()?;
6045            let current_modules = self
6046                .supervisor
6047                .list()
6048                .into_iter()
6049                .map(|module| module.module_id().to_string())
6050                .collect::<BTreeSet<_>>();
6051            let resulting_modules = configured_capabilities.clone();
6052            let removed = current_modules
6053                .into_iter()
6054                .filter(|module_id| {
6055                    !resulting_modules
6056                        .iter()
6057                        .any(|(configured_id, _)| configured_id == module_id)
6058                })
6059                .collect::<Vec<_>>();
6060            // A candidate preview also names providers the candidate disables.
6061            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6062            let disabled = match &mode {
6063                RescanMode::Candidate(_) => resulting_modules
6064                    .iter()
6065                    .filter(|(_, enabled)| !enabled)
6066                    .map(|(module_id, _)| module_id.clone())
6067                    .collect::<Vec<_>>(),
6068                _ => Vec::new(),
6069            };
6070            self.capability_evaluator.preview_warnings(
6071                resulting_modules,
6072                &removed,
6073                &disabled,
6074                &registrations,
6075            )
6076        } else {
6077            Vec::new()
6078        };
6079        let reconciled = match &mode {
6080            RescanMode::Candidate(candidate) => self
6081                .preview_candidate(&context, modules, candidate)
6082                .map(|result| (result, Vec::new())),
6083            _ => {
6084                self.reconcile_supervised_modules(
6085                    &context.supervisor,
6086                    modules,
6087                    preview,
6088                    requester.clone(),
6089                )
6090                .await
6091            }
6092        };
6093        let (result, changed) = match reconciled {
6094            Ok(result) => result,
6095            Err(message) => {
6096                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6097            }
6098        };
6099        if !preview {
6100            if let Some(ledger) = &self.activity {
6101                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6102                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6103            }
6104            if let Err(code) = self.configure_activity(activity_config).await {
6105                warn!(
6106                    code,
6107                    "activity retention could not be applied during rescan"
6108                );
6109            }
6110            self.capability_evaluator
6111                .configure(configured_capabilities, reserved_capabilities);
6112            self.capability_evaluator.wake_deadline_loop();
6113            self.refresh_capability_requirements();
6114        }
6115        let mut result = result;
6116        result.restart_required = restart_required;
6117        result.capability_warnings = preview_capability_warnings;
6118        let response = ClientControlResponse::SupervisorRescan { result };
6119        Ok(vec![control_response_body_frame(
6120            &frame,
6121            &response,
6122            "ClientControlResponse::SupervisorRescan",
6123        )?])
6124    }
6125
6126    async fn handle_supervisor_release_reserved(
6127        &self,
6128        frame: Frame,
6129        module_id: String,
6130    ) -> Result<Vec<Frame>, RouterError> {
6131        let Some(context) = self.rescan.clone() else {
6132            return Ok(vec![control_error_frame(
6133                &frame,
6134                "release_unavailable",
6135                "reserved-id release requires a daemon started with a reloadable config path",
6136            )?]);
6137        };
6138        let operation_lock = self.supervisor.operation_lock();
6139        let _operation_guard = operation_lock.lock().await;
6140        let loaded = match crate::daemon_config::load(&context.config_path) {
6141            Ok(Some(config)) => config,
6142            Ok(None) => {
6143                return Ok(vec![control_error_frame(
6144                    &frame,
6145                    "invalid_daemon_config",
6146                    format!(
6147                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6148                        context.config_path.display()
6149                    ),
6150                )?])
6151            }
6152            Err(err) => {
6153                return Ok(vec![control_error_frame(
6154                    &frame,
6155                    "invalid_daemon_config",
6156                    format!("unable to verify reserved-id release against daemon config: {err}"),
6157                )?])
6158            }
6159        };
6160        if loaded
6161            .modules
6162            .iter()
6163            .any(|configured| configured.module_id == module_id)
6164        {
6165            return Ok(vec![control_error_frame(
6166                &frame,
6167                "reserved_module_configured",
6168                format!(
6169                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6170                ),
6171            )?]);
6172        }
6173        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6174            return Ok(vec![control_error_frame(
6175                &frame,
6176                "reserved_gate_not_retained",
6177                format!(
6178                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6179                ),
6180            )?]);
6181        }
6182
6183        let response = ClientControlResponse::SupervisorAck {
6184            module_id,
6185            applied: true,
6186        };
6187        Ok(vec![control_response_body_frame(
6188            &frame,
6189            &response,
6190            "ClientControlResponse::SupervisorAck",
6191        )?])
6192    }
6193
6194    /// Compare the configured module set with the supervised one, changing
6195    /// nothing. The staged spec each supervised module holds is what the
6196    /// configured entry is compared with.
6197    fn plan_rescan(
6198        &self,
6199        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6200    ) -> Result<RescanPlan, String> {
6201        let mut current = BTreeMap::new();
6202        for module in self.supervisor.list() {
6203            let (spec, health) = module.configuration().map_err(|err| {
6204                format!(
6205                    "failed to read configuration for module_id '{}': {err}",
6206                    module.module_id()
6207                )
6208            })?;
6209            let enabled = module
6210                .status()
6211                .map_err(|err| {
6212                    format!(
6213                        "failed to read status for module_id '{}': {err}",
6214                        module.module_id()
6215                    )
6216                })?
6217                .enabled;
6218            current.insert(
6219                module.module_id().to_string(),
6220                (module, spec, health, enabled),
6221            );
6222        }
6223        let configured = configured_modules
6224            .into_iter()
6225            .map(|module| (module.module_id.clone(), module))
6226            .collect::<BTreeMap<_, _>>();
6227
6228        let added = configured
6229            .keys()
6230            .filter(|module_id| !current.contains_key(*module_id))
6231            .cloned()
6232            .collect::<Vec<_>>();
6233        let removed = current
6234            .keys()
6235            .filter(|module_id| !configured.contains_key(*module_id))
6236            .cloned()
6237            .collect::<Vec<_>>();
6238        let mut changed_pending_reload = Vec::new();
6239        let mut configuration_changes = BTreeSet::new();
6240        let mut enabled_changes = BTreeSet::new();
6241        let mut unchanged = 0_u32;
6242
6243        for (module_id, configured_module) in &configured {
6244            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6245            else {
6246                continue;
6247            };
6248            // Compare the whole launch spec so a future launch field cannot
6249            // accidentally become a live-only policy change. Health is stored
6250            // separately and applies live without replacing the process.
6251            let launch_changed = *current_spec != configured_module.module_spec();
6252            let configuration_changed =
6253                launch_changed || *current_health != configured_module.health;
6254            let enabled_changed = *current_enabled != configured_module.enabled;
6255            if configuration_changed {
6256                configuration_changes.insert(module_id.clone());
6257            }
6258            if launch_changed {
6259                changed_pending_reload.push(module_id.clone());
6260            }
6261            if enabled_changed {
6262                enabled_changes.insert(module_id.clone());
6263            }
6264            if !configuration_changed && !enabled_changed {
6265                unchanged = unchanged.saturating_add(1);
6266            }
6267        }
6268        Ok(RescanPlan {
6269            current,
6270            configured,
6271            added,
6272            removed,
6273            changed_pending_reload,
6274            configuration_changes,
6275            enabled_changes,
6276            unchanged,
6277        })
6278    }
6279
6280    /// Reconcile the running module set against the configured one.
6281    ///
6282    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6283    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6284    /// deliberately shares this function with the executing path rather than
6285    /// computing the same diff somewhere else -- two implementations of one
6286    /// decision agree until they do not, and the whole value of a preview is that
6287    /// it describes the operation that will actually run.
6288    async fn reconcile_supervised_modules(
6289        &self,
6290        supervisor: &Supervisor,
6291        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6292        preview: bool,
6293        requester: Principal,
6294    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6295        let plan = self.plan_rescan(configured_modules)?;
6296
6297        // Everything above this point is pure computation over two snapshots.
6298        // Everything below MUTATES. The preview returns here so the boundary is a
6299        // single early return rather than a condition repeated at each mutation
6300        // site, where one missed guard would apply part of a change the caller was
6301        // told would not happen.
6302        if preview {
6303            return Ok((plan.preview_result(), Vec::new()));
6304        }
6305        let changed = plan
6306            .configuration_changes
6307            .union(&plan.enabled_changes)
6308            .cloned()
6309            .collect::<Vec<_>>();
6310        let RescanPlan {
6311            current,
6312            configured,
6313            added,
6314            removed,
6315            changed_pending_reload,
6316            configuration_changes,
6317            enabled_changes,
6318            unchanged,
6319        } = plan;
6320        // The staged generation moves before the first change rather than
6321        // after the last, so a rescan that fails partway still invalidates
6322        // every earlier `supervisor.rescan_checked` preview: part of its diff
6323        // may have run.
6324        if !(added.is_empty()
6325            && removed.is_empty()
6326            && configuration_changes.is_empty()
6327            && enabled_changes.is_empty())
6328        {
6329            if let Some(context) = &self.rescan {
6330                context
6331                    .generation
6332                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6333            }
6334        }
6335
6336        for module_id in &removed {
6337            let module = &current
6338                .get(module_id)
6339                .expect("removed module came from current supervisor state")
6340                .0;
6341            module
6342                .retire_requested(Some(requester.clone()))
6343                .await
6344                .map_err(|err| {
6345                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6346                })?;
6347            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6348            //
6349            // `handle_route_open` resolves an absent module in three steps:
6350            // registry, then supervisor status, then tombstone. Retiring first
6351            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6352            // went with the teardown above, the supervisor entry went with
6353            // `retire`, and the tombstone does not exist yet -- so a route.open
6354            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6355            // it") for a module that was deliberately removed and whose caller
6356            // should get `module_removed` (TERMINAL, carrying a removal age).
6357            //
6358            // Writing the tombstone first closes it: during the window the
6359            // supervisor entry still answers, so the caller gets
6360            // `target_unavailable` -- retryable, and TRUE, because the module
6361            // is mid-teardown. After both statements it is `module_removed`.
6362            // No instant remains where a removed module reads as one that
6363            // never existed.
6364            //
6365            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6366            // the absence of a test beside a fix invites deletion: these are
6367            // two sync statements with no await between them, so reaching the
6368            // window needs a second worker thread to land exactly between them
6369            // and there is no hook to force it. MEASURED: the 25 daemon_config
6370            // tests pass identically with the old order and the new one, so
6371            // the existing suite cannot see this and a green run is not
6372            // evidence either way. What the suite does hold is the
6373            // post-condition -- a removed module answers `module_removed` --
6374            // which this preserves.
6375            //
6376            // Found by an Athena panel reading the shipped tree against a
6377            // design note (2026-09-19), as the one concrete instance of that
6378            // note's class that survived contact with source. Direction is
6379            // benign: retryable where terminal was intended, never the reverse.
6380            self.supervisor.record_rescan_removal(module_id);
6381            self.supervisor.retire(module_id);
6382            self.route_outages.forget(module_id);
6383        }
6384
6385        for module_id in configured.keys() {
6386            let Some((module, _, _, _)) = current.get(module_id) else {
6387                continue;
6388            };
6389            let configured_module = configured
6390                .get(module_id)
6391                .expect("configured module id came from configured map");
6392            if configuration_changes.contains(module_id) {
6393                module
6394                    .update_configuration(
6395                        configured_module.module_spec(),
6396                        configured_module.health.clone(),
6397                        configured_module.drain_timeout_ms,
6398                    )
6399                    .await
6400                    .map_err(|err| {
6401                        format!(
6402                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6403                        )
6404                    })?;
6405            }
6406            if enabled_changes.contains(module_id) {
6407                // A rescan that starts or stops a module applies an operator's
6408                // edit to the config, so the resulting outage was asked for.
6409                self.route_outages.mark_operator_action(module_id);
6410                module
6411                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6412                    .await
6413                    .map_err(|err| {
6414                        self.route_outages.operator_action_ended_unrefused(module_id);
6415                        format!(
6416                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6417                            configured_module.enabled
6418                        )
6419                    })?;
6420            }
6421        }
6422
6423        for module_id in &added {
6424            let configured_module = configured
6425                .get(module_id)
6426                .expect("added module id came from configured map");
6427            supervisor
6428                .supervise_configured_requested(
6429                    configured_module.module_spec(),
6430                    configured_module.enabled,
6431                    configured_module.health.clone(),
6432                    configured_module.drain_timeout_ms,
6433                    configured_module.restart,
6434                    configured_module.register_timeout,
6435                    Some(requester.clone()),
6436                )
6437                .map_err(|err| {
6438                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6439                })?;
6440        }
6441
6442        Ok((
6443            SupervisorRescanResult {
6444                added,
6445                removed,
6446                changed_pending_reload,
6447                enabled_changes: enabled_changes.iter().cloned().collect(),
6448                unchanged,
6449                preview: false,
6450                // Filled by the caller, which is the only layer that can see the
6451                // previous config to diff against.
6452                restart_required: Vec::new(),
6453                capability_warnings: Vec::new(),
6454                module_actions: Vec::new(),
6455                earlier_pending: Vec::new(),
6456                expect: None,
6457            },
6458            changed,
6459        ))
6460    }
6461
6462    async fn handle_supervisor_set_enabled(
6463        &self,
6464        frame: Frame,
6465        module_id: String,
6466        enabled: bool,
6467        requester: Principal,
6468    ) -> Result<Vec<Frame>, RouterError> {
6469        let operation_lock = self.supervisor.operation_lock();
6470        let _operation_guard = operation_lock.lock().await;
6471        let Some(module) = self.supervisor.get(&module_id) else {
6472            return Ok(vec![control_error_frame(
6473                &frame,
6474                "unknown_module",
6475                format!("module_id '{module_id}' is not supervised"),
6476            )?]);
6477        };
6478
6479        // Enabling counts as well as disabling: a module an operator starts
6480        // is refused until it registers, and that wait was asked for.
6481        self.route_outages.mark_operator_action(&module_id);
6482        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6483            Ok(applied) => applied,
6484            Err(err) => {
6485                self.route_outages
6486                    .operator_action_ended_unrefused(&module_id);
6487                return Ok(vec![control_error_frame(
6488                    &frame,
6489                    "target_unavailable",
6490                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6491                )?]);
6492            }
6493        };
6494        if !applied {
6495            // Already in the requested state: nothing was made unavailable,
6496            // so the mark must not outlive this request.
6497            self.route_outages
6498                .operator_action_ended_unrefused(&module_id);
6499        }
6500
6501        self.capability_evaluator.wake_deadline_loop();
6502        self.refresh_capability_requirements();
6503        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6504        Ok(vec![control_response_body_frame(
6505            &frame,
6506            &response,
6507            "ClientControlResponse::SupervisorAck",
6508        )?])
6509    }
6510
6511    async fn handle_supervisor_health_probe(
6512        &self,
6513        frame: Frame,
6514        module_id: String,
6515    ) -> Result<Vec<Frame>, RouterError> {
6516        self.refresh_capability_requirements();
6517        let Some(registration) = self
6518            .registry
6519            .get_module(&module_id)
6520            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6521        else {
6522            return Ok(vec![control_error_frame(
6523                &frame,
6524                "unknown_module",
6525                format!("module_id '{module_id}' is not registered"),
6526            )?]);
6527        };
6528
6529        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6530        // named for it. Making `module_registration_grants_op` return false
6531        // unconditionally reddens five tests, and every one is named for something
6532        // else -- capability relay, probe/bind demultiplexing, supervision-only
6533        // probing. They exercise a successful advertisement check on the way to their
6534        // own subject.
6535        //
6536        // Real protection, fragile in a specific way: narrowing any of those tests to
6537        // focus on its stated subject would silently remove coverage nobody knows
6538        // they are carrying. Recorded here rather than as a sixth test, because the
6539        // useful fact is WHICH tests hold the guard up -- a new test would add
6540        // coverage without telling the next person what the existing ones quietly do.
6541        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6542        {
6543            return Ok(vec![control_error_frame(
6544                &frame,
6545                "health_not_advertised",
6546                format!("module_id '{module_id}' did not advertise health.check"),
6547            )?]);
6548        }
6549
6550        let deadline = Instant::now() + self.health_probe_timeout;
6551        let pending = match self.forwarding.begin_module_control_rpc_for(
6552            &module_id,
6553            MODULE_CONTROL_OP_HEALTH_CHECK,
6554            deadline,
6555        ) {
6556            Ok(pending) => pending,
6557            Err(err) => {
6558                return Ok(vec![control_error_frame(
6559                    &frame,
6560                    forwarding_error_code(&err),
6561                    err.to_string(),
6562                )?])
6563            }
6564        };
6565
6566        let PendingModuleControlRpc {
6567            endpoint,
6568            module_sink,
6569            negotiated_ver,
6570            corr: probe_corr,
6571            receiver,
6572        } = pending;
6573        let mut guard =
6574            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6575        let probe_body =
6576            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6577                RouterError::backend(
6578                    0,
6579                    frame.header.corr,
6580                    format!("failed to encode health.check request: {err}"),
6581                )
6582            })?;
6583        let probe_frame = Frame::build_with_version(
6584            negotiated_ver,
6585            FrameType::Request,
6586            control_flags(),
6587            0,
6588            0,
6589            probe_corr,
6590            probe_body,
6591        )
6592        .map_err(RouterError::FrameBuild)?;
6593
6594        // Queue admission and the reply share one deadline, so a module that
6595        // stops reading cannot park the requesting connection's reader forever.
6596        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6597            Ok(Ok(())) => {}
6598            Ok(Err(err)) => {
6599                return Ok(vec![control_error_frame(
6600                    &frame,
6601                    "target_unavailable",
6602                    err.to_string(),
6603                )?]);
6604            }
6605            Err(_) => {
6606                return Ok(vec![control_error_frame(
6607                    &frame,
6608                    "module_timeout",
6609                    format!(
6610                        "module_id '{module_id}' did not answer health.check within {:?}",
6611                        self.health_probe_timeout
6612                    ),
6613                )?]);
6614            }
6615        }
6616
6617        match timeout_at(deadline, receiver).await {
6618            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6619                guard.disarm();
6620                let Some(report) = response.health_report() else {
6621                    return Ok(vec![control_error_frame(
6622                        &frame,
6623                        "invalid_control_body",
6624                        "health.check RPC returned a non-health response",
6625                    )?]);
6626                };
6627                // Metrics go out whole here. The supervisor's cached snapshot
6628                // caps this blob (see truncate_health_metrics), and this path
6629                // exists precisely to answer without that cap -- so applying it
6630                // here would leave no way to see what the cached view drops.
6631                let HealthReport {
6632                    status,
6633                    detail,
6634                    metrics,
6635                } = report;
6636                let capability_detail = self
6637                    .capability_evaluator
6638                    .required_problem_detail(&module_id);
6639                let response = ClientControlResponse::SupervisorHealthProbe {
6640                    module_id,
6641                    status,
6642                    detail: append_capability_problem_detail(detail, capability_detail),
6643                    metrics,
6644                };
6645                Ok(vec![control_response_body_frame(
6646                    &frame,
6647                    &response,
6648                    "ClientControlResponse::SupervisorHealthProbe",
6649                )?])
6650            }
6651            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6652                guard.disarm();
6653                Ok(vec![control_error_body_frame(&frame, body)?])
6654            }
6655            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6656                guard.disarm();
6657                Ok(vec![control_error_frame(
6658                    &frame,
6659                    "target_unavailable",
6660                    message,
6661                )?])
6662            }
6663            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6664                guard.disarm();
6665                Ok(vec![control_error_frame(
6666                    &frame,
6667                    "invalid_control_body",
6668                    message,
6669                )?])
6670            }
6671            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6672                guard.disarm();
6673                Ok(vec![control_error_frame(
6674                    &frame,
6675                    "invalid_control_body",
6676                    format!("expected module-control op '{expected}', got '{actual}'"),
6677                )?])
6678            }
6679            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6680                guard.disarm();
6681                Ok(vec![control_error_frame(
6682                    &frame,
6683                    "module_timeout",
6684                    format!(
6685                        "module_id '{module_id}' answered health.check after {:?}",
6686                        self.health_probe_timeout
6687                    ),
6688                )?])
6689            }
6690            Ok(Err(_)) => Ok(vec![control_error_frame(
6691                &frame,
6692                "target_unavailable",
6693                "health.check waiter was canceled before the module responded",
6694            )?]),
6695            Err(_) => Ok(vec![control_error_frame(
6696                &frame,
6697                "module_timeout",
6698                format!(
6699                    "module_id '{module_id}' did not answer health.check within {:?}",
6700                    self.health_probe_timeout
6701                ),
6702            )?]),
6703        }
6704    }
6705
6706    fn supervisor_status(
6707        &self,
6708        module_id: &str,
6709        corr: u64,
6710    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6711        self.supervisor
6712            .get(module_id)
6713            .map(|module| {
6714                let warming = module.is_warming_for_control("status").map_err(|err| {
6715                    RouterError::backend(
6716                        0,
6717                        corr,
6718                        format!(
6719                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6720                        ),
6721                    )
6722                })?;
6723                module.status_for_control("status").map_err(|err| {
6724                    RouterError::backend(
6725                        0,
6726                        corr,
6727                        format!(
6728                            "failed to read supervisor status for module_id '{module_id}': {err}"
6729                        ),
6730                    )
6731                }).map(|status| (status, warming))
6732            })
6733            .transpose()
6734    }
6735
6736    fn guard_module_control_op(
6737        &self,
6738        frame: &Frame,
6739        module_id: &str,
6740        op: &str,
6741    ) -> Result<Option<Frame>, RouterError> {
6742        if self.module_grants_op(module_id, op, frame.header.corr)? {
6743            return Ok(None);
6744        }
6745
6746        Ok(Some(control_error_frame(
6747            frame,
6748            "op_not_allowed",
6749            format!("module_id '{module_id}' did not grant control op '{op}'"),
6750        )?))
6751    }
6752
6753    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6754        let Some(registration) = self
6755            .registry
6756            .get_module(module_id)
6757            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6758        else {
6759            return Ok(false);
6760        };
6761        Ok(module_registration_grants_op(&registration.control_ops, op))
6762    }
6763
6764    fn handle_status_update(
6765        &self,
6766        endpoint: ModuleEndpointId,
6767        frame: Frame,
6768    ) -> Result<Vec<Frame>, RouterError> {
6769        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6770            Ok(update) => update,
6771            Err(err) => {
6772                // Forward-compat: a newer module may push a channel-0 op this subc
6773                // version doesn't know. The control contract says unknown push ops
6774                // are IGNORED, never answered with an error. Only a malformed body
6775                // for an op we DO know is a real error worth surfacing.
6776                if is_known_module_push_op(&frame.body) {
6777                    return Ok(vec![control_error_frame(
6778                        &frame,
6779                        "invalid_control_body",
6780                        format!("malformed module control push body: {err}"),
6781                    )?]);
6782                }
6783                return Ok(Vec::new());
6784            }
6785        };
6786
6787        match update {
6788            ModuleControlPush::RouteStatus {
6789                route_channel,
6790                route_epoch,
6791                status,
6792            } => {
6793                self.forwarding
6794                    .cache_status(endpoint, route_channel, route_epoch, status)
6795                    .map_err(RouterError::Forwarding)?;
6796            }
6797        }
6798        Ok(Vec::new())
6799    }
6800
6801    fn handle_route_poll(
6802        &self,
6803        ctx: &RouteCtx,
6804        frame: Frame,
6805        route_channel: u16,
6806        route_epoch: u32,
6807        kind: PollKind,
6808    ) -> Result<Vec<Frame>, RouterError> {
6809        let snapshot = self
6810            .forwarding
6811            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6812            .map_err(RouterError::Forwarding)?;
6813        let response = match (kind, snapshot) {
6814            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6815                ClientControlResponse::RoutePoll {
6816                    route_channel,
6817                    route_epoch,
6818                    status,
6819                    live: None,
6820                }
6821            }
6822            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6823                route_channel,
6824                route_epoch,
6825                status: None,
6826                live: None,
6827            },
6828            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6829                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6830                // is what makes reporting `true` correct rather than a
6831                // confident guess. `process_live` returns None only when the
6832                // module id has no supervisor snapshot at all -- an
6833                // externally-started module the daemon did not spawn -- and
6834                // for those the supervisor has no opinion to offer, ever. It
6835                // is never None for a supervised module in an unknown state:
6836                // a supervised module always has a snapshot, and the answer
6837                // comes from `state == Running && process_alive`.
6838                //
6839                // The route is Bound, so the module completed a HELLO on a
6840                // live connection; "the process this route points at is
6841                // running" is therefore attested by the binding rather than
6842                // assumed. Reporting `false` for an unsupervised module would
6843                // be the actual lie -- it would tell a client its healthy
6844                // route is dead because the daemon does not manage the
6845                // process.
6846                //
6847                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6848                // module whose liveness is genuinely unknown, e.g. a snapshot
6849                // that has not been populated yet -- THIS DEFAULT BECOMES
6850                // WRONG and must split: unsupervised stays true, unknown
6851                // becomes null so the client can tell the two apart. The
6852                // response field is already `Option<bool>`, so the wire can
6853                // carry that distinction today.
6854                let live = self
6855                    .process_liveness
6856                    .as_ref()
6857                    .and_then(|source| source.process_live(&module_id))
6858                    .unwrap_or(true);
6859                ClientControlResponse::RoutePoll {
6860                    route_channel,
6861                    route_epoch,
6862                    status: None,
6863                    live: Some(live),
6864                }
6865            }
6866            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6867                route_channel,
6868                route_epoch,
6869                status: None,
6870                live: Some(false),
6871            },
6872        };
6873
6874        Ok(vec![control_response_body_frame(
6875            &frame,
6876            &response,
6877            "ClientControlResponse::RoutePoll",
6878        )?])
6879    }
6880
6881    pub(crate) fn observe_module_control_completion(
6882        &self,
6883        completion: ModuleControlRpcCompletion,
6884    ) -> bool {
6885        match completion {
6886            ModuleControlRpcCompletion::Unknown => false,
6887            ModuleControlRpcCompletion::Settled => true,
6888            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6889                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6890                info!(
6891                    module_id = %module_id,
6892                    latency_ms,
6893                    "late health.check answer proves the module is alive"
6894                );
6895                match self
6896                    .supervisor
6897                    .record_late_health_answer(&module_id, latency_ms)
6898                {
6899                    Ok(true) => {}
6900                    Ok(false) => debug!(
6901                        module_id = %module_id,
6902                        latency_ms,
6903                        "late health.check answer has no active supervisor snapshot"
6904                    ),
6905                    Err(err) => warn!(
6906                        module_id = %module_id,
6907                        latency_ms,
6908                        error = %err,
6909                        "failed to record late health.check answer"
6910                    ),
6911                }
6912                true
6913            }
6914        }
6915    }
6916
6917    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6918    /// the module connection whose frame is being handled, or to the client that
6919    /// relay was opened for.
6920    ///
6921    /// This runs on the MODULE connection's frame handler, where returning `Err`
6922    /// ends that connection -- and a module connection carries every client's
6923    /// routes to that module, so ending it costs the whole fleet its tools.
6924    /// `ConnectionClosing` carries the id of the connection that is closing, and
6925    /// when that id is a CLIENT's, the condition is entirely about that one
6926    /// client's route.open. A client-scoped condition has no authority over a
6927    /// shared module connection, so it is logged and the single relay is dropped:
6928    /// the client is going away, and `complete_pending_relay` already removed the
6929    /// relay before failing, so there is nothing left to settle. Anything that
6930    /// relay still reserved is released by that client's own connection teardown,
6931    /// which is already under way -- that is what "closing" means.
6932    ///
6933    /// Every other failure is a statement about THIS connection and stays fatal:
6934    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6935    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6936    /// frames correctly.
6937    fn refuse_to_end_module_connection_for_a_client(
6938        &self,
6939        module_connection_id: ConnectionId,
6940        corr: u64,
6941        err: ForwardingError,
6942    ) -> Result<(), RouterError> {
6943        if let ForwardingError::ConnectionClosing { connection_id } = err {
6944            if connection_id != module_connection_id {
6945                warn!(
6946                    module_connection_id = module_connection_id.get(),
6947                    client_connection_id = connection_id.get(),
6948                    corr,
6949                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6950                );
6951                return Ok(());
6952            }
6953        }
6954        Err(RouterError::Forwarding(err))
6955    }
6956
6957    fn handle_module_relay_response(
6958        &self,
6959        connection_id: ConnectionId,
6960        frame: Frame,
6961    ) -> Result<Vec<Frame>, RouterError> {
6962        let mut secondary_error = None;
6963        let outcome = match frame.header.ty {
6964            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6965                Ok(probe) if probe.op == "route.bind" => {
6966                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6967                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6968                            RouteBindRelayOutcome::Accepted
6969                        }
6970                        Ok(other) => {
6971                            let message =
6972                                format!("route.bind response carried unexpected body: {other:?}");
6973                            secondary_error = Some(control_error_frame(
6974                                &frame,
6975                                "invalid_control_body",
6976                                message.clone(),
6977                            )?);
6978                            RouteBindRelayOutcome::ModuleGone(message)
6979                        }
6980                        Err(err) => {
6981                            let message = format!("malformed route.bind response body: {err}");
6982                            secondary_error = Some(control_error_frame(
6983                                &frame,
6984                                "invalid_control_body",
6985                                message.clone(),
6986                            )?);
6987                            RouteBindRelayOutcome::ModuleGone(message)
6988                        }
6989                    }
6990                }
6991                Ok(probe) => {
6992                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6993                    {
6994                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6995                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6996                            "malformed {} response body: {err}",
6997                            probe.op
6998                        )),
6999                    };
7000                    let completion = self
7001                        .forwarding
7002                        .complete_module_control_rpc(
7003                            connection_id,
7004                            frame.header.corr,
7005                            Some(&probe.op),
7006                            outcome,
7007                        )
7008                        .map_err(RouterError::Forwarding)?;
7009                    if !self.observe_module_control_completion(completion) {
7010                        debug!(
7011                            connection_id = connection_id.get(),
7012                            corr = frame.header.corr,
7013                            op = %probe.op,
7014                            "dropping late or unknown module-control RPC response"
7015                        );
7016                    }
7017                    return Ok(Vec::new());
7018                }
7019                Err(err) => {
7020                    if let Some(expected_op) = self
7021                        .forwarding
7022                        .pending_module_control_op(connection_id, frame.header.corr)
7023                        .map_err(RouterError::Forwarding)?
7024                    {
7025                        let completion = self
7026                            .forwarding
7027                            .complete_module_control_rpc(
7028                                connection_id,
7029                                frame.header.corr,
7030                                None,
7031                                ModuleControlRpcOutcome::MalformedResponse(format!(
7032                                    "malformed {expected_op} response body: {err}"
7033                                )),
7034                            )
7035                            .map_err(RouterError::Forwarding)?;
7036                        if !self.observe_module_control_completion(completion) {
7037                            debug!(
7038                                connection_id = connection_id.get(),
7039                                corr = frame.header.corr,
7040                                "dropping late malformed module-control RPC response"
7041                            );
7042                        }
7043                        return Ok(Vec::new());
7044                    }
7045                    let message = format!("malformed route.bind response body: {err}");
7046                    secondary_error = Some(control_error_frame(
7047                        &frame,
7048                        "invalid_control_body",
7049                        message.clone(),
7050                    )?);
7051                    RouteBindRelayOutcome::ModuleGone(message)
7052                }
7053            },
7054            FrameType::Error => {
7055                if self
7056                    .forwarding
7057                    .pending_module_control_op(connection_id, frame.header.corr)
7058                    .map_err(RouterError::Forwarding)?
7059                    .is_some()
7060                {
7061                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7062                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7063                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7064                            "malformed module-control ERROR body: {err}"
7065                        )),
7066                    };
7067                    let completion = self
7068                        .forwarding
7069                        .complete_module_control_rpc(
7070                            connection_id,
7071                            frame.header.corr,
7072                            None,
7073                            outcome,
7074                        )
7075                        .map_err(RouterError::Forwarding)?;
7076                    if !self.observe_module_control_completion(completion) {
7077                        debug!(
7078                            connection_id = connection_id.get(),
7079                            corr = frame.header.corr,
7080                            "dropping late or unknown module-control RPC error"
7081                        );
7082                    }
7083                    return Ok(Vec::new());
7084                }
7085                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7086                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7087                    Err(err) => {
7088                        let message = format!("malformed route.bind ERROR body: {err}");
7089                        secondary_error = Some(control_error_frame(
7090                            &frame,
7091                            "invalid_control_body",
7092                            message.clone(),
7093                        )?);
7094                        RouteBindRelayOutcome::ModuleGone(message)
7095                    }
7096                }
7097            }
7098            ty => {
7099                return Ok(vec![control_error_frame(
7100                    &frame,
7101                    "unsupported_control_frame",
7102                    format!("unsupported module channel-0 frame {ty:?}"),
7103                )?])
7104            }
7105        };
7106
7107        let settled =
7108            self.forwarding
7109                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7110        let completion = match settled {
7111            Ok(completion) => completion,
7112            Err(err) => {
7113                self.refuse_to_end_module_connection_for_a_client(
7114                    connection_id,
7115                    frame.header.corr,
7116                    err,
7117                )?;
7118                return Ok(secondary_error.into_iter().collect());
7119            }
7120        };
7121        if let Some(target) = completion.abandoned.as_ref() {
7122            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7123        }
7124        if !completion.settled {
7125            debug!(
7126                connection_id = connection_id.get(),
7127                corr = frame.header.corr,
7128                frame_type = ?frame.header.ty,
7129                "dropping late or unknown route.bind relay response"
7130            );
7131        }
7132        Ok(secondary_error.into_iter().collect())
7133    }
7134
7135    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7136        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7137        // GOODBYE ends the connection's logical session even when its socket
7138        // stays open. Use disconnect teardown so verdicts, client notices and
7139        // scope authority are released at the same lifecycle boundary.
7140        self.cleanup_connection_with_end_reason(
7141            connection_id,
7142            RegistrationEndReason::ExplicitGoodbye,
7143        )
7144        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7145        Ok(Vec::new())
7146    }
7147}
7148
7149impl Default for ControlHandler {
7150    fn default() -> Self {
7151        Self::new(Arc::new(Registry::default()))
7152    }
7153}
7154
7155impl crate::supervise::SwapPromotionObserver for ControlHandler {
7156    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7157        self.apply_registration_capabilities(registration);
7158    }
7159}
7160
7161fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7162    CapabilityRequirementStatus {
7163        consumer: status.consumer,
7164        capability: status.capability,
7165        need: match status.need {
7166            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7167            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7168        },
7169        verdict: status.verdict.as_str().to_string(),
7170        episode_seq: status.episode_seq,
7171        config_satisfiable: status.config_satisfiable,
7172        runtime_available: status.runtime_available,
7173        detail: status.detail,
7174    }
7175}
7176
7177fn append_capability_problem_detail(
7178    detail: Option<String>,
7179    capability_detail: Option<String>,
7180) -> Option<String> {
7181    match (detail, capability_detail) {
7182        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7183        (Some(detail), None) => Some(detail),
7184        (None, Some(capability_detail)) => Some(capability_detail),
7185        (None, None) => None,
7186    }
7187}
7188
7189fn subc_ops() -> Vec<String> {
7190    SUBC_CONTROL_OPS
7191        .iter()
7192        .map(|op| (*op).to_string())
7193        .collect()
7194}
7195
7196fn module_subc_ops() -> Vec<String> {
7197    SUBC_CONTROL_OPS
7198        .iter()
7199        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7200        .map(|op| (*op).to_string())
7201        .collect()
7202}
7203
7204#[cfg(test)]
7205fn module_baseline_control_ops() -> Vec<String> {
7206    MODULE_BASELINE_CONTROL_OPS
7207        .iter()
7208        .map(|op| (*op).to_string())
7209        .collect()
7210}
7211
7212fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7213    let mut seen = HashSet::new();
7214    let mut effective = Vec::new();
7215    for op in MODULE_BASELINE_CONTROL_OPS {
7216        if seen.insert((*op).to_string()) {
7217            effective.push((*op).to_string());
7218        }
7219    }
7220    for op in declared.unwrap_or_default() {
7221        if seen.insert(op.clone()) {
7222            effective.push(op);
7223        }
7224    }
7225    effective
7226}
7227
7228fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7229    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7230}
7231
7232fn target_module_id(target: &RouteTarget) -> &str {
7233    match target {
7234        RouteTarget::ToolProvider { module_id }
7235        | RouteTarget::ManagementSurface { module_id }
7236        | RouteTarget::InternalService { module_id, .. } => module_id,
7237    }
7238}
7239
7240fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7241    roles.iter().any(|role| match (target, role) {
7242        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7243        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7244        (
7245            RouteTarget::InternalService { service_id, .. },
7246            ProviderRole::InternalService {
7247                service_id: provided,
7248                ..
7249            },
7250        ) => service_id == provided,
7251        _ => false,
7252    })
7253}
7254
7255fn is_routable_role(role: &ProviderRole) -> bool {
7256    matches!(
7257        role,
7258        ProviderRole::ToolProvider { .. }
7259            | ProviderRole::ManagementSurface { .. }
7260            | ProviderRole::InternalService { .. }
7261    )
7262}
7263
7264#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7265enum ControlRequestBodyError {
7266    UnknownOp,
7267    InvalidBody,
7268}
7269
7270#[derive(Debug, Deserialize)]
7271struct ControlOpProbe {
7272    op: String,
7273}
7274
7275/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7276/// this set is treated as a forward-compat unknown and ignored rather than errored.
7277const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7278
7279fn is_known_module_push_op(body: &[u8]) -> bool {
7280    serde_json::from_slice::<ControlOpProbe>(body)
7281        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7282        .unwrap_or(false)
7283}
7284
7285fn is_known_module_request_op(body: &[u8]) -> bool {
7286    serde_json::from_slice::<ControlOpProbe>(body)
7287        .map(|probe| is_module_to_subc_op(&probe.op))
7288        .unwrap_or(false)
7289}
7290
7291fn is_module_to_subc_op(op: &str) -> bool {
7292    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7293}
7294
7295fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7296    debug!(
7297        op = %op,
7298        connection_id = connection_id.get(),
7299        corr,
7300        "control dispatch"
7301    );
7302}
7303
7304fn log_slow_control_dispatch(
7305    dispatch_started_at: Option<StdInstant>,
7306    op: &'static str,
7307    connection_id: ConnectionId,
7308    corr: u64,
7309    route_open_timings: Option<&RouteOpenTimings>,
7310) {
7311    let Some(dispatch_started_at) = dispatch_started_at else {
7312        return;
7313    };
7314    // Use the same monotonic clock as the phase timers and relay deadline.
7315    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7316    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7317        if let Some(timings) = route_open_timings {
7318            warn!(
7319                op = %op,
7320                connection_id = connection_id.get(),
7321                corr,
7322                elapsed_ms = elapsed.as_millis() as u64,
7323                module_id = ?timings.module_id,
7324                principal = %timings.principal,
7325                admission_ms = timings.admission.as_millis() as u64,
7326                project_root_ms = timings.project_root.as_millis() as u64,
7327                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7328                bind_send_ms = timings.bind_send.as_millis() as u64,
7329                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7330                "slow control dispatch"
7331            );
7332            return;
7333        }
7334        warn!(
7335            op = %op,
7336            connection_id = connection_id.get(),
7337            corr,
7338            elapsed_ms = elapsed.as_millis() as u64,
7339            "slow control dispatch"
7340        );
7341    }
7342}
7343
7344/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7345#[cfg(test)]
7346type TestCallerLookup =
7347    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7348
7349/// Every new client operation needs an explicit privilege decision here.
7350fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7351    match request {
7352        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7353            Some(PrivilegedOperation::Restart { module_id })
7354        }
7355        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7356            Some(PrivilegedOperation::Swap { module_id })
7357        }
7358        ClientControlRequest::SupervisorReload { module_id } => {
7359            Some(PrivilegedOperation::Reload { module_id })
7360        }
7361        ClientControlRequest::SupervisorRescan { preview } => {
7362            if *preview {
7363                None
7364            } else {
7365                Some(PrivilegedOperation::Rescan)
7366            }
7367        }
7368        // Only the well-formed apply asks for approval. A preview changes
7369        // nothing, and any other combination is refused before it touches
7370        // anything, so prompting the operator for it would ask about a request
7371        // that cannot run.
7372        ClientControlRequest::SupervisorRescanChecked {
7373            preview,
7374            candidate,
7375            expect,
7376        } => (!*preview && candidate.is_none() && expect.is_some())
7377            .then_some(PrivilegedOperation::Rescan),
7378        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7379            Some(PrivilegedOperation::SetEnabled {
7380                module_id,
7381                enabled: *enabled,
7382            })
7383        }
7384        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7385            Some(PrivilegedOperation::ReleaseReserved { module_id })
7386        }
7387        ClientControlRequest::ServerDescribe {}
7388        | ClientControlRequest::CatalogList { .. }
7389        | ClientControlRequest::RouteOpen { .. }
7390        | ClientControlRequest::RoutePoll { .. }
7391        | ClientControlRequest::SupervisorList {}
7392        | ClientControlRequest::SupervisorHealth {}
7393        | ClientControlRequest::SupervisorHealthProbe { .. }
7394        | ClientControlRequest::SupervisorRoutes { .. }
7395        | ClientControlRequest::SupervisorProvenance { .. }
7396        | ClientControlRequest::SupervisorStderrTail { .. }
7397        | ClientControlRequest::SupervisorTerminals { .. }
7398        | ClientControlRequest::SupervisorSpawnSnapshot {}
7399        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7400    }
7401}
7402
7403fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7404    match request {
7405        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7406        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7407        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7408        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7409        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7410        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7411        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7412        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7413        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7414        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7415        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7416        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7417        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7418        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7419        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7420        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7421        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7422        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7423        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7424        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7425    }
7426}
7427
7428/// The caller of one `route.open`, as its `route.open refused` line names it.
7429///
7430/// A connection id alone cannot name a caller after the fact: ids restart at
7431/// every daemon restart, so the same number belongs to different modules
7432/// before and after one. These three fields are what the refused line needs
7433/// to be attributable on its own, and they are rendered with the same helpers
7434/// as the `route.open accepted` line so both directions grep alike.
7435struct RouteOpenCaller {
7436    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
7437    /// when the request presented a consumer identity whose launch nonce did
7438    /// not verify. The claimed module id is never printed as a principal.
7439    principal: String,
7440    /// `none` for an unscoped open; the admitted stamp's label once scope
7441    /// admission issued one; before that, the requested selector's label.
7442    scope: String,
7443    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
7444    /// because it is a string the caller chose.
7445    harness: String,
7446}
7447
7448impl RouteOpenCaller {
7449    fn new(
7450        principal: Option<&Principal>,
7451        scope: Option<&ScopeSelector>,
7452        identity: &BindIdentity,
7453    ) -> Self {
7454        Self {
7455            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7456            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7457            harness: bounded_log_text(&identity.harness),
7458        }
7459    }
7460
7461    /// For a refusal that has no parsed request at hand: names whatever the
7462    /// frame still says, and `unknown` where it says nothing parseable.
7463    fn unknown() -> Self {
7464        Self {
7465            principal: "unknown".to_string(),
7466            scope: "unknown".to_string(),
7467            harness: "unknown".to_string(),
7468        }
7469    }
7470}
7471
7472/// Longest client-chosen text (a scope ref, a harness) a route.open log line
7473/// prints before cutting it.
7474const MAX_LOGGED_TEXT_CHARS: usize = 64;
7475
7476/// `value` Debug-quoted, so control characters land escaped and cannot forge
7477/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
7478/// `…` inside the quotes so one request cannot flood the line.
7479fn bounded_log_text(value: &str) -> String {
7480    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7481    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7482        bounded.push('…');
7483    }
7484    format!("{bounded:?}")
7485}
7486
7487/// The `scope=` field of the `route.open accepted` line, from the stamp that
7488/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7489/// so a ref holding control characters cannot forge a log line, and cut to 64
7490/// characters with a trailing `…` to bound the line. The bind proof and the
7491/// scope's attributes never appear.
7492fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7493    use subc_protocol::scope::ScopeKind;
7494    let kind = match stamp.kind {
7495        ScopeKind::Head => "head",
7496        ScopeKind::Worker => "worker",
7497        ScopeKind::Ephemeral => "ephemeral",
7498    };
7499    format!(
7500        "{kind}/{}/{}@{}",
7501        principal_label(&stamp.owner),
7502        bounded_log_text(&stamp.scope_ref),
7503        stamp.scope_epoch
7504    )
7505}
7506
7507/// The `scope=` field of a `route.open refused` line for an open refused
7508/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
7509/// The kind is not known yet, so the literal `requested` stands in its place.
7510/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
7511/// owner it has not been checked against the configured modules: a quoted
7512/// owner means the value was never validated. The bind proof never appears.
7513fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7514    let epoch = selector
7515        .scope_epoch
7516        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7517    format!(
7518        "requested/{}/{}@{epoch}",
7519        bounded_log_text(&principal_label(&selector.owner)),
7520        bounded_log_text(&selector.scope_ref),
7521    )
7522}
7523
7524fn principal_label(principal: &Principal) -> String {
7525    match principal {
7526        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7527        Principal::Direct => "direct".to_string(),
7528        other => format!("{other:?}"),
7529    }
7530}
7531
7532fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7533    match request {
7534        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7535        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7536        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7537        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7538        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7539    }
7540}
7541
7542fn parse_client_control_request(
7543    body: &[u8],
7544) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7545    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7546        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7547            Ok(probe)
7548                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7549                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7550            {
7551                ControlRequestBodyError::InvalidBody
7552            }
7553            Ok(_) => ControlRequestBodyError::UnknownOp,
7554            Err(_) => ControlRequestBodyError::InvalidBody,
7555        };
7556        (err, classification)
7557    })
7558}
7559
7560fn parse_module_control_request_from_module(
7561    body: &[u8],
7562) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7563    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7564        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7565            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7566            Ok(_) => ControlRequestBodyError::UnknownOp,
7567            Err(_) => ControlRequestBodyError::InvalidBody,
7568        };
7569        (err, classification)
7570    })
7571}
7572
7573#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7574enum ProviderRoleKind {
7575    ToolProvider,
7576    PipelineStage,
7577    ManagementSurface,
7578    InternalService,
7579}
7580
7581fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7582    match role {
7583        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7584        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7585        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7586        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7587    }
7588}
7589
7590fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7591    roles.iter().map(provider_role_kind).collect()
7592}
7593
7594/// Most refused scope records named individually in the log per sync; the
7595/// `refused` count on the accepted line is always complete.
7596const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7597
7598/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7599#[derive(Debug, Default, PartialEq, Eq)]
7600struct ScopeOutcomeCounts {
7601    created: usize,
7602    replaced: usize,
7603    updated: usize,
7604    unchanged: usize,
7605    refused: usize,
7606}
7607
7608impl ScopeOutcomeCounts {
7609    fn of(results: &[ScopeRecordResult]) -> Self {
7610        let mut counts = Self::default();
7611        for result in results {
7612            let slot = match result.outcome {
7613                ScopeRecordOutcome::Created => &mut counts.created,
7614                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7615                ScopeRecordOutcome::Updated => &mut counts.updated,
7616                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7617                ScopeRecordOutcome::Refused => &mut counts.refused,
7618            };
7619            *slot += 1;
7620        }
7621        counts
7622    }
7623}
7624
7625#[cfg(test)]
7626mod scope_outcome_count_tests {
7627    use super::*;
7628
7629    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7630        ScopeRecordResult {
7631            scope_ref: "r".to_string(),
7632            scope_epoch: 1,
7633            outcome,
7634            code: None,
7635            message: None,
7636            version: None,
7637            parent_state: None,
7638        }
7639    }
7640
7641    /// Each outcome lands in its own count, so a refused record can never be
7642    /// hidden inside the total the log already printed.
7643    #[test]
7644    fn every_outcome_is_counted_in_its_own_field() {
7645        let results = [
7646            result(ScopeRecordOutcome::Created),
7647            result(ScopeRecordOutcome::Created),
7648            result(ScopeRecordOutcome::Replaced),
7649            result(ScopeRecordOutcome::Updated),
7650            result(ScopeRecordOutcome::Unchanged),
7651            result(ScopeRecordOutcome::Refused),
7652            result(ScopeRecordOutcome::Refused),
7653            result(ScopeRecordOutcome::Refused),
7654        ];
7655        assert_eq!(
7656            ScopeOutcomeCounts::of(&results),
7657            ScopeOutcomeCounts {
7658                created: 2,
7659                replaced: 1,
7660                updated: 1,
7661                unchanged: 1,
7662                refused: 3,
7663            }
7664        );
7665    }
7666}
7667
7668/// Return whether a catalog change can create a newly violating live route.
7669/// Removing an attested claim is intentionally excluded: it makes fewer routes
7670/// forbidden and therefore must leave the existing route census untouched.
7671fn capability_census_trigger(
7672    old: Option<&CapabilityDeclarations>,
7673    new: Option<&CapabilityDeclarations>,
7674) -> bool {
7675    let old_provides = old
7676        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7677        .unwrap_or_default();
7678    let old_denies = old
7679        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7680        .unwrap_or_default();
7681    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7682        provides: Vec::new(),
7683        requires: Vec::new(),
7684        must_never_reach: Vec::new(),
7685    });
7686
7687    new.provides
7688        .iter()
7689        .any(|capability| !old_provides.contains(capability))
7690        || new
7691            .must_never_reach
7692            .iter()
7693            .any(|capability| !old_denies.contains(capability))
7694}
7695
7696/// Find the first capability an attested opener denies that an attested target
7697/// claims. Both manifests are live registry records, never cached or client data.
7698fn denied_capability<'a>(
7699    opening_manifest: &'a ModuleManifest,
7700    target_manifest: &ModuleManifest,
7701) -> Option<&'a str> {
7702    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7703    let target_capabilities = target_manifest.capabilities.as_ref()?;
7704    opening_capabilities
7705        .must_never_reach
7706        .iter()
7707        .find(|denied| {
7708            target_capabilities
7709                .provides
7710                .iter()
7711                .any(|provided| provided == *denied)
7712        })
7713        .map(String::as_str)
7714}
7715
7716fn catalog_update_frozen_field_message(
7717    registered: &ModuleManifest,
7718    provides: &[ProviderRole],
7719) -> Option<String> {
7720    let old_has_provides = !registered.provides.is_empty();
7721    let new_has_provides = !provides.is_empty();
7722    if old_has_provides != new_has_provides {
7723        return Some(format!(
7724            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7725            registered.module_id
7726        ));
7727    }
7728
7729    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7730        return Some(format!(
7731            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7732            registered.module_id
7733        ));
7734    }
7735
7736    let registered_concurrency = manifest_concurrency(registered);
7737    let mut candidate = registered.clone();
7738    candidate.provides = provides.to_vec();
7739    let candidate_concurrency = manifest_concurrency(&candidate);
7740    if candidate_concurrency != registered_concurrency {
7741        return Some(format!(
7742            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7743            registered.module_id, registered_concurrency, candidate_concurrency
7744        ));
7745    }
7746
7747    // control_ops live beside the manifest in the HELLO body, not inside
7748    // ModuleManifest, so a provides-only catalog.update cannot change them.
7749    None
7750}
7751
7752fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7753    manifest.provides.iter().any(is_routable_role)
7754}
7755
7756/// Returns the routable-provider concurrency subc should enforce for this manifest.
7757///
7758/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7759/// InternalService has no role-specific concurrency field, so it retains the
7760/// existing ModuleManaged default for backward compatibility.
7761fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7762    manifest
7763        .provides
7764        .iter()
7765        .find_map(|provider| match provider {
7766            ProviderRole::ToolProvider { concurrency, .. }
7767            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7768            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7769        })
7770        .unwrap_or(Concurrency::ModuleManaged)
7771}
7772
7773/// True when the manifest carries a ManagementSurface role whose concurrency
7774/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7775/// bytes because the typed manifest deliberately erases that distinction: the
7776/// default exists for wire compatibility, and this probe exists so the default
7777/// stays observable. Any parse irregularity returns false -- the caller only
7778/// logs, and a malformed body already failed registration upstream.
7779fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7780    let has_management_surface = manifest
7781        .provides
7782        .iter()
7783        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7784    if !has_management_surface {
7785        return false;
7786    }
7787    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7788        return false;
7789    };
7790    let Some(provides) = raw
7791        .get("manifest")
7792        .and_then(|manifest| manifest.get("provides"))
7793        .and_then(serde_json::Value::as_array)
7794    else {
7795        return false;
7796    };
7797    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7798    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7799    // against the management_surface_manifest_without_concurrency golden, not
7800    // recalled (the externally-tagged guess was this function's first bug).
7801    provides.iter().any(|role| {
7802        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7803            && role.get("concurrency").is_none()
7804    })
7805}
7806
7807fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7808    if peer_version != PROTOCOL_VERSION {
7809        return Err(format!(
7810            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7811        ));
7812    }
7813    Ok(PROTOCOL_VERSION)
7814}
7815
7816fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7817    Frame::build_with_version(
7818        response_version(frame),
7819        FrameType::Pong,
7820        frame.header.flags,
7821        0,
7822        0,
7823        frame.header.corr,
7824        Vec::new(),
7825    )
7826    .map_err(RouterError::FrameBuild)
7827}
7828
7829pub(crate) fn control_error_frame(
7830    frame: &Frame,
7831    code: &'static str,
7832    message: impl Into<String>,
7833) -> Result<Frame, RouterError> {
7834    control_error_body_frame(
7835        frame,
7836        ErrorBody {
7837            code: code.to_string(),
7838            message: message.into(),
7839            detail: None,
7840        },
7841    )
7842}
7843
7844fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7845    let body = serde_json::to_vec(&error).map_err(|err| {
7846        RouterError::backend(
7847            0,
7848            frame.header.corr,
7849            format!("failed to encode control ERROR: {err}"),
7850        )
7851    })?;
7852
7853    Frame::build_with_version(
7854        response_version(frame),
7855        FrameType::Error,
7856        control_flags(),
7857        0,
7858        0,
7859        frame.header.corr,
7860        body,
7861    )
7862    .map_err(RouterError::FrameBuild)
7863}
7864
7865fn control_response_body_frame<T: Serialize>(
7866    frame: &Frame,
7867    reply: &T,
7868    label: &'static str,
7869) -> Result<Frame, RouterError> {
7870    let body = serde_json::to_vec(reply).map_err(|err| {
7871        RouterError::backend(
7872            0,
7873            frame.header.corr,
7874            format!("failed to encode {label}: {err}"),
7875        )
7876    })?;
7877
7878    Frame::build_with_version(
7879        response_version(frame),
7880        FrameType::Response,
7881        control_flags(),
7882        0,
7883        0,
7884        frame.header.corr,
7885        body,
7886    )
7887    .map_err(RouterError::FrameBuild)
7888}
7889
7890/// Map a forwarding failure to the wire code a client sees.
7891///
7892/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7893/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7894/// chosen here decides whether a caller retries or gives up.
7895///
7896/// That makes attribution the load-bearing property, not merely having a code. A
7897/// permanent fault published as a retryable one produces a fleet-wide retry storm
7898/// against something that can never recover; a transient fault published as
7899/// permanent gives up on work that would have succeeded. Both look correct in a
7900/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7901/// the mapping per variant rather than merely asserting that some code exists.
7902///
7903/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7904/// two codes on the same side of the boundary passes it. Measured rather than
7905/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7906/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7907/// a test named for something else that happens to assert the string.
7908///
7909/// That accidental coverage is deliberately left alone rather than promoted to a
7910/// named test, because it guards a property this function does not promise.
7911/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7912/// specific code within a class, so identity is free to change and only the
7913/// partition is a contract. Splitting it out would assert a guarantee nothing
7914/// depends on — and a suite that promises more than the code does is the harder
7915/// thing to correct later, because the next reader cannot tell which assertions
7916/// are load-bearing.
7917///
7918/// Pin identity here the moment a consumer branches on a specific code.
7919fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7920    match err {
7921        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7922        ForwardingError::NoModuleConnection => "target_unavailable",
7923        ForwardingError::ModuleReloading { .. } => "module_reloading",
7924        ForwardingError::ClientRouteChannelExhausted { .. }
7925        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7926        ForwardingError::StaleModuleEndpoint
7927        | ForwardingError::UnknownReservation { .. }
7928        | ForwardingError::ConnectionClosing { .. }
7929        | ForwardingError::ClientEgressClosed { .. }
7930        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7931        // Only a swap candidate's registration can produce this, and it means
7932        // exactly what a second active HELLO for a live id means.
7933        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7934        ForwardingError::RelayCorrelationExhausted
7935        | ForwardingError::RouteOpenBuild(_)
7936        | ForwardingError::Poisoned => "forwarding_error",
7937    }
7938}
7939
7940fn response_version(frame: &Frame) -> u8 {
7941    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7942        frame.header.ver
7943    } else {
7944        PROTOCOL_VERSION
7945    }
7946}
7947
7948fn control_flags() -> Flags {
7949    Flags::new(false, Priority::Passive, false)
7950}
7951
7952/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7953/// channel. The target is the module (a client never saw the route), so this
7954/// takes the module path: delivered late rather than dropped when the module's
7955/// queue is momentarily full, and never closing its connection.
7956fn send_goodbye_target_best_effort(
7957    counters: &DaemonCounters,
7958    target: &GoodbyeTarget,
7959    context: &'static str,
7960) {
7961    let Ok(frame) = Frame::build_with_version(
7962        target.negotiated_ver,
7963        FrameType::Goodbye,
7964        control_flags(),
7965        target.channel,
7966        target.epoch,
7967        0,
7968        Vec::new(),
7969    ) else {
7970        return;
7971    };
7972    crate::forwarding::send_module_route_goodbye(
7973        counters,
7974        &target.sink,
7975        frame,
7976        target.module_id.as_deref(),
7977        context,
7978    );
7979}
7980
7981pub(crate) fn send_route_control_pushes(
7982    forwarding: &ForwardingTable,
7983    routes: Vec<EndpointRoute>,
7984    push: ClientControlPush,
7985) {
7986    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7987    for route in routes {
7988        let target = route.goodbye_target;
7989        if let Some((existing, channels)) = targets
7990            .iter_mut()
7991            .find(|(existing, _)| existing.connection_id == target.connection_id)
7992        {
7993            debug_assert_eq!(
7994                existing.negotiated_ver, target.negotiated_ver,
7995                "one connection cannot negotiate multiple frame versions"
7996            );
7997            if !channels.contains(&target.channel) {
7998                channels.push(target.channel);
7999            }
8000            continue;
8001        }
8002        let channel = target.channel;
8003        targets.push((target, vec![channel]));
8004    }
8005    for (target, mut channels) in targets {
8006        channels.sort_unstable();
8007        let mut push = push.clone();
8008        match &mut push {
8009            ClientControlPush::RouteClosing {
8010                channels: covered, ..
8011            }
8012            | ClientControlPush::RouteClosed {
8013                channels: covered, ..
8014            } => *covered = channels,
8015        }
8016        let body = match serde_json::to_vec(&push) {
8017            Ok(body) => body,
8018            Err(err) => {
8019                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8020                continue;
8021            }
8022        };
8023        let frame = match Frame::build_with_version(
8024            target.negotiated_ver,
8025            FrameType::Push,
8026            control_flags(),
8027            0,
8028            0,
8029            0,
8030            body.clone(),
8031        ) {
8032            Ok(frame) => frame,
8033            Err(err) => {
8034                warn!(
8035                    route_channel = target.channel,
8036                    error = %err,
8037                    "failed to build route lifecycle control PUSH frame"
8038                );
8039                continue;
8040            }
8041        };
8042        if let Err(err) = target.sink.try_send(frame) {
8043            if target.close_on_delivery_failure() {
8044                warn!(
8045                    target_connection_id = target.connection_id.get(),
8046                    route_channel = target.channel,
8047                    error = %err,
8048                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8049                );
8050                let _ = forwarding.escalate_client_delivery_failure(
8051                    target.connection_id,
8052                    target.channel,
8053                    target.epoch,
8054                    CloseReason::new(
8055                        "route_lifecycle_push_delivery_failed",
8056                        format!(
8057                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8058                            target.channel
8059                        ),
8060                    ),
8061                    crate::forwarding::UndeliveredFrame {
8062                        module_id: target.module_id.as_deref(),
8063                        sink: &target.sink,
8064                    },
8065                );
8066            }
8067        }
8068    }
8069}
8070
8071#[cfg(test)]
8072mod tests {
8073    mod activity_read {
8074        include!("activity_read_tests.rs");
8075    }
8076
8077    mod lifecycle_activity {
8078        include!("control_lifecycle_tests.rs");
8079    }
8080    #[cfg(unix)]
8081    #[tokio::test]
8082    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8083        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8084        let path = dir.join("subc.jsonc");
8085        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8086            "program":"/bin/sleep","args":["60"],"protocol":"none",
8087            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8088        }}}).to_string()).unwrap();
8089        let mut configured = crate::daemon_config::load(&path)
8090            .unwrap()
8091            .unwrap()
8092            .modules
8093            .pop()
8094            .unwrap();
8095        let registry = std::sync::Arc::new(crate::Registry::default());
8096        let handle = crate::SupervisorHandle::new();
8097        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8098            .with_handle(handle.clone());
8099        let module = supervisor
8100            .supervise_configured_with_health(
8101                configured.module_spec(),
8102                true,
8103                configured.health.clone(),
8104                None,
8105                configured.restart,
8106            )
8107            .unwrap();
8108        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8109        let before = module.status().unwrap().pid;
8110        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8111        configured.health.cadence = std::time::Duration::from_secs(3600);
8112        let health_only = handler
8113            .reconcile_supervised_modules(
8114                &supervisor,
8115                vec![configured.clone()],
8116                false,
8117                Principal::Direct,
8118            )
8119            .await
8120            .unwrap();
8121        assert!(
8122            health_only.0.changed_pending_reload.is_empty(),
8123            "health policy is already applied live"
8124        );
8125        assert_eq!(module.status().unwrap().pid, before);
8126        assert_eq!(
8127            module.configuration().unwrap().1.http,
8128            configured.health.http
8129        );
8130        configured.args = vec!["61".into()];
8131        let launch = handler
8132            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8133            .await
8134            .unwrap();
8135        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8136        assert_eq!(
8137            module.status().unwrap().pid,
8138            before,
8139            "a launch edit is stored until reload"
8140        );
8141        module.drain().await.unwrap();
8142    }
8143    use cortexkit_test_support::ScratchDir;
8144    use std::{
8145        collections::BTreeMap,
8146        fmt,
8147        path::PathBuf,
8148        sync::{Arc, Mutex},
8149        time::Duration,
8150    };
8151
8152    use serde_json::{json, Value};
8153    use subc_protocol::{
8154        manifest::{
8155            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8156            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8157        },
8158        session::HealthStatus,
8159        FrameType,
8160    };
8161
8162    use super::*;
8163    use crate::{
8164        forwarding::{DataRoute, DataRouteState},
8165        registry::ChannelState,
8166        router::FrameSink,
8167        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8168        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8169        RouteCtx, Router,
8170    };
8171    use tokio::{
8172        sync::mpsc,
8173        time::{sleep, Instant},
8174    };
8175    use tracing::{
8176        field::{Field, Visit},
8177        Event, Subscriber,
8178    };
8179    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8180
8181    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8182    ///
8183    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8184    /// is only populated for `tests/*.rs` integration test binaries -- this file
8185    /// compiles as part of the library target, which gets neither. This test's
8186    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8187    /// and the sibling binary lives two directories up at
8188    /// `<target-dir>/<profile>/fake-aft-stub`.
8189    ///
8190    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8191    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8192    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8193    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8194    /// `NotFound`, which reads as a broken test rather than an unbuilt
8195    /// dependency -- so state the cause and the remedy instead. Deliberately a
8196    /// panic and not a silent skip: a test that quietly passes when it could not
8197    /// run is worse than one that fails, because it reports health it never
8198    /// verified.
8199    fn fake_aft_stub_path() -> PathBuf {
8200        let mut path = std::env::current_exe().expect("current_exe available in tests");
8201        path.pop(); // .../deps/
8202        path.pop(); // .../<profile>/
8203        path.push(if cfg!(windows) {
8204            "fake-aft-stub.exe"
8205        } else {
8206            "fake-aft-stub"
8207        });
8208        assert!(
8209            path.exists(),
8210            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8211             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8212            path.display()
8213        );
8214        path
8215    }
8216
8217    /// Whether clients retry `code` in place: the predicate itself, never a copy
8218    /// of its set. A copied list breaks silently when a code is added to or
8219    /// removed from the real one, and a stale copy here would let exactly the
8220    /// failure this test exists to catch pass.
8221    fn client_retries(code: &str) -> bool {
8222        subc_protocol::error_codes::is_retryable_route_open(code)
8223    }
8224
8225    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8226    /// of failure is worse than publishing none. A permanent fault dressed as
8227    /// retryable makes every client in the fleet retry forever against something
8228    /// that cannot recover; a transient fault dressed as permanent abandons work
8229    /// that would have succeeded.
8230    ///
8231    /// Asserting "a code exists" cannot catch either, because the string is free
8232    /// to say anything. This enumerates every variant and pins which side of the
8233    /// retry boundary it lands on, so a new variant must be classified here
8234    /// deliberately rather than inheriting whichever arm it was appended to.
8235    #[test]
8236    fn retryability_of_forwarding_codes_matches_the_failure() {
8237        // Transient by nature: the target is booting, reloading, or its endpoint
8238        // was swapped mid-flight. Retrying is how these resolve.
8239        let transient = [
8240            ForwardingError::NoModuleConnection,
8241            ForwardingError::ModuleReloading {
8242                module_id: "m".into(),
8243            },
8244            ForwardingError::StaleModuleEndpoint,
8245            ForwardingError::UnknownReservation {
8246                client_channel: 1,
8247                module_channel: 1,
8248            },
8249            ForwardingError::ConnectionClosing {
8250                connection_id: ConnectionId::new(1),
8251            },
8252            ForwardingError::ClientEgressClosed {
8253                connection_id: ConnectionId::new(1),
8254            },
8255            ForwardingError::ModuleEgressUnavailable {
8256                connection_id: ConnectionId::new(1),
8257            },
8258        ];
8259        for err in transient {
8260            let code = forwarding_error_code(&err);
8261            assert!(
8262                client_retries(code),
8263                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8264            );
8265        }
8266
8267        // Not fixed by retrying. Channel and correlation exhaustion need the
8268        // caller to close routes, and a poisoned lock is a daemon that cannot
8269        // recover at all — the worst thing to advertise as retryable, since every
8270        // client would storm a daemon that will never answer.
8271        let permanent = [
8272            ForwardingError::ConnectionRoleConflict {
8273                connection_id: ConnectionId::new(1),
8274            },
8275            ForwardingError::ClientRouteChannelExhausted {
8276                connection_id: ConnectionId::new(1),
8277            },
8278            ForwardingError::ModuleRouteChannelExhausted {
8279                endpoint: ModuleEndpointId {
8280                    connection_id: ConnectionId::new(1),
8281                    generation: 1,
8282                },
8283            },
8284            ForwardingError::RelayCorrelationExhausted,
8285            ForwardingError::RouteOpenBuild("x".into()),
8286            ForwardingError::Poisoned,
8287        ];
8288        for err in permanent {
8289            let code = forwarding_error_code(&err);
8290            assert!(
8291                !client_retries(code),
8292                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8293            );
8294        }
8295    }
8296
8297    /// The principal is the daemon's answer to "who is calling", and modules
8298    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8299    /// plexus gates connector invocation. So a stamp is an authorization input in
8300    /// another process, not a label — and both possible answers SUCCEED, which is
8301    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8302    /// first-party capability to something that never proved it; a supervised one
8303    /// stamped `Direct` silently strips a module of capability it is entitled to.
8304    ///
8305    /// Neither shows up in a test that only checks the bind succeeded. Before this
8306    /// test the only coverage was accidental —
8307    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8308    /// stamped principal on its way past, so narrowing that wire-shape test to its
8309    /// stated subject would have deleted the last assertion on this value. It
8310    /// still asserts the stamp, which is now redundancy rather than the only
8311    /// guard: both fail under the same mutation, and this one names the reason.
8312    /// SCOPE: this handler's supervisor has spawned nothing, so
8313    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8314    /// is unreachable here. Both assertions below are refusals, and a mutant that
8315    /// refuses everything would satisfy them.
8316    ///
8317    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8318    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8319    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8320    /// at source rather than assumed, since a citation is a claim about another
8321    /// file and ages like one. Recorded because a harness that structurally
8322    /// cannot reach an arm reports "none" for that arm identically to one that
8323    /// covers it and found nothing.
8324    #[tokio::test]
8325    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8326        let handler = ControlHandler::default();
8327        let frame =
8328            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8329
8330        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8331        // any process holding the connection file. Nothing was proved, so nothing
8332        // may be granted beyond the unattested floor.
8333        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8334        assert_eq!(
8335            stamped,
8336            Principal::Direct,
8337            "a caller that proved nothing must not be stamped as a supervised module"
8338        );
8339
8340        // A claimed module_id with a nonce no supervised child was given is a
8341        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8342        // quietly demoted to Direct, or an impersonation attempt looks identical
8343        // to an ordinary unattested connection.
8344        let forged = handler
8345            .route_open_principal(
8346                &frame,
8347                Some(ConsumerIdentity {
8348                    module_id: "aft".to_string(),
8349                    launch_nonce: "not-a-real-nonce".to_string(),
8350                }),
8351            )
8352            .unwrap();
8353        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8354        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8355    }
8356
8357    /// The test above hands `route_open_principal` an identity it built itself,
8358    /// which proves the stamping rule and nothing about where the identity comes
8359    /// from. The real producer is a wire body, and the two are joined by a serde
8360    /// field name that nothing else asserts.
8361    ///
8362    /// That join fails quietly in one specific way: an unrecognised key is simply
8363    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8364    /// yields `None` and every supervised module silently drops to `Direct`.
8365    /// Capability-wise that is the safe direction, but it surfaces far from its
8366    /// cause — as a module mysteriously refused bash — and it would pass every
8367    /// test that builds its own input.
8368    ///
8369    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8370    /// would break every client the moment the daemon gains a field, trading a
8371    /// quiet demotion for a hard refusal on additive change. Asserting the join
8372    /// instead means a rename breaks a test here rather than the fleet.
8373    #[test]
8374    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8375        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8376        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8377        let ClientControlRequest::RouteOpen {
8378            consumer_identity, ..
8379        } = parsed
8380        else {
8381            panic!("route.open body must parse as RouteOpen");
8382        };
8383        assert_eq!(
8384            consumer_identity,
8385            Some(ConsumerIdentity {
8386                module_id: "aft".to_string(),
8387                launch_nonce: "n".to_string(),
8388            }),
8389            "the wire field name must reach the value route_open_principal reads"
8390        );
8391    }
8392
8393    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8394        ModuleManifest::builder(module_id, "0.1.0")
8395            .protocol_ver(protocol_ver)
8396            .provides(vec![ProviderRole::ToolProvider {
8397                tools: vec![Tool {
8398                    name: "read".to_string(),
8399                    description: None,
8400                    execution_mode: ExecutionMode::Pure,
8401                    schema: json!({"type": "object"}),
8402                }],
8403                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8404                concurrency: Concurrency::ModuleManaged,
8405                emits_push: true,
8406                sub_supervises: true,
8407            }])
8408            .build()
8409    }
8410
8411    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8412        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8413    }
8414
8415    fn hello_frame_with_control_ops(
8416        module_id: &str,
8417        protocol_ver: u8,
8418        corr: u64,
8419        control_ops: Option<Vec<String>>,
8420    ) -> Frame {
8421        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8422    }
8423
8424    fn hello_frame_with_nonce(
8425        module_id: &str,
8426        protocol_ver: u8,
8427        corr: u64,
8428        launch_nonce: Option<&str>,
8429    ) -> Frame {
8430        hello_frame_full(
8431            module_id,
8432            protocol_ver,
8433            corr,
8434            None,
8435            launch_nonce.map(ToOwned::to_owned),
8436        )
8437    }
8438
8439    fn hello_frame_full(
8440        module_id: &str,
8441        protocol_ver: u8,
8442        corr: u64,
8443        control_ops: Option<Vec<String>>,
8444        launch_nonce: Option<String>,
8445    ) -> Frame {
8446        let body = serde_json::to_vec(&ModuleHelloBody {
8447            manifest: manifest(module_id, protocol_ver),
8448            protocol_ver,
8449            control_ops,
8450            launch_nonce,
8451        })
8452        .unwrap();
8453        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8454    }
8455
8456    fn hello_frame_with_rich_ui(
8457        module_id: &str,
8458        protocol_ver: u8,
8459        corr: u64,
8460        launch_nonce: Option<&str>,
8461        rich_ui: Value,
8462    ) -> Frame {
8463        let mut body = serde_json::to_value(ModuleHelloBody {
8464            manifest: manifest(module_id, protocol_ver),
8465            protocol_ver,
8466            control_ops: None,
8467            launch_nonce: launch_nonce.map(str::to_string),
8468        })
8469        .unwrap();
8470        body["manifest"]["rich_ui"] = rich_ui;
8471        Frame::build(
8472            FrameType::Hello,
8473            control_flags(),
8474            0,
8475            0,
8476            corr,
8477            serde_json::to_vec(&body).unwrap(),
8478        )
8479        .unwrap()
8480    }
8481
8482    fn non_routable_hello_frame_with_control_ops(
8483        module_id: &str,
8484        corr: u64,
8485        control_ops: Option<Vec<String>>,
8486    ) -> Frame {
8487        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8488        manifest.provides.clear();
8489        let body = serde_json::to_vec(&ModuleHelloBody {
8490            manifest,
8491            protocol_ver: PROTOCOL_VERSION,
8492            control_ops,
8493            launch_nonce: None,
8494        })
8495        .unwrap();
8496        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8497    }
8498
8499    fn capability_grammar_hello_frame(
8500        capabilities: Value,
8501        runtime_computed: Option<Value>,
8502        corr: u64,
8503    ) -> Frame {
8504        let mut body = serde_json::to_value(ModuleHelloBody {
8505            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8506            protocol_ver: PROTOCOL_VERSION,
8507            control_ops: None,
8508            launch_nonce: None,
8509        })
8510        .expect("HELLO body serializes");
8511        body["manifest"]["capabilities"] = capabilities;
8512        if let Some(runtime_computed) = runtime_computed {
8513            body["runtime_computed"] = runtime_computed;
8514        }
8515        Frame::build(
8516            FrameType::Hello,
8517            control_flags(),
8518            0,
8519            0,
8520            corr,
8521            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8522        )
8523        .expect("HELLO frame builds")
8524    }
8525
8526    fn channel_request(channel: u16, corr: u64) -> Frame {
8527        Frame::build(
8528            FrameType::Request,
8529            Flags::new(true, Priority::Interactive, false),
8530            channel,
8531            0,
8532            corr,
8533            b"opaque".to_vec(),
8534        )
8535        .unwrap()
8536    }
8537
8538    fn route_ctx(
8539        connection_id: ConnectionId,
8540    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8541        let (tx, rx) = mpsc::channel(8);
8542        (
8543            RouteCtx {
8544                connection_id,
8545                egress: FrameSink::new(tx),
8546            },
8547            rx,
8548        )
8549    }
8550
8551    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8552        serde_json::from_slice(&frame.body).unwrap()
8553    }
8554
8555    /// Register a module over a connection that has a sink and return the
8556    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8557    /// module's own sink rather than returning it as a reply, so the ack is
8558    /// taken off `rx` here and whatever the test reads next is what followed it.
8559    async fn hello_via_sink(
8560        handler: &ControlHandler,
8561        ctx: &RouteCtx,
8562        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8563        hello: Frame,
8564    ) -> Frame {
8565        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8566        assert!(
8567            replies.is_empty(),
8568            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8569        );
8570        let ack = rx
8571            .try_recv()
8572            .expect("HELLO_ACK is queued on the module sink")
8573            .frame;
8574        assert_eq!(ack.header.ty, FrameType::HelloAck);
8575        ack
8576    }
8577
8578    fn parse_error(frame: &Frame) -> Value {
8579        serde_json::from_slice(&frame.body).unwrap()
8580    }
8581
8582    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8583        serde_json::from_slice(&frame.body).unwrap()
8584    }
8585
8586    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8587        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8588            route_channel,
8589            route_epoch: 0,
8590            kind,
8591        })
8592        .unwrap();
8593        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8594    }
8595
8596    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8597        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8598            module_id: module_id.to_string(),
8599        })
8600        .unwrap();
8601        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8602    }
8603
8604    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8605        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8606    }
8607
8608    fn route_open_frame_with_consumer_capabilities(
8609        corr: u64,
8610        module_id: &str,
8611        project_root: ScratchDir,
8612        consumer_capabilities: Option<Vec<String>>,
8613    ) -> Frame {
8614        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8615            target: RouteTarget::ToolProvider {
8616                module_id: module_id.to_string(),
8617            },
8618            identity: BindIdentity::new(
8619                project_root.path().to_path_buf(),
8620                "unit".to_string(),
8621                "session".to_string(),
8622            ),
8623            consumer_identity: None,
8624            consumer_capabilities,
8625            role_versions: None,
8626            admission_facts: None,
8627            scope: None,
8628        })
8629        .unwrap();
8630        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8631    }
8632
8633    fn route_open_frame_with_role_versions(
8634        corr: u64,
8635        module_id: &str,
8636        project_root: ScratchDir,
8637        role_versions: Option<BTreeMap<String, String>>,
8638    ) -> Frame {
8639        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8640            target: RouteTarget::ToolProvider {
8641                module_id: module_id.to_string(),
8642            },
8643            identity: BindIdentity::new(
8644                project_root.path().to_path_buf(),
8645                "unit".to_string(),
8646                format!("session-{corr}"),
8647            ),
8648            consumer_identity: None,
8649            consumer_capabilities: None,
8650            role_versions,
8651            admission_facts: None,
8652            scope: None,
8653        })
8654        .unwrap();
8655        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8656    }
8657
8658    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8659        entries
8660            .iter()
8661            .map(|(role, version)| (role.to_string(), version.to_string()))
8662            .collect()
8663    }
8664
8665    fn route_open_frame_with_admission_facts(
8666        corr: u64,
8667        module_id: &str,
8668        project_root: ScratchDir,
8669        consumer_identity: Option<subc_control::ConsumerIdentity>,
8670        facts: Option<Value>,
8671    ) -> Frame {
8672        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8673            target: RouteTarget::ToolProvider {
8674                module_id: module_id.to_string(),
8675            },
8676            identity: BindIdentity::new(
8677                project_root.path().to_path_buf(),
8678                "unit".to_string(),
8679                format!("session-{corr}"),
8680            ),
8681            consumer_identity,
8682            consumer_capabilities: None,
8683            role_versions: None,
8684            admission_facts: facts,
8685            scope: None,
8686        })
8687        .unwrap();
8688        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8689    }
8690
8691    #[derive(Clone, Default)]
8692    struct EventCapture {
8693        events: Arc<Mutex<Vec<CapturedEvent>>>,
8694    }
8695
8696    #[derive(Clone, Debug)]
8697    struct CapturedEvent {
8698        target: String,
8699        level: tracing::Level,
8700        fields: BTreeMap<String, String>,
8701    }
8702
8703    impl EventCapture {
8704        fn events(&self) -> Vec<CapturedEvent> {
8705            self.events.lock().unwrap().clone()
8706        }
8707    }
8708
8709    impl<S> Layer<S> for EventCapture
8710    where
8711        S: Subscriber,
8712    {
8713        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8714            let mut visitor = EventFieldVisitor::default();
8715            event.record(&mut visitor);
8716            self.events.lock().unwrap().push(CapturedEvent {
8717                target: event.metadata().target().to_string(),
8718                level: *event.metadata().level(),
8719                fields: visitor.fields,
8720            });
8721        }
8722    }
8723
8724    #[derive(Default)]
8725    struct EventFieldVisitor {
8726        fields: BTreeMap<String, String>,
8727    }
8728
8729    impl Visit for EventFieldVisitor {
8730        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8731            self.fields
8732                .insert(field.name().to_string(), format!("{value:?}"));
8733        }
8734    }
8735
8736    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8737        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8738            status,
8739            detail: Some("warming".to_string()),
8740            metrics: Some(json!({"queue_depth": 3})),
8741        })
8742        .unwrap();
8743        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8744    }
8745
8746    fn route_bind_ack(corr: u64) -> Frame {
8747        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8748        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8749    }
8750
8751    fn unique_project_root(label: &str) -> ScratchDir {
8752        ScratchDir::new(label)
8753    }
8754
8755    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8756        match parse_route_poll(frame) {
8757            ClientControlResponse::RoutePoll {
8758                status: None,
8759                live: Some(live),
8760                ..
8761            } => assert_eq!(live, expected_live),
8762            other => panic!("unexpected route.poll response: {other:?}"),
8763        }
8764    }
8765
8766    fn bind_liveness_route(
8767        registry: &Registry,
8768        forwarding: &ForwardingTable,
8769        module_id: &str,
8770    ) -> (RouteCtx, u16, u32) {
8771        let module_connection = ConnectionId::new(101);
8772        let client_connection = ConnectionId::new(202);
8773        let registration = registry
8774            .register_with_control_ops(
8775                manifest(module_id, PROTOCOL_VERSION),
8776                PROTOCOL_VERSION,
8777                module_connection,
8778                module_baseline_control_ops(),
8779            )
8780            .unwrap();
8781        let (module_tx, _module_rx) = mpsc::channel(8);
8782        let endpoint = forwarding
8783            .register_module_connection(
8784                module_connection,
8785                module_id.to_string(),
8786                PROTOCOL_VERSION,
8787                manifest_concurrency(&registration.manifest),
8788                FrameSink::new(module_tx),
8789            )
8790            .unwrap();
8791        let (client_ctx, _client_rx) = route_ctx(client_connection);
8792        let pending = forwarding
8793            .begin_route_bind_relay_for_test(
8794                client_connection,
8795                client_ctx.egress.clone(),
8796                1,
8797                module_id,
8798            )
8799            .unwrap();
8800        assert_eq!(pending.endpoint, endpoint);
8801        let route_channel = pending.client_channel;
8802        let route_epoch = pending.client_epoch;
8803        forwarding
8804            .complete_pending_relay(
8805                module_connection,
8806                pending.corr,
8807                RouteBindRelayOutcome::Accepted,
8808            )
8809            .unwrap();
8810        (client_ctx, route_channel, route_epoch)
8811    }
8812
8813    struct FakeProcessLiveness {
8814        live: Option<bool>,
8815    }
8816
8817    impl ModuleProcessLiveness for FakeProcessLiveness {
8818        fn process_live(&self, _module_id: &str) -> Option<bool> {
8819            self.live
8820        }
8821    }
8822
8823    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8824    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8825    {
8826        let registry = Arc::new(Registry::default());
8827        let supervisor_handle = SupervisorHandle::new();
8828        let supervisor = Supervisor::new_for_test(
8829            Arc::clone(&registry),
8830            RestartPolicy::new(1, Duration::from_millis(10)),
8831        )
8832        .with_handle(supervisor_handle.clone());
8833        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8834        let module = supervisor
8835            .spawn(ModuleSpec {
8836                module_id: "stderr-tail-wire".to_string(),
8837                program: fake_aft_stub_path(),
8838                args: Vec::new(),
8839                env: vec![
8840                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8841                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8842                ],
8843                reserved: false,
8844                reserved_prefixes: Vec::new(),
8845                protocol: ModuleProtocol::Subc,
8846                overlap: Default::default(),
8847            })
8848            .unwrap();
8849
8850        let deadline = Instant::now() + Duration::from_secs(5);
8851        loop {
8852            let tail = module.stderr_tail(None, None);
8853            if tail
8854                .entries
8855                .iter()
8856                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8857                && tail.entries.iter().any(|entry| {
8858                    matches!(
8859                        entry,
8860                        TailEntry::Line {
8861                            truncated: true,
8862                            ..
8863                        }
8864                    )
8865                })
8866            {
8867                break;
8868            }
8869            assert!(
8870                Instant::now() < deadline,
8871                "module did not produce a truncated line and restart boundary: {tail:?}"
8872            );
8873            sleep(Duration::from_millis(10)).await;
8874        }
8875
8876        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8877        let request = ClientControlRequest::SupervisorStderrTail {
8878            module_id: "stderr-tail-wire".to_string(),
8879            max_lines: None,
8880            max_bytes: None,
8881        };
8882        let frame = Frame::build(
8883            FrameType::Request,
8884            control_flags(),
8885            0,
8886            0,
8887            1,
8888            serde_json::to_vec(&request).unwrap(),
8889        )
8890        .unwrap();
8891        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8892        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8893        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8894            serde_json::from_slice(&responses[0].body).unwrap()
8895        else {
8896            panic!("expected supervisor.stderr_tail response");
8897        };
8898
8899        assert!(
8900            tail.entries
8901                .iter()
8902                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8903            "the control response lost the restart boundary"
8904        );
8905        let Some(StderrTailEntry::Line {
8906            text,
8907            truncated,
8908            at_ms,
8909        }) = tail.entries.iter().find(|entry| {
8910            matches!(
8911                entry,
8912                StderrTailEntry::Line {
8913                    truncated: true,
8914                    ..
8915                }
8916            )
8917        })
8918        else {
8919            panic!("the control response lost the truncated line");
8920        };
8921        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8922        assert!(*truncated);
8923        assert!(
8924            at_ms.is_some(),
8925            "the control response lost the line's capture time"
8926        );
8927    }
8928
8929    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8930    /// read done on the worker thread would stall every other task until it
8931    /// finished; the read must run off the worker so this test's own task keeps
8932    /// running while the read is paused.
8933    #[tokio::test(flavor = "current_thread")]
8934    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8935        let dir = ScratchDir::new("terminals-off-worker");
8936        let journal_path = dir.join("terminals.jsonl");
8937        let registry = Arc::new(Registry::default());
8938        let supervisor_handle = SupervisorHandle::new();
8939        let supervisor =
8940            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8941                .with_handle(supervisor_handle.clone())
8942                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8943        let module = supervisor
8944            .spawn(ModuleSpec {
8945                module_id: "terminal-off-worker".to_string(),
8946                program: fake_aft_stub_path(),
8947                args: Vec::new(),
8948                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8949                reserved: false,
8950                reserved_prefixes: Vec::new(),
8951                protocol: ModuleProtocol::Subc,
8952                overlap: Default::default(),
8953            })
8954            .unwrap();
8955        let deadline = Instant::now() + Duration::from_secs(5);
8956        while module.terminal_history().entries.len() != 2 {
8957            assert!(Instant::now() < deadline, "module did not record two exits");
8958            sleep(Duration::from_millis(10)).await;
8959        }
8960
8961        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8962        let handler =
8963            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8964        let frame = Frame::build(
8965            FrameType::Request,
8966            control_flags(),
8967            0,
8968            0,
8969            1,
8970            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8971                module_id: "terminal-off-worker".to_string(),
8972            })
8973            .unwrap(),
8974        )
8975        .unwrap();
8976        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8977        let spawned_at = std::time::Instant::now();
8978        let read = tokio::spawn({
8979            let handler = Arc::clone(&handler);
8980            async move { handler.handle_control_frame(&ctx, frame).await }
8981        });
8982        // Waiting for the pause from a blocking thread keeps this task pending,
8983        // so the runtime's single worker is free to run the read task.
8984        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8985            .await
8986            .unwrap()
8987            .expect("the history read reached its pause");
8988        let elapsed = spawned_at.elapsed();
8989        assert!(
8990            elapsed < Duration::from_secs(2) && !read.is_finished(),
8991            "this task could not run while the history read was paused \
8992             (resumed after {elapsed:?}, read finished: {})",
8993            read.is_finished()
8994        );
8995
8996        drop(release);
8997        let responses = read.await.unwrap().unwrap();
8998        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8999        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9000            panic!("expected supervisor.terminals response");
9001        };
9002        assert_eq!(terminals.entries.len(), 2);
9003        assert_eq!(terminals.journal_skipped_lines, 0);
9004        assert_eq!(terminals.journal_read_errors, 0);
9005    }
9006
9007    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9008    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9009        let registry = Arc::new(Registry::default());
9010        let supervisor_handle = SupervisorHandle::new();
9011        let supervisor =
9012            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9013                .with_handle(supervisor_handle.clone());
9014        let module = supervisor
9015            .spawn(ModuleSpec {
9016                module_id: "terminal-golden".to_string(),
9017                program: fake_aft_stub_path(),
9018                args: Vec::new(),
9019                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9020                reserved: false,
9021                reserved_prefixes: Vec::new(),
9022                protocol: ModuleProtocol::Subc,
9023                overlap: Default::default(),
9024            })
9025            .unwrap();
9026
9027        let deadline = Instant::now() + Duration::from_secs(5);
9028        while module.terminal_history().entries.len() != 2 {
9029            assert!(
9030                Instant::now() < deadline,
9031                "module did not retain two terminal exits: {:?}",
9032                module.terminal_history()
9033            );
9034            sleep(Duration::from_millis(10)).await;
9035        }
9036
9037        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9038        let request = ClientControlRequest::SupervisorTerminals {
9039            module_id: "terminal-golden".to_string(),
9040        };
9041        let frame = Frame::build(
9042            FrameType::Request,
9043            control_flags(),
9044            0,
9045            0,
9046            1,
9047            serde_json::to_vec(&request).unwrap(),
9048        )
9049        .unwrap();
9050        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9051        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9052        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9053        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9054            panic!("expected supervisor.terminals response");
9055        };
9056        assert_eq!(terminals.entries.len(), 2);
9057        assert_eq!(terminals.dropped, 0);
9058
9059        let mut rendered = serde_json::to_value(response).unwrap();
9060        // Wall-clock fields are the observation contract, but not stable fixture
9061        // bytes; normalize only them after the real handler has shaped the response.
9062        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9063        for (index, entry) in rendered["entries"]
9064            .as_array_mut()
9065            .expect("terminal response entries array")
9066            .iter_mut()
9067            .enumerate()
9068        {
9069            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9070        }
9071
9072        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9073            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9074        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9075        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9076            std::fs::write(&golden_path, &serialized).unwrap();
9077        }
9078        let expected: Value =
9079            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9080        assert_eq!(rendered, expected);
9081    }
9082
9083    #[test]
9084    fn hello_registers_manifest_and_returns_ack() {
9085        let registry = Arc::new(Registry::default());
9086        let handler = ControlHandler::new(Arc::clone(&registry));
9087        let conn = ConnectionId::new(1);
9088
9089        let responses = handler
9090            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9091            .unwrap();
9092
9093        assert_eq!(responses.len(), 1);
9094        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9095        assert_eq!(responses[0].header.channel, 0);
9096        assert_eq!(responses[0].header.corr, 7);
9097        let ack = parse_ack(&responses[0]);
9098        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9099        assert!(ack
9100            .subc_capabilities
9101            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9102        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9103        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9104        assert!(ack
9105            .subc_ops
9106            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9107        assert!(ack
9108            .subc_ops
9109            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9110
9111        let registration = registry.get_module("aft").unwrap().unwrap();
9112        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9113        assert_eq!(registration.state, ChannelState::Active);
9114        assert_eq!(registration.connection_id, conn);
9115        assert_eq!(registration.control_ops, module_baseline_control_ops());
9116    }
9117
9118    #[test]
9119    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9120        let invalid_identifiers = [
9121            ("case_change", "credentials-Provider/v1"),
9122            ("leading_zero", "credentials-provider/v01"),
9123            ("trailing_hyphen", "credentials-provider-/v1"),
9124            ("consecutive_hyphens", "credentials--provider/v1"),
9125            ("uppercase", "Credentials-provider/v1"),
9126            ("missing_v", "credentials-provider/1"),
9127            ("whitespace", "credentials provider/v1"),
9128            ("zero_version", "credentials-provider/v0"),
9129            ("out_of_range_version", "credentials-provider/v4294967296"),
9130            (
9131                "overlength_name",
9132                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9133            ),
9134        ];
9135        let mut cases = invalid_identifiers
9136            .into_iter()
9137            .map(|(name, identifier)| {
9138                (
9139                    format!("identifier_{name}"),
9140                    "capabilities.provides[0]".to_string(),
9141                    identifier.to_string(),
9142                    json!({ "provides": [identifier] }),
9143                    None,
9144                )
9145            })
9146            .collect::<Vec<_>>();
9147        cases.extend([
9148            (
9149                "unknown_need".to_string(),
9150                "capabilities.requires[0].need".to_string(),
9151                "deferred".to_string(),
9152                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9153                None,
9154            ),
9155            (
9156                "duplicate_provides".to_string(),
9157                "capabilities.provides[1]".to_string(),
9158                "credentials-provider/v1".to_string(),
9159                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9160                None,
9161            ),
9162            (
9163                "duplicate_must_never_reach".to_string(),
9164                "capabilities.must_never_reach[1]".to_string(),
9165                "credentials-provider/v1".to_string(),
9166                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9167                None,
9168            ),
9169            (
9170                "duplicate_requires_same_need".to_string(),
9171                "capabilities.requires[1]".to_string(),
9172                "credentials-provider/v1".to_string(),
9173                json!({ "requires": [
9174                    { "capability": "credentials-provider/v1", "need": "required" },
9175                    { "capability": "credentials-provider/v1", "need": "required" }
9176                ] }),
9177                None,
9178            ),
9179            (
9180                "duplicate_requires_conflicting_need".to_string(),
9181                "capabilities.requires[1]".to_string(),
9182                "credentials-provider/v1".to_string(),
9183                json!({ "requires": [
9184                    { "capability": "credentials-provider/v1", "need": "required" },
9185                    { "capability": "credentials-provider/v1", "need": "optional" }
9186                ] }),
9187                None,
9188            ),
9189            (
9190                "capabilities_root_pointer".to_string(),
9191                "runtime_computed[0]".to_string(),
9192                "/capabilities".to_string(),
9193                json!({}),
9194                Some(json!(["/capabilities"])),
9195            ),
9196            (
9197                "capabilities_descendant_pointer".to_string(),
9198                "runtime_computed[0]".to_string(),
9199                "/capabilities/provides".to_string(),
9200                json!({}),
9201                Some(json!(["/capabilities/provides"])),
9202            ),
9203            (
9204                "malformed_pointer_without_leading_slash".to_string(),
9205                "runtime_computed[0]".to_string(),
9206                "capabilities".to_string(),
9207                json!({}),
9208                Some(json!(["capabilities"])),
9209            ),
9210            (
9211                "malformed_pointer_escape".to_string(),
9212                "runtime_computed[0]".to_string(),
9213                "/roles/~2/tools".to_string(),
9214                json!({}),
9215                Some(json!(["/roles/~2/tools"])),
9216            ),
9217            (
9218                "unknown_capabilities_field".to_string(),
9219                "capabilities.future".to_string(),
9220                "<array>".to_string(),
9221                json!({ "future": [] }),
9222                None,
9223            ),
9224        ]);
9225
9226        for (index, (name, field, value, capabilities, runtime_computed)) in
9227            cases.into_iter().enumerate()
9228        {
9229            let registry = Arc::new(Registry::default());
9230            let handler = ControlHandler::new(Arc::clone(&registry));
9231            let response = handler
9232                .handle_control(
9233                    ConnectionId::new((index + 1) as u64),
9234                    capability_grammar_hello_frame(
9235                        capabilities,
9236                        runtime_computed,
9237                        index as u64 + 1,
9238                    ),
9239                )
9240                .expect("invalid HELLO returns a refusal");
9241
9242            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9243            let error = parse_error(&response[0]);
9244            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9245            let message = error["message"]
9246                .as_str()
9247                .expect("error message is a string");
9248            assert!(
9249                message.contains(&field),
9250                "{name}: field missing from {message}"
9251            );
9252            assert!(
9253                message.contains(&value),
9254                "{name}: value missing from {message}"
9255            );
9256            assert_eq!(
9257                registry
9258                    .active_registration_count()
9259                    .expect("registry reads"),
9260                0,
9261                "{name}: refused HELLO must not create a catalog entry"
9262            );
9263        }
9264    }
9265
9266    #[test]
9267    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9268        let registry = Arc::new(Registry::default());
9269        let handler = ControlHandler::new(Arc::clone(&registry));
9270        let capabilities = json!({
9271            "provides": ["credentials-provider/v1"],
9272            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9273            "must_never_reach": ["federation-transport/v1"]
9274        });
9275        let response = handler
9276            .handle_control(
9277                ConnectionId::new(99),
9278                capability_grammar_hello_frame(
9279                    capabilities.clone(),
9280                    Some(json!(["/roles/0/tools"])),
9281                    99,
9282                ),
9283            )
9284            .expect("valid HELLO registers");
9285        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9286
9287        let request = Frame::build(
9288            FrameType::Request,
9289            control_flags(),
9290            0,
9291            0,
9292            100,
9293            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9294                .expect("catalog request serializes"),
9295        )
9296        .expect("catalog request frame builds");
9297        let response = handler
9298            .handle_catalog_list(request, None)
9299            .expect("catalog list succeeds");
9300        let ClientControlResponse::CatalogList { modules, .. } =
9301            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9302        else {
9303            panic!("catalog request must return catalog.list");
9304        };
9305        assert_eq!(modules.len(), 1);
9306        assert_eq!(
9307            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9308            capabilities
9309        );
9310    }
9311
9312    #[test]
9313    fn catalog_list_mirrors_management_operation_description() {
9314        let registry = Arc::new(Registry::default());
9315        let handler = ControlHandler::new(Arc::clone(&registry));
9316        let description = "List managed records and return their identifiers and metadata.";
9317        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9318        manifest.provides = vec![ProviderRole::ManagementSurface {
9319            operations: vec![ManagementOperation {
9320                name: "records.list".to_string(),
9321                kind: ManagementOperationKind::Query,
9322                description: Some(description.to_string()),
9323            }],
9324            config_schema: json!({"type": "object"}),
9325            observability: vec![ObservabilitySurface {
9326                name: "records.stats".to_string(),
9327                kind: ObservabilityKind::Snapshot,
9328            }],
9329            identity_scope: vec![IdentityScope::Project],
9330            concurrency: Concurrency::ModuleManaged,
9331        }];
9332        registry
9333            .register_with_control_ops(
9334                manifest,
9335                PROTOCOL_VERSION,
9336                ConnectionId::new(99),
9337                Vec::new(),
9338            )
9339            .expect("described management manifest registers");
9340
9341        let request = Frame::build(
9342            FrameType::Request,
9343            control_flags(),
9344            0,
9345            0,
9346            100,
9347            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9348                .expect("catalog request serializes"),
9349        )
9350        .expect("catalog request frame builds");
9351        let response = handler
9352            .handle_catalog_list(request, None)
9353            .expect("catalog list succeeds");
9354        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9355        assert_eq!(
9356            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9357            "catalog.list must preserve the declared operation description verbatim"
9358        );
9359    }
9360
9361    #[test]
9362    fn catalog_list_mirrors_registered_operation_delivery() {
9363        let registry = Arc::new(Registry::default());
9364        let handler = ControlHandler::new(Arc::clone(&registry));
9365        let delivery =
9366            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9367        let mut body = serde_json::to_value(ModuleHelloBody {
9368            manifest: manifest("watch-management", PROTOCOL_VERSION),
9369            protocol_ver: PROTOCOL_VERSION,
9370            control_ops: None,
9371            launch_nonce: None,
9372        })
9373        .unwrap();
9374        body["manifest"]["provides"] = json!([{
9375            "role": "management_surface",
9376            "operations": [
9377                {"name": "status", "kind": "query"},
9378                {"name": "future", "kind": "query"},
9379                {"name": "reset", "kind": "mutate"}
9380            ],
9381            "config_schema": {}, "observability": [], "identity_scope": []
9382        }]);
9383        body["manifest"]["operation_delivery"] = delivery.clone();
9384        let hello = Frame::build(
9385            FrameType::Hello,
9386            control_flags(),
9387            0,
9388            0,
9389            99,
9390            serde_json::to_vec(&body).unwrap(),
9391        )
9392        .unwrap();
9393        let response = handler
9394            .handle_control(ConnectionId::new(99), hello)
9395            .unwrap();
9396        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9397
9398        let (_, registrations) = registry.list_modules().unwrap();
9399        let registered = &registrations[0].manifest;
9400        assert!(registered.is_watch_operation("status"));
9401        assert!(!registered.is_watch_operation("future"));
9402        assert!(!registered.is_watch_operation("reset"));
9403        assert!(!registered.is_watch_operation("missing"));
9404
9405        let request = Frame::build(
9406            FrameType::Request,
9407            control_flags(),
9408            0,
9409            0,
9410            100,
9411            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9412        )
9413        .unwrap();
9414        let response = handler.handle_catalog_list(request, None).unwrap();
9415        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9416        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9417        let ClientControlResponse::CatalogList { modules, .. } =
9418            serde_json::from_slice(&response[0].body).unwrap()
9419        else {
9420            panic!("expected catalog.list");
9421        };
9422        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9423    }
9424
9425    #[test]
9426    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9427        let registry = Arc::new(Registry::default());
9428        let supervisor = SupervisorHandle::new();
9429        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9430        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9431        let descriptor = json!({
9432            "schema": "ckios.richui.manifest/1",
9433            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9434            "bytes": 4096,
9435            "chunks": 4
9436        });
9437        let connection = ConnectionId::new(201);
9438        let hello = handler
9439            .handle_control(
9440                connection,
9441                hello_frame_with_rich_ui(
9442                    "vault",
9443                    PROTOCOL_VERSION,
9444                    201,
9445                    Some("the-real-nonce"),
9446                    descriptor.clone(),
9447                ),
9448            )
9449            .unwrap();
9450        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9451
9452        let request =
9453            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9454        let provides = registry
9455            .get_module_by_connection(connection)
9456            .unwrap()
9457            .unwrap()
9458            .manifest
9459            .provides;
9460        handler
9461            .handle_catalog_update(connection, request, provides, None, Some(false))
9462            .unwrap();
9463
9464        let request = Frame::build(
9465            FrameType::Request,
9466            control_flags(),
9467            0,
9468            0,
9469            203,
9470            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9471        )
9472        .unwrap();
9473        let response = handler.handle_catalog_list(request, None).unwrap();
9474        let ClientControlResponse::CatalogList { modules, .. } =
9475            serde_json::from_slice(&response[0].body).unwrap()
9476        else {
9477            panic!("expected catalog.list");
9478        };
9479        assert_eq!(
9480            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9481            descriptor
9482        );
9483        assert!(modules[0].nonce_verified);
9484    }
9485
9486    fn hello_with_role_declarations(declarations: Value) -> Frame {
9487        let mut body = serde_json::to_value(ModuleHelloBody {
9488            manifest: manifest("role-test", PROTOCOL_VERSION),
9489            protocol_ver: PROTOCOL_VERSION,
9490            control_ops: None,
9491            launch_nonce: None,
9492        })
9493        .unwrap();
9494        body["manifest"]["role_declarations"] = declarations;
9495        Frame::build(
9496            FrameType::Hello,
9497            control_flags(),
9498            0,
9499            0,
9500            301,
9501            serde_json::to_vec(&body).unwrap(),
9502        )
9503        .unwrap()
9504    }
9505
9506    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9507        let request = Frame::build(
9508            FrameType::Request,
9509            control_flags(),
9510            0,
9511            0,
9512            302,
9513            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9514        )
9515        .unwrap();
9516        let response = handler.handle_catalog_list(request, None).unwrap();
9517        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9518        let ClientControlResponse::CatalogList { mut modules, .. } =
9519            serde_json::from_slice(&response[0].body).unwrap()
9520        else {
9521            panic!("expected catalog.list");
9522        };
9523        (raw["modules"][0].clone(), modules.remove(0))
9524    }
9525
9526    #[test]
9527    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9528        let registry = Arc::new(Registry::default());
9529        let handler = ControlHandler::new(Arc::clone(&registry));
9530        let declarations = json!({
9531            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9532            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9533        });
9534        let connection = ConnectionId::new(301);
9535        let response = handler
9536            .handle_control(
9537                connection,
9538                hello_with_role_declarations(declarations.clone()),
9539            )
9540            .unwrap();
9541        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9542        let registered = registry
9543            .get_module_by_connection(connection)
9544            .unwrap()
9545            .unwrap()
9546            .manifest;
9547        // Compared as JSON values, not bytes: object key order is not part of
9548        // the relay's promise, and workspace builds unify serde_json's
9549        // preserve_order feature, which changes the order a literal keeps.
9550        assert_eq!(
9551            serde_json::to_value(&registered.role_declarations).unwrap(),
9552            declarations
9553        );
9554        let (raw, entry) = role_declarations_catalog(&handler);
9555        assert_eq!(raw["role_declarations"], declarations);
9556        assert_eq!(entry.role_declarations, registered.role_declarations);
9557
9558        let request =
9559            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9560        handler
9561            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9562            .unwrap();
9563        let (raw, entry) = role_declarations_catalog(&handler);
9564        assert_eq!(raw["role_declarations"], declarations);
9565        assert_eq!(entry.role_declarations, registered.role_declarations);
9566    }
9567
9568    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9569        let registry = Arc::new(Registry::default());
9570        let handler = ControlHandler::new(Arc::clone(&registry));
9571        let capture = EventCapture::default();
9572        let _subscriber =
9573            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9574        let response = handler
9575            .handle_control(
9576                ConnectionId::new(301),
9577                hello_with_role_declarations(declarations),
9578            )
9579            .unwrap();
9580        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9581        assert!(registry
9582            .get_module_by_connection(ConnectionId::new(301))
9583            .unwrap()
9584            .unwrap()
9585            .manifest
9586            .role_declarations
9587            .is_empty());
9588        let (raw, entry) = role_declarations_catalog(&handler);
9589        assert!(raw.get("role_declarations").is_none());
9590        assert!(entry.role_declarations.is_empty());
9591        let warnings: Vec<_> = capture
9592            .events()
9593            .into_iter()
9594            .filter(|event| {
9595                event.fields.get("message").map(String::as_str)
9596                    == Some("dropping whole role_declarations map from HELLO")
9597            })
9598            .collect();
9599        assert_eq!(warnings.len(), 1);
9600        assert_eq!(warnings[0].level, tracing::Level::WARN);
9601        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9602        assert_eq!(warnings[0].fields["reason"], reason);
9603    }
9604
9605    #[test]
9606    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9607        for value in [
9608            json!(null),
9609            json!(false),
9610            json!(0),
9611            json!("opaque"),
9612            json!([]),
9613        ] {
9614            assert_hello_drops_role_declarations_with_warning(
9615                json!({"routing/v1": {}, "broken/v1": value}),
9616                "role_declarations values must be objects",
9617            );
9618        }
9619        assert_hello_drops_role_declarations_with_warning(
9620            json!([]),
9621            "role_declarations must be an object",
9622        );
9623    }
9624
9625    #[test]
9626    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9627        assert_hello_drops_role_declarations_with_warning(
9628            json!({"routing/v1": {}, "bad key": {}}),
9629            "role_declarations key must use capability-name grammar",
9630        );
9631    }
9632
9633    #[test]
9634    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9635        assert_hello_drops_role_declarations_with_warning(
9636            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9637            "role_declarations exceeds 16384 compact JSON bytes",
9638        );
9639    }
9640
9641    #[test]
9642    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9643        let registry = Arc::new(Registry::default());
9644        let handler = ControlHandler::new(registry.clone());
9645        let descriptor = json!({
9646            "schema": "ckios.richui.manifest/1",
9647            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9648            "bytes": 128,
9649            "chunks": 2
9650        });
9651        let malformed = [
9652            ("schema", json!("")),
9653            ("schema", json!("a".repeat(65))),
9654            ("schema", json!("Bad schema")),
9655            (
9656                "manifest_digest",
9657                json!(format!("sha256:{}", "A".repeat(64))),
9658            ),
9659            ("manifest_digest", json!("sha256:abc")),
9660            ("bytes", json!(0)),
9661            ("bytes", json!(16_777_217)),
9662            ("bytes", json!("128")),
9663            ("chunks", json!(0)),
9664            ("chunks", json!(1025)),
9665            ("chunks", json!("2")),
9666        ];
9667        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9668            let mut invalid = descriptor.clone();
9669            invalid[field] = invalid_value;
9670            let response = handler
9671                .handle_control(
9672                    ConnectionId::new(210 + index as u64),
9673                    hello_frame_with_rich_ui(
9674                        &format!("bad-rich-ui-{index}"),
9675                        PROTOCOL_VERSION,
9676                        210 + index as u64,
9677                        None,
9678                        invalid,
9679                    ),
9680                )
9681                .unwrap();
9682            assert_eq!(response[0].header.ty, FrameType::Error);
9683            let error = parse_error(&response[0]);
9684            assert_eq!(error["code"], "invalid_manifest");
9685            assert!(error["message"]
9686                .as_str()
9687                .unwrap()
9688                .contains(&format!("rich_ui.{field}")));
9689            assert!(registry
9690                .get_module(&format!("bad-rich-ui-{index}"))
9691                .unwrap()
9692                .is_none());
9693        }
9694    }
9695
9696    #[test]
9697    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9698        let registry = Arc::new(Registry::default());
9699        let handler = ControlHandler::new(Arc::clone(&registry));
9700        let connection = ConnectionId::new(219);
9701        registry
9702            .register_with_control_ops(
9703                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9704                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9705                        "bad schema",
9706                        format!("sha256:{}", "a".repeat(64)),
9707                        128,
9708                        2,
9709                    )))
9710                    .build(),
9711                PROTOCOL_VERSION,
9712                connection,
9713                Vec::new(),
9714            )
9715            .unwrap();
9716        let request =
9717            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9718        let response = handler
9719            .handle_catalog_update(connection, request, Vec::new(), None, None)
9720            .unwrap();
9721        assert_eq!(response[0].header.ty, FrameType::Error);
9722        let error = parse_error(&response[0]);
9723        assert_eq!(error["code"], "invalid_manifest");
9724        assert!(error["message"]
9725            .as_str()
9726            .unwrap()
9727            .contains("rich_ui.schema"));
9728    }
9729
9730    #[test]
9731    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9732        let registry = Arc::new(Registry::default());
9733        let supervisor = SupervisorHandle::new();
9734        supervisor.open_swap("provider", "candidate-nonce".to_string());
9735        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9736        let connection = ConnectionId::new(218);
9737        let hello = handler
9738            .handle_control(
9739                connection,
9740                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9741            )
9742            .unwrap();
9743        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9744        assert!(registry.list_modules().unwrap().1.is_empty());
9745
9746        registry.promote_candidate("provider").unwrap().unwrap();
9747        let request = Frame::build(
9748            FrameType::Request,
9749            control_flags(),
9750            0,
9751            0,
9752            219,
9753            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9754        )
9755        .unwrap();
9756        let response = handler.handle_catalog_list(request, None).unwrap();
9757        let ClientControlResponse::CatalogList { modules, .. } =
9758            serde_json::from_slice(&response[0].body).unwrap()
9759        else {
9760            panic!("expected catalog.list");
9761        };
9762        assert_eq!(modules.len(), 1);
9763        assert!(modules[0].nonce_verified);
9764    }
9765
9766    #[test]
9767    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9768        let registry = Arc::new(Registry::default());
9769        let handler = ControlHandler::new(Arc::clone(&registry));
9770        let hello = Frame::build(
9771            FrameType::Hello,
9772            control_flags(),
9773            0,
9774            0,
9775            220,
9776            serde_json::to_vec(&ModuleHelloBody {
9777                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9778                protocol_ver: PROTOCOL_VERSION,
9779                control_ops: None,
9780                launch_nonce: Some("unverified".to_string()),
9781            })
9782            .unwrap(),
9783        )
9784        .unwrap();
9785        assert_eq!(
9786            handler
9787                .handle_control(ConnectionId::new(220), hello)
9788                .unwrap()[0]
9789                .header
9790                .ty,
9791            FrameType::HelloAck
9792        );
9793        let request = Frame::build(
9794            FrameType::Request,
9795            control_flags(),
9796            0,
9797            0,
9798            221,
9799            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9800        )
9801        .unwrap();
9802        let response = handler.handle_catalog_list(request, None).unwrap();
9803        let ClientControlResponse::CatalogList { modules, .. } =
9804            serde_json::from_slice(&response[0].body).unwrap()
9805        else {
9806            panic!("expected catalog.list");
9807        };
9808        assert_eq!(
9809            serde_json::to_vec(&modules[0]).unwrap(),
9810            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9811        );
9812        assert!(!modules[0].nonce_verified);
9813    }
9814
9815    #[test]
9816    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9817        let registry = Arc::new(Registry::default());
9818        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9819            [("vault".to_string(), true), ("squatter".to_string(), true)],
9820            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9821        );
9822        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9823        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9824            provides: vec!["credentials-provider/v1".to_string()],
9825            requires: Vec::new(),
9826            must_never_reach: Vec::new(),
9827        });
9828        let frame = Frame::build(
9829            FrameType::Hello,
9830            control_flags(),
9831            0,
9832            0,
9833            77,
9834            serde_json::to_vec(&ModuleHelloBody {
9835                manifest: squatter,
9836                protocol_ver: PROTOCOL_VERSION,
9837                control_ops: None,
9838                launch_nonce: None,
9839            })
9840            .expect("HELLO serializes"),
9841        )
9842        .expect("HELLO frame builds");
9843        let response = handler
9844            .handle_control(ConnectionId::new(77), frame)
9845            .expect("reserved claim receives a typed refusal");
9846        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9847        assert_eq!(
9848            registry
9849                .active_registration_count()
9850                .expect("registry reads"),
9851            0,
9852            "a reserved capability refusal must not leave a catalog entry"
9853        );
9854    }
9855
9856    #[test]
9857    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9858        for settlement in ["timeout", "inconclusive", "drop"] {
9859            let breakers = RouteBindBreakers::default();
9860            let RouteBindAdmission::Admitted {
9861                guard: mut old,
9862                probe: false,
9863            } = breakers.admit("prov")
9864            else {
9865                panic!("ordinary relay admitted")
9866            };
9867            let RouteBindAdmission::Admitted {
9868                guard: mut opener, ..
9869            } = breakers.admit("prov")
9870            else {
9871                panic!("second relay admitted")
9872            };
9873            assert!(
9874                !opener
9875                    .record_timeout(1, Duration::ZERO)
9876                    .unwrap()
9877                    .reopened_after_probe
9878            );
9879            let RouteBindAdmission::Admitted {
9880                guard: mut probe,
9881                probe: true,
9882            } = breakers.admit("prov")
9883            else {
9884                panic!("one cooldown probe admitted")
9885            };
9886            match settlement {
9887                "timeout" => assert!(
9888                    !old.record_timeout(1, Duration::ZERO)
9889                        .unwrap()
9890                        .reopened_after_probe
9891                ),
9892                "inconclusive" => old.record_inconclusive(),
9893                "drop" => drop(old),
9894                _ => unreachable!(),
9895            }
9896            assert!(
9897                matches!(
9898                    breakers.admit("prov"),
9899                    RouteBindAdmission::Refused {
9900                        probe_in_flight: true,
9901                        ..
9902                    }
9903                ),
9904                "{settlement} of a pre-open relay cannot release the real probe"
9905            );
9906            assert!(
9907                probe
9908                    .record_timeout(1, Duration::ZERO)
9909                    .unwrap()
9910                    .reopened_after_probe
9911            );
9912            assert!(matches!(
9913                breakers.admit("prov"),
9914                RouteBindAdmission::Admitted { probe: true, .. }
9915            ));
9916        }
9917        let breakers = RouteBindBreakers::default();
9918        let admit = || match breakers.admit("prov") {
9919            RouteBindAdmission::Admitted { guard, .. } => guard,
9920            _ => panic!("relay admitted"),
9921        };
9922        admit().record_timeout(1, Duration::ZERO);
9923        let mut old_probe = admit();
9924        breakers.reset_for_new_module_connection("prov");
9925        admit().record_timeout(1, Duration::ZERO);
9926        let _new_probe = admit();
9927        old_probe.record_inconclusive();
9928        assert!(matches!(
9929            breakers.admit("prov"),
9930            RouteBindAdmission::Refused {
9931                probe_in_flight: true,
9932                ..
9933            }
9934        ));
9935    }
9936
9937    #[tokio::test]
9938    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9939        for candidate in [false, true] {
9940            let registry = Arc::new(Registry::default());
9941            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9942                [("vault".to_string(), true), ("squatter".to_string(), true)],
9943                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9944            );
9945            let conn = ConnectionId::new(77);
9946            let (ctx, mut rx) = route_ctx(conn);
9947            let initial = capability_manifest("squatter", &[], &[]);
9948            if candidate {
9949                registry
9950                    .register_candidate_with_control_ops(
9951                        initial.clone(),
9952                        PROTOCOL_VERSION,
9953                        conn,
9954                        module_baseline_control_ops(),
9955                    )
9956                    .unwrap();
9957                handler
9958                    .forwarding
9959                    .register_candidate_module_connection(
9960                        conn,
9961                        "squatter".to_string(),
9962                        PROTOCOL_VERSION,
9963                        manifest_concurrency(&initial),
9964                        ctx.egress.clone(),
9965                    )
9966                    .unwrap();
9967            } else {
9968                hello_via_sink(
9969                    &handler,
9970                    &ctx,
9971                    &mut rx,
9972                    hello_frame_with_manifest(initial.clone(), 1),
9973                )
9974                .await;
9975            }
9976            let response = handler
9977                .handle_control_frame(
9978                    &ctx,
9979                    catalog_update_with_capabilities_frame(
9980                        2,
9981                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
9982                            .capabilities
9983                            .unwrap(),
9984                    ),
9985                )
9986                .await
9987                .unwrap();
9988            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9989            assert_eq!(
9990                registry
9991                    .get_module_by_connection(conn)
9992                    .unwrap()
9993                    .unwrap()
9994                    .manifest,
9995                initial
9996            );
9997        }
9998    }
9999
10000    #[test]
10001    fn server_describe_surfaces_required_capability_verdict_fields() {
10002        let registry = Arc::new(Registry::default());
10003        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10004            [
10005                ("consumer".to_string(), true),
10006                ("provider".to_string(), false),
10007            ],
10008            BTreeMap::new(),
10009        );
10010        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10011        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10012            provides: Vec::new(),
10013            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10014                capability: "credentials-provider/v1".to_string(),
10015                need: subc_protocol::manifest::CapabilityNeed::Required,
10016            }],
10017            must_never_reach: Vec::new(),
10018        });
10019        let hello = Frame::build(
10020            FrameType::Hello,
10021            control_flags(),
10022            0,
10023            0,
10024            78,
10025            serde_json::to_vec(&ModuleHelloBody {
10026                manifest: consumer,
10027                protocol_ver: PROTOCOL_VERSION,
10028                control_ops: None,
10029                launch_nonce: None,
10030            })
10031            .expect("HELLO serializes"),
10032        )
10033        .expect("HELLO frame builds");
10034        handler
10035            .handle_control(ConnectionId::new(78), hello)
10036            .expect("consumer registers");
10037        let describe = Frame::build(
10038            FrameType::Request,
10039            control_flags(),
10040            0,
10041            0,
10042            79,
10043            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10044                .expect("request serializes"),
10045        )
10046        .expect("describe frame builds");
10047        let response = handler
10048            .handle_server_describe(describe)
10049            .expect("server.describe succeeds");
10050        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10051        let requirement = &rendered["capability_requirements"][0];
10052        assert_eq!(requirement["consumer"], "consumer");
10053        assert_eq!(requirement["verdict"], "never_provided");
10054        assert_eq!(requirement["episode_seq"], 1);
10055        assert_eq!(requirement["config_satisfiable"], false);
10056        assert_eq!(requirement["runtime_available"], false);
10057        assert!(requirement["detail"]
10058            .as_str()
10059            .expect("detail string")
10060            .contains("credentials-provider/v1"));
10061    }
10062
10063    /// Register a consumer that requires `project-identity/v1` and the provider
10064    /// of it, optionally mark daemon shutdown as begun, then deregister the
10065    /// provider as its connection closes. Returns the `capability.requirement`
10066    /// events that deregistration logged and the verdict the evaluator then
10067    /// holds for the requirement (the one `route.open` refusals read).
10068    fn deregister_required_provider(
10069        daemon_shutdown_begun: bool,
10070    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10071        const CAPABILITY: &str = "project-identity/v1";
10072        let registry = Arc::new(Registry::default());
10073        let daemon_shutdown = DaemonShutdownFlag::default();
10074        let handler = ControlHandler::new(Arc::clone(&registry))
10075            .with_capability_config(
10076                [
10077                    ("consumer".to_string(), true),
10078                    ("provider".to_string(), true),
10079                ],
10080                BTreeMap::new(),
10081            )
10082            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10083        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10084        consumer.capabilities = Some(CapabilityDeclarations {
10085            provides: Vec::new(),
10086            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10087                capability: CAPABILITY.to_string(),
10088                need: subc_protocol::manifest::CapabilityNeed::Required,
10089            }],
10090            must_never_reach: Vec::new(),
10091        });
10092        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10093        let provider_conn = ConnectionId::new(182);
10094        for (conn, manifest, corr) in [
10095            (ConnectionId::new(181), consumer, 181),
10096            (provider_conn, provider, 182),
10097        ] {
10098            let hello = handler
10099                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10100                .expect("module registers");
10101            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10102        }
10103        assert_eq!(
10104            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10105            Some(CapabilityVerdict::Provided),
10106            "the registered provider must satisfy the requirement first"
10107        );
10108        if daemon_shutdown_begun {
10109            daemon_shutdown.set_for_test();
10110        }
10111
10112        let capture = EventCapture::default();
10113        let _subscriber =
10114            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10115        handler
10116            .cleanup_connection(provider_conn)
10117            .expect("provider deregisters");
10118        let events = capture
10119            .events()
10120            .into_iter()
10121            .filter(|event| {
10122                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10123            })
10124            .collect();
10125        (
10126            events,
10127            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10128        )
10129    }
10130
10131    #[test]
10132    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10133        let (events, verdict) = deregister_required_provider(true);
10134        assert_eq!(
10135            verdict,
10136            Some(CapabilityVerdict::NeverProvided),
10137            "shutdown must not change the verdict route.open refusals read"
10138        );
10139        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10140        let event = &events[0];
10141        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10142        assert_eq!(
10143            event.fields.get("cause").map(String::as_str),
10144            Some("\"daemon_shutdown\""),
10145            "{event:?}"
10146        );
10147        assert_eq!(
10148            event.fields.get("verdict").map(String::as_str),
10149            Some("\"never_provided\""),
10150            "{event:?}"
10151        );
10152    }
10153
10154    #[test]
10155    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10156        let (events, verdict) = deregister_required_provider(false);
10157        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10158        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10159        let event = &events[0];
10160        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10161        assert_eq!(event.fields.get("cause"), None, "{event:?}");
10162        assert_eq!(
10163            event.fields.get("verdict").map(String::as_str),
10164            Some("\"never_provided\""),
10165            "{event:?}"
10166        );
10167    }
10168
10169    #[test]
10170    fn catalog_list_omits_capabilities_for_legacy_manifest() {
10171        let registry = Arc::new(Registry::default());
10172        let handler = ControlHandler::new(Arc::clone(&registry));
10173        let hello = handler
10174            .handle_control(
10175                ConnectionId::new(101),
10176                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10177            )
10178            .expect("legacy HELLO registers");
10179        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10180
10181        let request = Frame::build(
10182            FrameType::Request,
10183            control_flags(),
10184            0,
10185            0,
10186            102,
10187            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10188                .expect("catalog request serializes"),
10189        )
10190        .expect("catalog request frame builds");
10191        let response = handler
10192            .handle_catalog_list(request, None)
10193            .expect("catalog list succeeds");
10194        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10195        assert!(
10196            body["modules"][0].get("capabilities").is_none(),
10197            "legacy manifest must retain an absent capabilities field on catalog.list"
10198        );
10199    }
10200
10201    #[test]
10202    fn hello_ack_omits_storage_when_no_storage_config() {
10203        let registry = Arc::new(Registry::default());
10204        let handler = ControlHandler::new(Arc::clone(&registry));
10205        let responses = handler
10206            .handle_control(
10207                ConnectionId::new(1),
10208                hello_frame("aft", PROTOCOL_VERSION, 7),
10209            )
10210            .unwrap();
10211        let ack = parse_ack(&responses[0]);
10212        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10213        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10214    }
10215
10216    #[tokio::test]
10217    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10218        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10219        let registry = Arc::new(Registry::default());
10220        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
10221        let responses = handler
10222            .handle_control(
10223                ConnectionId::new(1),
10224                hello_frame("aft", PROTOCOL_VERSION, 7),
10225            )
10226            .unwrap();
10227        let ack = parse_ack(&responses[0]);
10228        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10229
10230        let described = handler
10231            .handle_control_frame(
10232                &route_ctx(ConnectionId::new(2)).0,
10233                Frame::build(
10234                    FrameType::Request,
10235                    control_flags(),
10236                    0,
10237                    0,
10238                    9,
10239                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10240                )
10241                .unwrap(),
10242            )
10243            .await
10244            .unwrap();
10245        let ClientControlResponse::ServerDescribe { machine_id, .. } =
10246            serde_json::from_slice(&described[0].body).unwrap()
10247        else {
10248            panic!("server.describe answered with another shape");
10249        };
10250        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10251    }
10252
10253    #[test]
10254    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10255        // With a central sqlite storage policy, each registering module gets its
10256        // own resolved descriptor in HELLO_ACK, keyed by its module id.
10257        let registry = Arc::new(Registry::default());
10258        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
10259            crate::daemon_config::StorageConfig::Sqlite {
10260                data_home: std::path::PathBuf::from("/data"),
10261            },
10262        ));
10263
10264        let responses = handler
10265            .handle_control(
10266                ConnectionId::new(1),
10267                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10268            )
10269            .unwrap();
10270        let ack = parse_ack(&responses[0]);
10271        assert_eq!(
10272            ack.storage,
10273            Some(serde_json::json!({
10274                "module_id": "alfonso-routing",
10275                "storage_namespace": "default",
10276                "isolation": { "kind": "module" },
10277                "backend": {
10278                    "backend": "sqlite",
10279                    "path": "/data/cortexkit/alfonso-routing/store.db"
10280                }
10281            })),
10282            "the delivered descriptor is the module's own sqlite store path"
10283        );
10284    }
10285
10286    #[test]
10287    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10288        let registry = Arc::new(Registry::default());
10289        let handler = ControlHandler::new(Arc::clone(&registry));
10290        let conn = ConnectionId::new(1);
10291        let responses = handler
10292            .handle_control(
10293                conn,
10294                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10295            )
10296            .unwrap();
10297        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10298        let registration = registry.get_module("aft").unwrap().unwrap();
10299        assert_eq!(registration.control_ops, module_baseline_control_ops());
10300
10301        let frame =
10302            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10303        assert!(handler
10304            .guard_module_control_op(&frame, "aft", "route.bind")
10305            .unwrap()
10306            .is_none());
10307        let error = handler
10308            .guard_module_control_op(&frame, "aft", "test.synthetic")
10309            .unwrap()
10310            .expect("synthetic ungranted op should be rejected");
10311        assert_eq!(error.header.ty, FrameType::Error);
10312        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10313    }
10314
10315    #[test]
10316    fn hello_control_ops_some_adds_optional_grants() {
10317        let registry = Arc::new(Registry::default());
10318        let handler = ControlHandler::new(Arc::clone(&registry));
10319        handler
10320            .handle_control(
10321                ConnectionId::new(1),
10322                hello_frame_with_control_ops(
10323                    "aft",
10324                    PROTOCOL_VERSION,
10325                    7,
10326                    Some(vec![
10327                        "future.synthetic".to_string(),
10328                        "route.bind".to_string(),
10329                    ]),
10330                ),
10331            )
10332            .unwrap();
10333        let registration = registry.get_module("aft").unwrap().unwrap();
10334        assert_eq!(
10335            registration.control_ops,
10336            vec![
10337                "route.bind".to_string(),
10338                "route.status".to_string(),
10339                "future.synthetic".to_string(),
10340            ]
10341        );
10342        let frame =
10343            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10344        assert!(handler
10345            .guard_module_control_op(&frame, "aft", "future.synthetic")
10346            .unwrap()
10347            .is_none());
10348    }
10349
10350    #[tokio::test]
10351    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10352        let registry = Arc::new(Registry::default());
10353        let forwarding = Arc::new(ForwardingTable::default());
10354        let handler =
10355            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10356        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10357        hello_via_sink(
10358            &handler,
10359            &module_ctx,
10360            &mut module_rx,
10361            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10362        )
10363        .await;
10364
10365        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10366        let responses = handler
10367            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10368            .await
10369            .unwrap();
10370        assert_eq!(responses.len(), 1);
10371        assert_eq!(responses[0].header.ty, FrameType::Error);
10372        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10373        assert!(module_rx.try_recv().is_err());
10374    }
10375
10376    #[tokio::test]
10377    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10378        let registry = Arc::new(Registry::default());
10379        let forwarding = Arc::new(ForwardingTable::default());
10380        let handler =
10381            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10382        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10383        hello_via_sink(
10384            &handler,
10385            &module_ctx,
10386            &mut module_rx,
10387            hello_frame_with_control_ops(
10388                "aft",
10389                PROTOCOL_VERSION,
10390                7,
10391                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10392            ),
10393        )
10394        .await;
10395
10396        let project_root = unique_project_root("demux");
10397        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10398        let route_handler = handler.clone();
10399        let route_task = tokio::spawn(async move {
10400            route_handler
10401                .handle_control_frame(
10402                    &route_client_ctx,
10403                    route_open_frame(100, "aft", project_root),
10404                )
10405                .await
10406                .unwrap()
10407        });
10408        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10409            .await
10410            .unwrap()
10411            .unwrap();
10412        assert!(matches!(
10413            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10414            ModuleControlRequest::RouteBind { .. }
10415        ));
10416
10417        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10418        let health_handler = handler.clone();
10419        let health_task = tokio::spawn(async move {
10420            health_handler
10421                .handle_control_frame(
10422                    &health_client_ctx,
10423                    supervisor_health_probe_frame(101, "aft"),
10424                )
10425                .await
10426                .unwrap()
10427        });
10428        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10429            .await
10430            .unwrap()
10431            .unwrap();
10432        assert_eq!(
10433            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10434            ModuleControlRequest::HealthCheck {}
10435        );
10436
10437        handler
10438            .handle_control_frame(
10439                &module_ctx,
10440                health_response(health_frame.header.corr, HealthStatus::Degraded),
10441            )
10442            .await
10443            .unwrap();
10444        let health_response = health_task.await.unwrap();
10445        assert_eq!(health_response.len(), 1);
10446        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10447            ClientControlResponse::SupervisorHealthProbe {
10448                module_id,
10449                status,
10450                detail,
10451                metrics,
10452            } => {
10453                assert_eq!(module_id, "aft");
10454                assert_eq!(status, HealthStatus::Degraded);
10455                assert_eq!(detail.as_deref(), Some("warming"));
10456                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10457            }
10458            other => panic!("unexpected health response: {other:?}"),
10459        }
10460
10461        handler
10462            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10463            .await
10464            .unwrap();
10465        let route_response = route_task.await.unwrap();
10466        assert!(route_response.is_empty());
10467        let published = route_client_rx.recv().await.unwrap();
10468        assert!(matches!(
10469            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10470            ClientControlResponse::RouteOpen { .. }
10471        ));
10472    }
10473
10474    /// Start one `route.open` on `client_connection` and return its still-running
10475    /// handler task together with the `route.bind` the module received for it.
10476    /// The handler blocks until the module answers, so it has to run as a task
10477    /// while the test drives the module side.
10478    async fn relay_route_open(
10479        handler: &ControlHandler,
10480        client_connection: ConnectionId,
10481        client_egress: &FrameSink,
10482        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10483        corr: u64,
10484        module_id: &str,
10485        project_root_label: &str,
10486    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10487        let ctx = RouteCtx {
10488            connection_id: client_connection,
10489            egress: client_egress.clone(),
10490        };
10491        let handler = handler.clone();
10492        let project_root = unique_project_root(project_root_label);
10493        let module_id = module_id.to_string();
10494        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10495        let task = tokio::spawn(async move {
10496            let _guard = tracing::dispatcher::set_default(&dispatch);
10497            handler
10498                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10499                .await
10500                .unwrap()
10501        });
10502        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10503            .await
10504            .expect("module receives the relayed route.bind")
10505            .expect("module egress is open");
10506        (task, bind.frame)
10507    }
10508
10509    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10510        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10511            ModuleControlRequest::RouteBind {
10512                route_channel,
10513                epoch,
10514                ..
10515            } => (route_channel, epoch),
10516            other => panic!("expected a route.bind request, got {other:?}"),
10517        }
10518    }
10519
10520    fn published_route(frame: &Frame) -> (u16, u32) {
10521        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10522            ClientControlResponse::RouteOpen {
10523                route_channel,
10524                route_epoch,
10525            } => (route_channel, route_epoch),
10526            other => panic!("expected a route.open response, got {other:?}"),
10527        }
10528    }
10529
10530    /// Reproduction of a production outage. A client had `route.open`s in
10531    /// flight to a module and was already marked closing -- its egress had refused a
10532    /// module frame, so the daemon asked its connection to end -- while its sink
10533    /// was still open. When the module acked those binds, the daemon refused to
10534    /// commit a route for a closing client, and that refusal was returned from
10535    /// the MODULE connection's frame handler, where a router error that has no
10536    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
10537    /// the supervisor correctly did not respawn a clean exit, and every seat lost
10538    /// its tools for hours -- one client's teardown took down a connection
10539    /// carrying ~170 other routes.
10540    ///
10541    /// The window is opened here by calling the production path that opens it
10542    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
10543    /// state that matters is "in `closing_connections`, sink still open, relay
10544    /// still pending", and it lasts only from the close request until the
10545    /// connection loop reacts to it; a socket-level test can flood a client into
10546    /// that escalation but cannot pin the module's ack inside the window. Closing
10547    /// the socket instead takes the other path entirely -- connection teardown
10548    /// removes the pending relay under the same lock, so the ack finds nothing.
10549    #[tokio::test]
10550    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10551        let registry = Arc::new(Registry::default());
10552        let forwarding = Arc::new(ForwardingTable::default());
10553        let handler =
10554            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10555
10556        let module_connection = ConnectionId::new(30);
10557        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10558        hello_via_sink(
10559            &handler,
10560            &module_ctx,
10561            &mut module_rx,
10562            hello_frame("aft", PROTOCOL_VERSION, 7),
10563        )
10564        .await;
10565
10566        let dying_client = ConnectionId::new(31);
10567        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10568
10569        // A published route on the dying client. The escalation below only marks
10570        // a connection closing for a route it has already published.
10571        let (first_task, first_bind) = relay_route_open(
10572            &handler,
10573            dying_client,
10574            &dying_ctx.egress,
10575            &mut module_rx,
10576            100,
10577            "aft",
10578            "closing-first",
10579        )
10580        .await;
10581        handler
10582            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10583            .await
10584            .unwrap();
10585        assert!(first_task.await.unwrap().is_empty());
10586        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10587
10588        // A second route.open from the same client, relayed and awaiting its ack.
10589        let (second_task, second_bind) = relay_route_open(
10590            &handler,
10591            dying_client,
10592            &dying_ctx.egress,
10593            &mut module_rx,
10594            101,
10595            "aft",
10596            "closing-second",
10597        )
10598        .await;
10599        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10600
10601        // The window: the client is closing, its sink is still open, and its
10602        // second bind is still pending.
10603        assert!(forwarding
10604            .escalate_client_delivery_failure(
10605                dying_client,
10606                first_channel,
10607                first_epoch,
10608                CloseReason::new(
10609                    "module_to_client_delivery_failed",
10610                    "client egress refused a module frame",
10611                ),
10612                crate::forwarding::UndeliveredFrame {
10613                    module_id: None,
10614                    sink: &dying_ctx.egress,
10615                },
10616            )
10617            .unwrap());
10618        assert!(!dying_ctx.egress.is_closed());
10619
10620        // The frame that used to end the module connection.
10621        let ack = handler
10622            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10623            .await;
10624        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10625        if module_loop_error.is_some() {
10626            // What the server's connection loop does with a router error that has
10627            // no ERROR-frame translation: end the connection, which releases the
10628            // module's registration and every route on it.
10629            handler.cleanup_connection(module_connection).unwrap();
10630        }
10631        // Read the module's next frame before opening the co-tenant's route, so
10632        // the GOODBYE assertion below is about THIS ack and not about later
10633        // traffic. `None` means the module was told nothing.
10634        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10635            .await
10636            .ok()
10637            .flatten();
10638
10639        // 1. The module connection is still registered.
10640        assert!(
10641            registry
10642                .get_module_by_connection(module_connection)
10643                .unwrap()
10644                .is_some(),
10645            "one client's closing connection ended the shared module connection: \
10646             {module_loop_error:?}"
10647        );
10648        // ...and still serving: another client can open and use a route on it.
10649        let cotenant = ConnectionId::new(32);
10650        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10651        let (cotenant_task, cotenant_bind) = relay_route_open(
10652            &handler,
10653            cotenant,
10654            &cotenant_ctx.egress,
10655            &mut module_rx,
10656            102,
10657            "aft",
10658            "closing-cotenant",
10659        )
10660        .await;
10661        handler
10662            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10663            .await
10664            .unwrap();
10665        assert!(cotenant_task.await.unwrap().is_empty());
10666        let (cotenant_channel, cotenant_epoch) =
10667            published_route(&cotenant_rx.recv().await.unwrap());
10668        assert!(matches!(
10669            forwarding
10670                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10671                .unwrap(),
10672            DataRoute::Client(DataRouteState::Bound(_))
10673        ));
10674
10675        // 2. The module was told to drop the binding it created for the route
10676        //    that will never be published.
10677        let goodbye = post_ack_module_frame
10678            .expect("module receives a GOODBYE for the abandoned route channel");
10679        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10680        assert_eq!(goodbye.header.channel, abandoned_channel);
10681        assert_eq!(goodbye.header.epoch, abandoned_epoch);
10682
10683        // 3. The dying client received nothing: no route was ever published to
10684        //    it. Its route.open is answered as unavailable, which the connection
10685        //    loop would write to a socket that is already going away.
10686        assert!(dying_rx.try_recv().is_err());
10687        let second_response = second_task.await.unwrap();
10688        assert_eq!(second_response.len(), 1);
10689        assert_eq!(
10690            parse_error(&second_response[0])["code"],
10691            "target_unavailable"
10692        );
10693    }
10694
10695    /// One `catalog.list` as a client sees it: the generation and the listed
10696    /// module ids with their readiness.
10697    async fn catalog_snapshot(
10698        handler: &ControlHandler,
10699        client_ctx: &RouteCtx,
10700        corr: u64,
10701    ) -> (u64, Vec<(String, bool)>) {
10702        let body =
10703            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10704        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10705        let reply = handler
10706            .handle_control_frame(client_ctx, request)
10707            .await
10708            .unwrap()
10709            .pop()
10710            .unwrap();
10711        let ClientControlResponse::CatalogList {
10712            generation,
10713            modules,
10714            ..
10715        } = serde_json::from_slice(&reply.body).unwrap()
10716        else {
10717            panic!("not a catalog.list reply");
10718        };
10719        (
10720            generation,
10721            modules
10722                .into_iter()
10723                .map(|entry| (entry.module_id, entry.ready))
10724                .collect(),
10725        )
10726    }
10727
10728    /// A handler whose HELLO for module "aft" on connection 40 is parked
10729    /// between reserving its registry entry and installing its forwarding
10730    /// endpoint. Returns the handler, the two-party barrier (wait once to know
10731    /// the HELLO is parked, once more to release it), the HELLO's thread, and
10732    /// the module's context and receiver.
10733    fn hello_held_before_forwarding() -> (
10734        ControlHandler,
10735        Arc<std::sync::Barrier>,
10736        std::thread::JoinHandle<Vec<Frame>>,
10737        RouteCtx,
10738        mpsc::Receiver<crate::router::OutboundFrame>,
10739    ) {
10740        let hold = Arc::new(std::sync::Barrier::new(2));
10741        let handler = ControlHandler::with_forwarding(
10742            Arc::new(Registry::default()),
10743            Arc::new(ForwardingTable::default()),
10744        )
10745        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10746        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10747        let hello = {
10748            let handler = handler.clone();
10749            let module_ctx = module_ctx.clone();
10750            std::thread::spawn(move || {
10751                handler
10752                    .handle_hello(
10753                        module_ctx.connection_id,
10754                        Some(module_ctx.egress.clone()),
10755                        hello_frame("aft", PROTOCOL_VERSION, 7),
10756                    )
10757                    .unwrap()
10758            })
10759        };
10760        (handler, hold, hello, module_ctx, module_rx)
10761    }
10762
10763    /// While a HELLO has reserved its registry entry but its forwarding
10764    /// endpoint is not installed yet, `route.open` to the module can only be
10765    /// refused. `catalog.list` must not list the module in that window: a
10766    /// client that opens a route as soon as it sees a module listed and ready
10767    /// would be refused for a module the daemon had just advertised. The
10768    /// refusal in the window stays the retryable `target_unavailable`, never
10769    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10770    /// Once the HELLO completes, the module is listed and a route to it opens.
10771    #[tokio::test]
10772    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10773        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10774        let client = ConnectionId::new(41);
10775        let (client_ctx, mut client_rx) = route_ctx(client);
10776
10777        hold.wait();
10778        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10779        assert!(handler.registry.get_module("aft").unwrap().is_some());
10780        assert!(!handler
10781            .forwarding
10782            .has_live_module_connection("aft")
10783            .unwrap());
10784        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10785        assert!(
10786            listed.is_empty(),
10787            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10788        );
10789        let refused = handler
10790            .handle_control_frame(
10791                &client_ctx,
10792                route_open_frame(301, "aft", unique_project_root("listed-window")),
10793            )
10794            .await
10795            .unwrap();
10796        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10797        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10798        let duplicate = handler
10799            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10800            .await
10801            .unwrap();
10802        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10803
10804        hold.wait();
10805        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10806        let ack = module_rx.try_recv().unwrap().frame;
10807        assert_eq!(ack.header.ty, FrameType::HelloAck);
10808        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10809        assert_eq!(listed, vec![("aft".to_string(), true)]);
10810
10811        let (open_task, bind) = relay_route_open(
10812            &handler,
10813            client,
10814            &client_ctx.egress,
10815            &mut module_rx,
10816            303,
10817            "aft",
10818            "listed-after",
10819        )
10820        .await;
10821        assert!(handler
10822            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10823            .await
10824            .unwrap()
10825            .is_empty());
10826        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10827        published_route(&client_rx.try_recv().unwrap().frame);
10828    }
10829
10830    /// A catalog watcher keys on the generation: when it changes, it re-reads
10831    /// the listing. So the generation must advance exactly when the module
10832    /// appears. If it advanced while the module was still hidden, a watcher
10833    /// would read the new generation without the module, and the module would
10834    /// later appear under that same generation with nothing telling the
10835    /// watcher to look again.
10836    #[tokio::test]
10837    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10838        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10839        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10840        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10841
10842        hold.wait();
10843        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10844        assert!(listed.is_empty());
10845        assert_eq!(
10846            during, before,
10847            "the generation moved while the module was not listed"
10848        );
10849
10850        hold.wait();
10851        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10852        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10853        assert_eq!(listed, vec![("aft".to_string(), true)]);
10854        assert_eq!(
10855            after,
10856            before + 1,
10857            "the module appeared without exactly one generation bump"
10858        );
10859    }
10860
10861    /// A HELLO whose forwarding endpoint cannot be installed (here the
10862    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
10863    /// refused and leaves nothing behind: no registry entry, no catalog entry,
10864    /// and the catalog generation where it was.
10865    #[tokio::test]
10866    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10867        let registry = Arc::new(Registry::default());
10868        let handler = ControlHandler::with_forwarding(
10869            Arc::clone(&registry),
10870            Arc::new(ForwardingTable::default()),
10871        );
10872        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10873        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10874        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10875        drop(module_rx);
10876
10877        let replies = handler
10878            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10879            .await
10880            .unwrap();
10881        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10882        assert!(registry.get_module("aft").unwrap().is_none());
10883        assert!(registry
10884            .get_module_by_connection(module_ctx.connection_id)
10885            .unwrap()
10886            .is_none());
10887        assert_eq!(
10888            catalog_snapshot(&handler, &client_ctx, 301).await,
10889            (before, Vec::new())
10890        );
10891    }
10892
10893    /// A handler whose `route.open` parks after queueing the route.bind and
10894    /// before marking it enqueued, with module "aft" registered on connection
10895    /// 40. Returns the two-party barrier that parks and releases it.
10896    async fn route_open_held_after_bind_send() -> (
10897        ControlHandler,
10898        Arc<tokio::sync::Barrier>,
10899        RouteCtx,
10900        mpsc::Receiver<crate::router::OutboundFrame>,
10901    ) {
10902        let hold = Arc::new(tokio::sync::Barrier::new(2));
10903        let handler = ControlHandler::with_forwarding(
10904            Arc::new(Registry::default()),
10905            Arc::new(ForwardingTable::default()),
10906        )
10907        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10908        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10909        hello_via_sink(
10910            &handler,
10911            &module_ctx,
10912            &mut module_rx,
10913            hello_frame("aft", PROTOCOL_VERSION, 7),
10914        )
10915        .await;
10916        (handler, hold, module_ctx, module_rx)
10917    }
10918
10919    /// A fast module acks the route.bind, and the reader commits the route,
10920    /// before `route.open` has marked the relay enqueued. The daemon used to
10921    /// read the missing relay as an abandoned bind and send the module a
10922    /// GOODBYE for the route it had just committed: the client was told its
10923    /// route was open, the module unbound the channel, and the client's first
10924    /// request was dropped on the module side as unbound.
10925    #[tokio::test]
10926    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10927        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10928        let client = ConnectionId::new(41);
10929        let (client_ctx, mut client_rx) = route_ctx(client);
10930
10931        let (open_task, bind) = relay_route_open(
10932            &handler,
10933            client,
10934            &client_ctx.egress,
10935            &mut module_rx,
10936            200,
10937            "aft",
10938            "acked-before-mark",
10939        )
10940        .await;
10941        let (module_channel, module_epoch) = route_bind_channel(&bind);
10942        // route.open is now parked between queueing the bind and marking it.
10943        hold.wait().await;
10944
10945        // The module answers inside that window, and the route commits.
10946        assert!(handler
10947            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10948            .await
10949            .unwrap()
10950            .is_empty());
10951        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10952
10953        hold.wait().await;
10954        assert!(
10955            open_task.await.unwrap().is_empty(),
10956            "route.open answers accepted"
10957        );
10958
10959        // Every frame route.open could have sent is already queued, because it
10960        // has returned; the module must have been sent nothing.
10961        match module_rx.try_recv() {
10962            Err(mpsc::error::TryRecvError::Empty) => {}
10963            Ok(outbound) => panic!(
10964                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10965                outbound.frame.header.ty,
10966                outbound.frame.header.channel,
10967                outbound.frame.header.epoch
10968            ),
10969            Err(err) => panic!("module egress closed: {err}"),
10970        }
10971
10972        // The client's first request on the route is the next thing the module
10973        // reads, on the binding it just created.
10974        let router = Router::with_control_handler(Arc::new(handler.clone()));
10975        let request = Frame::build(
10976            FrameType::Request,
10977            Flags::new(true, Priority::Interactive, false),
10978            client_channel,
10979            client_epoch,
10980            201,
10981            b"first request".to_vec(),
10982        )
10983        .unwrap();
10984        router
10985            .route_for_connection(&client_ctx, request)
10986            .await
10987            .unwrap();
10988        let delivered = module_rx.try_recv().unwrap().frame;
10989        assert_eq!(delivered.header.ty, FrameType::Request);
10990        assert_eq!(
10991            (delivered.header.channel, delivered.header.epoch),
10992            (module_channel, module_epoch)
10993        );
10994        assert_eq!(delivered.body, b"first request");
10995    }
10996
10997    /// The client goes away while `route.open` is still queueing the bind, so
10998    /// the relay is dropped with no module reply. The teardown cannot send the
10999    /// GOODBYE itself (it could reach the module ahead of the bind), so
11000    /// `route.open` must send it once the bind is queued; otherwise the module
11001    /// keeps a binding the daemon will never commit.
11002    #[tokio::test]
11003    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11004        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11005        let client = ConnectionId::new(42);
11006        let (client_ctx, _client_rx) = route_ctx(client);
11007
11008        let (open_task, bind) = relay_route_open(
11009            &handler,
11010            client,
11011            &client_ctx.egress,
11012            &mut module_rx,
11013            300,
11014            "aft",
11015            "released-before-mark",
11016        )
11017        .await;
11018        let (module_channel, module_epoch) = route_bind_channel(&bind);
11019        hold.wait().await;
11020
11021        handler.cleanup_connection(client).unwrap();
11022        assert!(
11023            module_rx.try_recv().is_err(),
11024            "client teardown must not GOODBYE a bind route.open is still queueing"
11025        );
11026
11027        hold.wait().await;
11028        let response = open_task.await.unwrap();
11029        assert_eq!(response.len(), 1);
11030        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11031
11032        let goodbye = module_rx
11033            .try_recv()
11034            .expect("module is told to drop the binding the daemon abandoned")
11035            .frame;
11036        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11037        assert_eq!(
11038            (goodbye.header.channel, goodbye.header.epoch),
11039            (module_channel, module_epoch)
11040        );
11041        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11042    }
11043
11044    /// The module acks inside the same window, but the client's egress closed
11045    /// first, so the reader refuses to commit. The ack proves the module holds
11046    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
11047    /// send a second one.
11048    #[tokio::test]
11049    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11050        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11051        let client = ConnectionId::new(43);
11052        let (client_ctx, client_rx) = route_ctx(client);
11053
11054        let (open_task, bind) = relay_route_open(
11055            &handler,
11056            client,
11057            &client_ctx.egress,
11058            &mut module_rx,
11059            400,
11060            "aft",
11061            "gone-client-before-mark",
11062        )
11063        .await;
11064        let (module_channel, module_epoch) = route_bind_channel(&bind);
11065        hold.wait().await;
11066
11067        drop(client_rx);
11068        assert!(client_ctx.egress.is_closed());
11069        handler
11070            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11071            .await
11072            .unwrap();
11073
11074        hold.wait().await;
11075        let response = open_task.await.unwrap();
11076        assert_eq!(response.len(), 1);
11077        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11078
11079        let goodbye = module_rx
11080            .try_recv()
11081            .expect("module is told to drop the binding it acked")
11082            .frame;
11083        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11084        assert_eq!(
11085            (goodbye.header.channel, goodbye.header.epoch),
11086            (module_channel, module_epoch)
11087        );
11088        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11089        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11090    }
11091
11092    /// The fence at the module-loop boundary, stated as its own contract: which
11093    /// forwarding failures are allowed to end the module connection that is being
11094    /// served. A `ConnectionClosing` naming some client is about that client, and
11095    /// a module connection is shared; the same error naming the module's own
11096    /// connection is about this connection and must stay fatal, as must failures
11097    /// that are about the forwarding table itself.
11098    #[test]
11099    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11100        let handler = ControlHandler::default();
11101        let module_connection = ConnectionId::new(30);
11102        let client_connection = ConnectionId::new(31);
11103
11104        handler
11105            .refuse_to_end_module_connection_for_a_client(
11106                module_connection,
11107                77,
11108                ForwardingError::ConnectionClosing {
11109                    connection_id: client_connection,
11110                },
11111            )
11112            .expect("a closing client must never end the module connection");
11113
11114        assert!(matches!(
11115            handler.refuse_to_end_module_connection_for_a_client(
11116                module_connection,
11117                78,
11118                ForwardingError::ConnectionClosing {
11119                    connection_id: module_connection,
11120                },
11121            ),
11122            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11123                connection_id
11124            })) if connection_id == module_connection
11125        ));
11126        assert!(matches!(
11127            handler.refuse_to_end_module_connection_for_a_client(
11128                module_connection,
11129                79,
11130                ForwardingError::Poisoned,
11131            ),
11132            Err(RouterError::Forwarding(ForwardingError::Poisoned))
11133        ));
11134        assert!(matches!(
11135            handler.refuse_to_end_module_connection_for_a_client(
11136                module_connection,
11137                80,
11138                ForwardingError::StaleModuleEndpoint,
11139            ),
11140            Err(RouterError::Forwarding(
11141                ForwardingError::StaleModuleEndpoint
11142            ))
11143        ));
11144    }
11145
11146    /// The spawn-attestation guard is what stops a connected module from claiming
11147    /// another module's identity and being stamped `Reserved` for it. Every other
11148    /// test that supplies a consumer_identity supplies a CORRECT one, because a
11149    /// correct one is what the rest of the flow needs -- so the guard's rejection
11150    /// branch was never the subject of an assertion, only its acceptance branch.
11151    ///
11152    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
11153    /// whole subc-core library suite green; only the forwarding integration tests
11154    /// notice, and they notice for unrelated reasons. This test exists so the
11155    /// refusal itself is asserted where the guard lives: it fails if the identity
11156    /// check stops refusing, which is the direction that matters, since a guard
11157    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
11158    #[tokio::test]
11159    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11160        let registry = Arc::new(Registry::default());
11161        let forwarding = Arc::new(ForwardingTable::default());
11162        let supervisor = SupervisorHandle::new();
11163        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11164        let handler =
11165            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11166                .with_supervisor(supervisor);
11167
11168        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11169        hello_via_sink(
11170            &handler,
11171            &target_ctx,
11172            &mut target_rx,
11173            hello_frame("target", PROTOCOL_VERSION, 1),
11174        )
11175        .await;
11176
11177        // A real supervised module id presenting the wrong nonce. This is the
11178        // impersonation case: the attacker knows a privileged module_id, which is
11179        // public, and guesses at the nonce, which is not.
11180        let wrong_nonce = handler
11181            .handle_control_frame(
11182                &route_ctx(ConnectionId::new(91)).0,
11183                route_open_frame_with_admission_facts(
11184                    20,
11185                    "target",
11186                    unique_project_root("admission-facts"),
11187                    Some(subc_control::ConsumerIdentity {
11188                        module_id: "fed".to_string(),
11189                        launch_nonce: "not-the-real-nonce".to_string(),
11190                    }),
11191                    None,
11192                ),
11193            )
11194            .await
11195            .unwrap();
11196        assert_eq!(
11197            parse_error(&wrong_nonce[0])["code"],
11198            "bad_consumer_identity",
11199            "a mismatched launch nonce must be refused, not stamped Reserved"
11200        );
11201
11202        // A module id the supervisor never spawned at all, so no nonce exists to
11203        // compare against. An implementation that treats "no record" as "nothing
11204        // to check" fails open here while passing the case above.
11205        let never_spawned = handler
11206            .handle_control_frame(
11207                &route_ctx(ConnectionId::new(92)).0,
11208                route_open_frame_with_admission_facts(
11209                    21,
11210                    "target",
11211                    unique_project_root("admission-facts"),
11212                    Some(subc_control::ConsumerIdentity {
11213                        module_id: "never-spawned".to_string(),
11214                        launch_nonce: "any-nonce".to_string(),
11215                    }),
11216                    None,
11217                ),
11218            )
11219            .await
11220            .unwrap();
11221        assert_eq!(
11222            parse_error(&never_spawned[0])["code"],
11223            "bad_consumer_identity",
11224            "an unspawned module_id must be refused rather than accepted for lack of a record"
11225        );
11226    }
11227
11228    /// The refusal test above proves the guard says NO. Nothing proved it can say
11229    /// YES, and the difference is not academic: replacing the whole authorization
11230    /// with `false` -- admitting no consumer identity at all, revoking Reserved
11231    /// standing for every supervised module in the fleet -- leaves 110 of the 111
11232    /// library tests GREEN. The one that notices does so by HANGING, because it
11233    /// waits for a bind that can no longer happen.
11234    ///
11235    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
11236    /// or flaky test, invites a RETRY rather than an investigation, and the retry
11237    /// hangs too and gets blamed on the runner. So a total revocation of the
11238    /// daemon's trust grant would have shipped behind a symptom nobody attributes
11239    /// to code.
11240    ///
11241    /// The bias is structural rather than accidental. A REFUSAL looks like a
11242    /// failure someone writes a test for; a GRANT looks like the happy path. Every
11243    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
11244    /// suite for that reason, and this one is the purest case in the daemon.
11245    ///
11246    /// This test asserts the EFFECT rather than the absence of an error: the module
11247    /// receives a RouteBind and it carries `Reserved` naming the attested module.
11248    /// A guard that admitted nobody would produce no bind at all; one that admitted
11249    /// everybody would stamp the wrong principal, which the refusal test catches.
11250    #[tokio::test]
11251    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11252        let registry = Arc::new(Registry::default());
11253        let forwarding = Arc::new(ForwardingTable::default());
11254        let supervisor = SupervisorHandle::new();
11255        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11256        let handler =
11257            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11258                .with_supervisor(supervisor);
11259
11260        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11261        hello_via_sink(
11262            &handler,
11263            &target_ctx,
11264            &mut target_rx,
11265            hello_frame("target", PROTOCOL_VERSION, 1),
11266        )
11267        .await;
11268
11269        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11270        let route_handler = handler.clone();
11271        let route_task = tokio::spawn(async move {
11272            route_handler
11273                .handle_control_frame(
11274                    &client_ctx,
11275                    route_open_frame_with_admission_facts(
11276                        30,
11277                        "target",
11278                        unique_project_root("admission-facts"),
11279                        Some(subc_control::ConsumerIdentity {
11280                            module_id: "fed".to_string(),
11281                            launch_nonce: "fed-nonce".to_string(),
11282                        }),
11283                        None,
11284                    ),
11285                )
11286                .await
11287                .unwrap()
11288        });
11289
11290        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
11291        // the very mutation it exists to catch -- a guard that admits nobody -- no
11292        // bind is ever sent, so it HUNG rather than failing. That reproduces the
11293        // exact defect being fixed: a total revocation detected only as a stalled
11294        // suite, which reads as flakiness and invites a retry. An acceptance test
11295        // that waits for an effect must bound the wait, or a red becomes a hang.
11296        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11297            .await
11298            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11299            .expect("module control channel closed before route.bind");
11300        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11301        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11302            panic!("expected route.bind")
11303        };
11304        assert_eq!(
11305            principal,
11306            Some(Principal::Reserved {
11307                module_id: "fed".to_string()
11308            }),
11309            "a correctly attested consumer must be stamped Reserved for its own id"
11310        );
11311
11312        handler
11313            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11314            .await
11315            .unwrap();
11316        assert!(route_task.await.unwrap().is_empty());
11317        assert!(
11318            matches!(
11319                serde_json::from_slice::<ClientControlResponse>(
11320                    &client_rx.recv().await.unwrap().body
11321                )
11322                .unwrap(),
11323                ClientControlResponse::RouteOpen { .. }
11324            ),
11325            "the route must actually open, not merely avoid an error"
11326        );
11327    }
11328
11329    #[tokio::test(start_paused = true)]
11330    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11331        let registry = Arc::new(Registry::default());
11332        let forwarding = Arc::new(ForwardingTable::default());
11333        let supervisor = SupervisorHandle::new();
11334        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11335        let handler =
11336            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11337                .with_supervisor(supervisor);
11338
11339        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11340        hello_via_sink(
11341            &handler,
11342            &target_ctx,
11343            &mut target_rx,
11344            hello_frame("target", PROTOCOL_VERSION, 1),
11345        )
11346        .await;
11347
11348        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11349        let direct_handler = handler.clone();
11350        let direct_open = tokio::spawn(async move {
11351            direct_handler
11352                .handle_control_frame(
11353                    &direct_ctx,
11354                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
11355                )
11356                .await
11357                .unwrap()
11358        });
11359        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11360            .await
11361            .expect("no direct route.bind within 5s")
11362            .expect("target control channel closed before direct route.bind");
11363        handler
11364            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11365            .await
11366            .unwrap();
11367        assert!(direct_open.await.unwrap().is_empty());
11368        let _ = direct_rx.recv().await.unwrap();
11369
11370        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11371        let reserved_handler = handler.clone();
11372        let reserved_open = tokio::spawn(async move {
11373            reserved_handler
11374                .handle_control_frame(
11375                    &reserved_ctx,
11376                    route_open_frame_with_admission_facts(
11377                        3,
11378                        "target",
11379                        unique_project_root("admission-facts"),
11380                        Some(ConsumerIdentity {
11381                            module_id: "fed".to_string(),
11382                            launch_nonce: "fed-nonce".to_string(),
11383                        }),
11384                        None,
11385                    ),
11386                )
11387                .await
11388                .unwrap()
11389        });
11390        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11391            .await
11392            .expect("no reserved route.bind within 5s")
11393            .expect("target control channel closed before reserved route.bind");
11394        handler
11395            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11396            .await
11397            .unwrap();
11398        assert!(reserved_open.await.unwrap().is_empty());
11399        let _ = reserved_rx.recv().await.unwrap();
11400
11401        forwarding
11402            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11403            .unwrap();
11404        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11405        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11406            module_id: Some("target".to_string()),
11407        })
11408        .unwrap();
11409        let census_frame =
11410            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11411        let response = handler
11412            .handle_control_frame(&census_ctx, census_frame)
11413            .await
11414            .unwrap()
11415            .pop()
11416            .unwrap();
11417        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11418        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11419        assert!(matches!(
11420            decoded,
11421            ClientControlResponse::SupervisorRoutes { .. }
11422        ));
11423        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11424        assert_eq!(routes.len(), 2);
11425        assert!(routes.iter().all(|route| route["draining"] == true));
11426        // The census carries WHY: the reason the drain was begun with, in the
11427        // route.closing vocabulary, on every draining route this drain marked.
11428        assert!(
11429            routes.iter().all(|route| route["drain_reason"] == "reload"),
11430            "draining routes must name the drain's reason: {routes:?}"
11431        );
11432        assert!(routes.iter().any(|route| {
11433            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11434        }));
11435        assert!(routes.iter().any(|route| {
11436            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11437        }));
11438
11439        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11440            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11441        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11442            std::fs::write(
11443                &golden_path,
11444                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11445            )
11446            .unwrap();
11447        }
11448        let expected: Value =
11449            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11450        assert_eq!(actual, expected);
11451    }
11452
11453    async fn query_live_roots(
11454        handler: &ControlHandler,
11455        module_ctx: &RouteCtx,
11456    ) -> ModuleControlResponseToModule {
11457        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11458        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11459        let response = handler
11460            .handle_control_frame(module_ctx, frame)
11461            .await
11462            .unwrap()
11463            .pop()
11464            .unwrap();
11465        serde_json::from_slice(&response.body).unwrap()
11466    }
11467
11468    #[tokio::test(start_paused = true)]
11469    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11470        let registry = Arc::new(Registry::default());
11471        let forwarding = Arc::new(ForwardingTable::default());
11472        let handler = ControlHandler::with_forwarding(registry, forwarding);
11473        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11474        hello_via_sink(
11475            &handler,
11476            &target_ctx,
11477            &mut target_rx,
11478            hello_frame("target", PROTOCOL_VERSION, 1),
11479        )
11480        .await;
11481        let root = unique_project_root("live-roots-known");
11482        let path = ProjectRootId::from_path_allowing_missing(root.path())
11483            .unwrap()
11484            .as_path()
11485            .to_path_buf();
11486        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11487        let open_handler = handler.clone();
11488        let opened = tokio::spawn(async move {
11489            open_handler
11490                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11491                .await
11492                .unwrap()
11493        });
11494        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11495            .await
11496            .unwrap()
11497            .unwrap();
11498        handler
11499            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11500            .await
11501            .unwrap();
11502        assert!(opened.await.unwrap().is_empty());
11503        let _ = client_rx.recv().await.unwrap();
11504
11505        let root = unique_project_root("live-roots-pending");
11506        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11507            .unwrap()
11508            .as_path()
11509            .to_path_buf();
11510        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11511        let open_handler = handler.clone();
11512        let pending = tokio::spawn(async move {
11513            open_handler
11514                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11515                .await
11516                .unwrap()
11517        });
11518        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11519            .await
11520            .unwrap()
11521            .unwrap();
11522        let actual = query_live_roots(&handler, &target_ctx).await;
11523        let ModuleControlResponseToModule::LiveRoots {
11524            roots,
11525            unknown_root_bindings,
11526            total_bindings,
11527        } = actual
11528        else {
11529            panic!("expected live roots")
11530        };
11531        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11532        assert_eq!(unknown_root_bindings, 0);
11533        assert_eq!(
11534            roots.len(),
11535            2,
11536            "root-known arm must retain each canonical root"
11537        );
11538        assert_eq!(
11539            total_bindings,
11540            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11541        );
11542        let counts = roots
11543            .iter()
11544            .map(|root| (root.project_root.clone(), root.bound, root.pending))
11545            .collect::<Vec<_>>();
11546        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11547        expected.sort_by(|a, b| a.0.cmp(&b.0));
11548        assert_eq!(
11549            counts, expected,
11550            "roots must sort by path and count pending separately"
11551        );
11552        handler
11553            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11554            .await
11555            .unwrap();
11556        assert!(pending.await.unwrap().is_empty());
11557    }
11558
11559    #[tokio::test(start_paused = true)]
11560    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11561        let forwarding = Arc::new(ForwardingTable::default());
11562        let handler =
11563            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11564        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11565        hello_via_sink(
11566            &handler,
11567            &target_ctx,
11568            &mut target_rx,
11569            hello_frame("target", PROTOCOL_VERSION, 1),
11570        )
11571        .await;
11572        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11573        let pending = forwarding
11574            .begin_route_bind_relay_for_test(
11575                client_ctx.connection_id,
11576                client_ctx.egress.clone(),
11577                2,
11578                "target",
11579            )
11580            .unwrap();
11581        forwarding
11582            .complete_pending_relay(
11583                target_ctx.connection_id,
11584                pending.corr,
11585                RouteBindRelayOutcome::Accepted,
11586            )
11587            .unwrap();
11588        let actual = query_live_roots(&handler, &target_ctx).await;
11589        let ModuleControlResponseToModule::LiveRoots {
11590            roots,
11591            unknown_root_bindings,
11592            total_bindings,
11593        } = actual
11594        else {
11595            panic!("expected live roots")
11596        };
11597        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11598        assert_eq!(
11599            unknown_root_bindings, 1,
11600            "unknown-root arm must not read as no bindings"
11601        );
11602        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11603        assert_eq!(
11604            total_bindings,
11605            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11606        );
11607    }
11608
11609    /// A module reads HELLO_ACK as its first frame and exits on anything else,
11610    /// so the ack has to be on its outbound queue before the module is
11611    /// routable. The connection loop writes a handler's replies only after the
11612    /// handler returns; this test stops in exactly that gap, runs a real
11613    /// route.open from another connection, and only then writes whatever the
11614    /// HELLO handler returned, the way the loop would. If the ack were still a
11615    /// reply, the route.bind request would reach the module first.
11616    #[tokio::test(start_paused = true)]
11617    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11618        let forwarding = Arc::new(ForwardingTable::default());
11619        let handler =
11620            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11621        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11622        let replies = handler
11623            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11624            .await
11625            .unwrap();
11626        let queued_by_hello = module_rx.len();
11627
11628        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11629        let open_handler = handler.clone();
11630        let open = tokio::spawn(async move {
11631            open_handler
11632                .handle_control_frame(
11633                    &client_ctx,
11634                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11635                )
11636                .await
11637                .unwrap()
11638        });
11639        // Let the route.open run until its route.bind is on the module's queue.
11640        let mut spins = 0;
11641        while module_rx.len() == queued_by_hello {
11642            spins += 1;
11643            assert!(spins < 10_000, "route.open never queued a route.bind");
11644            tokio::task::yield_now().await;
11645        }
11646
11647        // Now the connection loop's half: write the HELLO handler's replies.
11648        for reply in replies {
11649            module_ctx.egress.send(reply).await.unwrap();
11650        }
11651
11652        let first = module_rx.recv().await.unwrap().frame;
11653        assert_eq!(
11654            first.header.ty,
11655            FrameType::HelloAck,
11656            "the first frame a registering module reads must be its HELLO_ACK"
11657        );
11658        assert_eq!(first.header.corr, 7);
11659        let second = module_rx.recv().await.unwrap().frame;
11660        assert_eq!(second.header.ty, FrameType::Request);
11661        assert!(
11662            matches!(
11663                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11664                ModuleControlRequest::RouteBind { .. }
11665            ),
11666            "the route.bind follows the ack"
11667        );
11668        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11669
11670        handler
11671            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11672            .await
11673            .unwrap();
11674        assert!(open.await.unwrap().is_empty());
11675        let _ = client_rx.recv().await.unwrap();
11676    }
11677
11678    #[tokio::test(start_paused = true)]
11679    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11680        let handler = ControlHandler::with_forwarding(
11681            Arc::new(Registry::default()),
11682            Arc::new(ForwardingTable::default()),
11683        );
11684        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11685        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11686        hello_via_sink(
11687            &handler,
11688            &first_ctx,
11689            &mut first_rx,
11690            hello_frame("first", PROTOCOL_VERSION, 1),
11691        )
11692        .await;
11693        hello_via_sink(
11694            &handler,
11695            &second_ctx,
11696            &mut second_rx,
11697            hello_frame("second", PROTOCOL_VERSION, 2),
11698        )
11699        .await;
11700        let root = unique_project_root("second-only");
11701        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11702        let cloned = handler.clone();
11703        let open = tokio::spawn(async move {
11704            cloned
11705                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11706                .await
11707                .unwrap()
11708        });
11709        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11710            .await
11711            .unwrap()
11712            .unwrap();
11713        let first = query_live_roots(&handler, &first_ctx).await;
11714        let second = query_live_roots(&handler, &second_ctx).await;
11715        assert!(
11716            matches!(
11717                first,
11718                ModuleControlResponseToModule::LiveRoots {
11719                    total_bindings: 0,
11720                    ..
11721                }
11722            ),
11723            "cross-module scope must not expose another module's roots"
11724        );
11725        assert!(
11726            matches!(
11727                second,
11728                ModuleControlResponseToModule::LiveRoots {
11729                    total_bindings: 1,
11730                    ..
11731                }
11732            ),
11733            "second module must see its pending route"
11734        );
11735        handler
11736            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11737            .await
11738            .unwrap();
11739        assert!(open.await.unwrap().is_empty());
11740    }
11741
11742    #[tokio::test(start_paused = true)]
11743    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11744        let handler = ControlHandler::with_forwarding(
11745            Arc::new(Registry::default()),
11746            Arc::new(ForwardingTable::default()),
11747        );
11748        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11749        hello_via_sink(
11750            &handler,
11751            &target_ctx,
11752            &mut target_rx,
11753            hello_frame("target", PROTOCOL_VERSION, 1),
11754        )
11755        .await;
11756        let actual = query_live_roots(&handler, &target_ctx).await;
11757        let ModuleControlResponseToModule::LiveRoots {
11758            roots,
11759            unknown_root_bindings,
11760            total_bindings,
11761        } = actual
11762        else {
11763            panic!("expected live roots")
11764        };
11765        assert!(roots.is_empty());
11766        assert_eq!(unknown_root_bindings, 0);
11767        assert_eq!(total_bindings, 0);
11768        assert_eq!(
11769            total_bindings,
11770            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11771        );
11772    }
11773
11774    /// Read the vendored fed corpus rather than hand-building a package.
11775    ///
11776    /// A hand-built object encodes what the test author believed the carrier
11777    /// emits. These vectors are what it actually emits, and one of them exists
11778    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11779    /// ignores additive unknown fields at the traversal emit terminus."
11780    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11781        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11782            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11783        let text = std::fs::read_to_string(&path)
11784            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11785        let vectors: Vec<(String, Value)> = text
11786            .lines()
11787            .filter(|line| !line.trim().is_empty())
11788            .map(|line| {
11789                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11790                let id = entry["corpus_id"]
11791                    .as_str()
11792                    .expect("every vector carries a corpus_id")
11793                    .to_string();
11794                (id, entry["package"].clone())
11795            })
11796            .collect();
11797        // Pin the count: a corpus that silently shrinks would take its coverage
11798        // with it, and a suite reading N-1 vectors reports the same clean pass
11799        // as one reading N.
11800        assert_eq!(
11801            vectors.len(),
11802            3,
11803            "vendored fed corpus changed size; re-sync from subc-federation"
11804        );
11805
11806        // Pin what makes the corpus DISCRIMINATING, not just present.
11807        //
11808        // The relay test below takes its expected value from the corpus, so the
11809        // corpus supplies the test's power to detect a lossy relay rather than
11810        // its correctness. A relay that dropped unrecognised fields would still
11811        // be caught -- but only by a package carrying fields it does not know.
11812        // Shrink every package to the handful of keys any implementation would
11813        // recognise and the test keeps passing over an input that can no longer
11814        // fail, which is the same clean green as a corpus that shrank away.
11815        //
11816        // So assert the precondition rather than duplicating the packages here:
11817        // at least one vector must carry a field beyond the small common set.
11818        // That is one claim to maintain instead of nine, and it fails loudly if
11819        // a re-sync ever flattens the corpus.
11820        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11821        let richest = vectors
11822            .iter()
11823            .filter_map(|(_, package)| package.as_object())
11824            .map(|object| {
11825                object
11826                    .keys()
11827                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11828                    .count()
11829            })
11830            .max()
11831            .unwrap_or(0);
11832        assert!(
11833            richest >= 2,
11834            "vendored corpus no longer carries a package with unmodelled fields, \
11835             so the relay test can no longer distinguish a verbatim relay from a lossy one"
11836        );
11837
11838        vectors
11839    }
11840
11841    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
11842    ///
11843    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
11844    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
11845    /// three-key object, so a relay that quietly dropped fields it did not
11846    /// recognise would satisfy it. These vectors carry nine keys including ones
11847    /// this crate has no type for, so a typed relay fails here and only here.
11848    #[tokio::test]
11849    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11850        for (corpus_id, package) in fed_admission_facts_vectors() {
11851            let registry = Arc::new(Registry::default());
11852            let forwarding = Arc::new(ForwardingTable::default());
11853            let supervisor = SupervisorHandle::new();
11854            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11855            let handler =
11856                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11857                    .with_supervisor(supervisor)
11858                    .with_admission_facts_config(
11859                        Some("fed".to_string()),
11860                        Some(vec!["target".to_string()]),
11861                    );
11862
11863            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11864            hello_via_sink(
11865                &handler,
11866                &target_ctx,
11867                &mut target_rx,
11868                hello_frame("target", PROTOCOL_VERSION, 1),
11869            )
11870            .await;
11871
11872            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11873            let route_handler = handler.clone();
11874            let expected = package.clone();
11875            let route_task = tokio::spawn(async move {
11876                route_handler
11877                    .handle_control_frame(
11878                        &client_ctx,
11879                        route_open_frame_with_admission_facts(
11880                            20,
11881                            "target",
11882                            unique_project_root("admission-facts"),
11883                            Some(subc_control::ConsumerIdentity {
11884                                module_id: "fed".to_string(),
11885                                launch_nonce: "fed-nonce".to_string(),
11886                            }),
11887                            Some(package),
11888                        ),
11889                    )
11890                    .await
11891                    .unwrap()
11892            });
11893
11894            let bind_frame = target_rx.recv().await.unwrap();
11895            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11896            let ModuleControlRequest::RouteBind {
11897                admission_facts, ..
11898            } = bind
11899            else {
11900                panic!("{corpus_id}: expected route.bind")
11901            };
11902            assert_eq!(
11903                admission_facts,
11904                Some(expected),
11905                "{corpus_id}: relay must not add, drop or reshape any field"
11906            );
11907
11908            handler
11909                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11910                .await
11911                .unwrap();
11912            route_task.await.unwrap();
11913        }
11914    }
11915
11916    #[tokio::test]
11917    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11918        let registry = Arc::new(Registry::default());
11919        let forwarding = Arc::new(ForwardingTable::default());
11920        let supervisor = SupervisorHandle::new();
11921        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11922        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11923        let handler =
11924            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11925                .with_supervisor(supervisor)
11926                .with_admission_facts_config(
11927                    Some("fed".to_string()),
11928                    Some(vec!["target".to_string()]),
11929                );
11930
11931        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11932        hello_via_sink(
11933            &handler,
11934            &target_ctx,
11935            &mut target_rx,
11936            hello_frame("target", PROTOCOL_VERSION, 1),
11937        )
11938        .await;
11939        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11940        hello_via_sink(
11941            &handler,
11942            &other_ctx,
11943            &mut other_rx,
11944            hello_frame("other", PROTOCOL_VERSION, 2),
11945        )
11946        .await;
11947
11948        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11949        let expected_facts = facts.clone();
11950        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11951        let route_handler = handler.clone();
11952        let route_task = tokio::spawn(async move {
11953            route_handler
11954                .handle_control_frame(
11955                    &client_ctx,
11956                    route_open_frame_with_admission_facts(
11957                        10,
11958                        "target",
11959                        unique_project_root("admission-facts"),
11960                        Some(subc_control::ConsumerIdentity {
11961                            module_id: "fed".to_string(),
11962                            launch_nonce: "fed-nonce".to_string(),
11963                        }),
11964                        Some(facts.clone()),
11965                    ),
11966                )
11967                .await
11968                .unwrap()
11969        });
11970        let bind_frame = target_rx.recv().await.unwrap();
11971        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11972        let ModuleControlRequest::RouteBind {
11973            admission_facts, ..
11974        } = bind
11975        else {
11976            panic!("expected route.bind")
11977        };
11978        assert_eq!(admission_facts, Some(expected_facts));
11979        handler
11980            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11981            .await
11982            .unwrap();
11983        assert!(route_task.await.unwrap().is_empty());
11984        assert!(matches!(
11985            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11986                .unwrap(),
11987            ClientControlResponse::RouteOpen { .. }
11988        ));
11989
11990        let direct = handler
11991            .handle_control_frame(
11992                &route_ctx(ConnectionId::new(73)).0,
11993                route_open_frame_with_admission_facts(
11994                    11,
11995                    "target",
11996                    unique_project_root("admission-facts"),
11997                    None,
11998                    Some(json!({"x": 1})),
11999                ),
12000            )
12001            .await
12002            .unwrap();
12003        assert_eq!(
12004            parse_error(&direct[0])["code"],
12005            "admission_facts_not_permitted"
12006        );
12007
12008        let different_reserved = handler
12009            .handle_control_frame(
12010                &route_ctx(ConnectionId::new(77)).0,
12011                route_open_frame_with_admission_facts(
12012                    15,
12013                    "target",
12014                    unique_project_root("admission-facts"),
12015                    Some(subc_control::ConsumerIdentity {
12016                        module_id: "other".to_string(),
12017                        launch_nonce: "other-nonce".to_string(),
12018                    }),
12019                    Some(json!({"x": 1})),
12020                ),
12021            )
12022            .await
12023            .unwrap();
12024        assert_eq!(
12025            parse_error(&different_reserved[0])["code"],
12026            "admission_facts_not_permitted"
12027        );
12028
12029        let other_target = handler
12030            .handle_control_frame(
12031                &route_ctx(ConnectionId::new(74)).0,
12032                route_open_frame_with_admission_facts(
12033                    12,
12034                    "other",
12035                    unique_project_root("admission-facts"),
12036                    Some(subc_control::ConsumerIdentity {
12037                        module_id: "fed".to_string(),
12038                        launch_nonce: "fed-nonce".to_string(),
12039                    }),
12040                    Some(json!({"x": 1})),
12041                ),
12042            )
12043            .await
12044            .unwrap();
12045        assert_eq!(
12046            parse_error(&other_target[0])["code"],
12047            "admission_facts_target_not_allowed"
12048        );
12049
12050        let nonexistent = handler
12051            .handle_control_frame(
12052                &route_ctx(ConnectionId::new(75)).0,
12053                route_open_frame_with_admission_facts(
12054                    13,
12055                    "missing",
12056                    unique_project_root("admission-facts"),
12057                    None,
12058                    Some(json!({"x": 1})),
12059                ),
12060            )
12061            .await
12062            .unwrap();
12063        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12064
12065        let described = handler
12066            .handle_control_frame(
12067                &route_ctx(ConnectionId::new(76)).0,
12068                Frame::build(
12069                    FrameType::Request,
12070                    control_flags(),
12071                    0,
12072                    0,
12073                    14,
12074                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12075                )
12076                .unwrap(),
12077            )
12078            .await
12079            .unwrap();
12080        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12081            serde_json::from_slice(&described[0].body).unwrap()
12082        else {
12083            panic!("expected server.describe response")
12084        };
12085        assert!(capabilities
12086            .iter()
12087            .any(|cap| cap == "admission_facts_relay_v1"));
12088    }
12089
12090    #[tokio::test]
12091    async fn admission_facts_without_configured_carrier_are_rejected() {
12092        let registry = Arc::new(Registry::default());
12093        let forwarding = Arc::new(ForwardingTable::default());
12094        let handler = ControlHandler::with_forwarding(registry, forwarding);
12095        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12096        hello_via_sink(
12097            &handler,
12098            &target_ctx,
12099            &mut target_rx,
12100            hello_frame("target", PROTOCOL_VERSION, 1),
12101        )
12102        .await;
12103
12104        let responses = handler
12105            .handle_control_frame(
12106                &route_ctx(ConnectionId::new(79)).0,
12107                route_open_frame_with_admission_facts(
12108                    16,
12109                    "target",
12110                    unique_project_root("admission-facts"),
12111                    None,
12112                    Some(json!({"x": 1})),
12113                ),
12114            )
12115            .await
12116            .unwrap();
12117        assert_eq!(
12118            parse_error(&responses[0])["code"],
12119            "admission_facts_not_permitted"
12120        );
12121    }
12122
12123    #[tokio::test]
12124    async fn route_open_relays_consumer_capabilities_verbatim() {
12125        let registry = Arc::new(Registry::default());
12126        let forwarding = Arc::new(ForwardingTable::default());
12127        let handler =
12128            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12129        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12130        hello_via_sink(
12131            &handler,
12132            &module_ctx,
12133            &mut module_rx,
12134            hello_frame("aft", PROTOCOL_VERSION, 7),
12135        )
12136        .await;
12137
12138        let expected = vec!["elicitation".to_string(), "roots".to_string()];
12139        let expected_for_request = expected.clone();
12140        let project_root = unique_project_root("consumer-capabilities-present");
12141        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12142        let route_handler = handler.clone();
12143        let route_task = tokio::spawn(async move {
12144            route_handler
12145                .handle_control_frame(
12146                    &client_ctx,
12147                    route_open_frame_with_consumer_capabilities(
12148                        401,
12149                        "aft",
12150                        project_root,
12151                        Some(expected_for_request),
12152                    ),
12153                )
12154                .await
12155                .unwrap()
12156        });
12157        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12158            .await
12159            .unwrap()
12160            .unwrap();
12161        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12162        let ModuleControlRequest::RouteBind {
12163            consumer_capabilities,
12164            ..
12165        } = bind
12166        else {
12167            panic!("expected route.bind request, got {bind:?}");
12168        };
12169        assert_eq!(consumer_capabilities, Some(expected.clone()));
12170
12171        handler
12172            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12173            .await
12174            .unwrap();
12175        let route_response = route_task.await.unwrap();
12176        assert!(route_response.is_empty());
12177        let published = client_rx.recv().await.unwrap();
12178        assert!(matches!(
12179            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12180            ClientControlResponse::RouteOpen { .. }
12181        ));
12182    }
12183
12184    #[tokio::test]
12185    async fn route_open_without_consumer_capabilities_relays_none() {
12186        let registry = Arc::new(Registry::default());
12187        let forwarding = Arc::new(ForwardingTable::default());
12188        let handler =
12189            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12190        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12191        hello_via_sink(
12192            &handler,
12193            &module_ctx,
12194            &mut module_rx,
12195            hello_frame("aft", PROTOCOL_VERSION, 7),
12196        )
12197        .await;
12198
12199        let project_root = unique_project_root("consumer-capabilities-absent");
12200        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12201        let route_handler = handler.clone();
12202        let route_task = tokio::spawn(async move {
12203            route_handler
12204                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12205                .await
12206                .unwrap()
12207        });
12208        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12209            .await
12210            .unwrap()
12211            .unwrap();
12212        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12213        let ModuleControlRequest::RouteBind {
12214            consumer_capabilities,
12215            ..
12216        } = bind
12217        else {
12218            panic!("expected route.bind request, got {bind:?}");
12219        };
12220        assert_eq!(consumer_capabilities, None);
12221
12222        handler
12223            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12224            .await
12225            .unwrap();
12226        let route_response = route_task.await.unwrap();
12227        assert!(route_response.is_empty());
12228        let published = client_rx.recv().await.unwrap();
12229        assert!(matches!(
12230            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12231            ClientControlResponse::RouteOpen { .. }
12232        ));
12233    }
12234
12235    /// Opens a route to a freshly registered `aft` with `sent` as the
12236    /// route.open's role_versions, acks the bind, and returns the role_versions
12237    /// the module's bind carried.
12238    async fn bind_role_versions_for(
12239        sent: Option<BTreeMap<String, String>>,
12240        connection: u64,
12241    ) -> Option<BTreeMap<String, String>> {
12242        let registry = Arc::new(Registry::default());
12243        let forwarding = Arc::new(ForwardingTable::default());
12244        let handler =
12245            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12246        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12247        hello_via_sink(
12248            &handler,
12249            &module_ctx,
12250            &mut module_rx,
12251            hello_frame("aft", PROTOCOL_VERSION, 7),
12252        )
12253        .await;
12254        let project_root = unique_project_root("role-versions");
12255        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12256        let route_handler = handler.clone();
12257        let route_task = tokio::spawn(async move {
12258            route_handler
12259                .handle_control_frame(
12260                    &client_ctx,
12261                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
12262                )
12263                .await
12264                .unwrap()
12265        });
12266        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12267            .await
12268            .expect("a well-formed route.open reaches the module as a bind")
12269            .unwrap();
12270        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12271        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12272            panic!("expected route.bind request, got {bind:?}");
12273        };
12274        handler
12275            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12276            .await
12277            .unwrap();
12278        assert!(route_task.await.unwrap().is_empty());
12279        let published = client_rx.recv().await.unwrap();
12280        assert!(matches!(
12281            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12282            ClientControlResponse::RouteOpen { .. }
12283        ));
12284        role_versions
12285    }
12286
12287    #[tokio::test]
12288    async fn route_open_relays_role_versions_verbatim() {
12289        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12290        assert_eq!(
12291            bind_role_versions_for(Some(sent.clone()), 141).await,
12292            Some(sent)
12293        );
12294    }
12295
12296    /// An empty map declares nothing, so the provider sees no field rather
12297    /// than an empty object it would have to treat as a second "none".
12298    #[tokio::test]
12299    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12300        assert_eq!(bind_role_versions_for(None, 143).await, None);
12301        assert_eq!(
12302            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12303            None
12304        );
12305    }
12306
12307    #[tokio::test]
12308    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12309        let registry = Arc::new(Registry::default());
12310        let forwarding = Arc::new(ForwardingTable::default());
12311        let handler =
12312            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12313        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12314        hello_via_sink(
12315            &handler,
12316            &module_ctx,
12317            &mut module_rx,
12318            hello_frame("aft", PROTOCOL_VERSION, 7),
12319        )
12320        .await;
12321
12322        let nine: BTreeMap<String, String> = (0..9)
12323            .map(|index| (format!("role-{index}"), "v1".to_string()))
12324            .collect();
12325        for (label, malformed) in [
12326            (
12327                "invalid role name",
12328                role_versions(&[("Tool_Provider", "v1")]),
12329            ),
12330            ("invalid version", role_versions(&[("tool-provider", "v0")])),
12331            ("nine entries", nine),
12332        ] {
12333            // A refused open answers at once; one that reached the module
12334            // would wait for its bind ack and trip this timeout.
12335            let responses = tokio::time::timeout(
12336                Duration::from_secs(1),
12337                handler.handle_control_frame(
12338                    &route_ctx(ConnectionId::new(148)).0,
12339                    route_open_frame_with_role_versions(
12340                        404,
12341                        "aft",
12342                        unique_project_root("role-versions-malformed"),
12343                        Some(malformed),
12344                    ),
12345                ),
12346            )
12347            .await
12348            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12349            .unwrap();
12350            assert_eq!(responses.len(), 1, "{label}");
12351            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12352            let error = parse_error(&responses[0]);
12353            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12354            assert_eq!(
12355                error["detail"]["field"], "role_versions",
12356                "{label}: {error}"
12357            );
12358            assert!(
12359                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12360                "{label}: a malformed declaration is terminal"
12361            );
12362            assert!(
12363                module_rx.try_recv().is_err(),
12364                "{label}: the module must never see a bind"
12365            );
12366        }
12367    }
12368
12369    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
12370    /// consumer can tell this daemon forwards the field from one that would
12371    /// drop it.
12372    #[tokio::test]
12373    async fn route_role_versions_capability_is_advertised() {
12374        let handler = ControlHandler::new(Arc::new(Registry::default()));
12375        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12376        let ack = hello_via_sink(
12377            &handler,
12378            &ctx,
12379            &mut rx,
12380            hello_frame("m", PROTOCOL_VERSION, 1),
12381        )
12382        .await;
12383        let ack = parse_ack(&ack);
12384        assert!(
12385            ack.subc_capabilities
12386                .iter()
12387                .any(|c| c == "route-role-versions/v1"),
12388            "{:?}",
12389            ack.subc_capabilities
12390        );
12391
12392        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12393        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12394        let reply = handler
12395            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12396            .await
12397            .unwrap()
12398            .pop()
12399            .unwrap();
12400        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12401            serde_json::from_slice(&reply.body).unwrap()
12402        else {
12403            panic!("not a server.describe reply");
12404        };
12405        assert!(
12406            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12407            "{capabilities:?}"
12408        );
12409    }
12410
12411    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12412    /// catalog entry without `events` (the module declares none) from a daemon
12413    /// that never reports declared events.
12414    #[tokio::test]
12415    async fn catalog_events_capability_is_advertised() {
12416        let handler = ControlHandler::new(Arc::new(Registry::default()));
12417        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12418        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12419        let reply = handler
12420            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12421            .await
12422            .unwrap()
12423            .pop()
12424            .unwrap();
12425        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12426            serde_json::from_slice(&reply.body).unwrap()
12427        else {
12428            panic!("not a server.describe reply");
12429        };
12430        assert!(
12431            capabilities
12432                .iter()
12433                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12434            "{capabilities:?}"
12435        );
12436    }
12437
12438    #[tokio::test]
12439    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12440        let handler = ControlHandler::new(Arc::new(Registry::default()));
12441        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12442        let ack = hello_via_sink(
12443            &handler,
12444            &module,
12445            &mut module_rx,
12446            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12447        )
12448        .await;
12449        let ack = parse_ack(&ack);
12450        assert!(
12451            ack.subc_capabilities
12452                .iter()
12453                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12454            "{:?}",
12455            ack.subc_capabilities
12456        );
12457        assert!(ack
12458            .subc_capabilities
12459            .iter()
12460            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12461
12462        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12463        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12464        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12465        let reply = handler
12466            .handle_control_frame(&client, frame)
12467            .await
12468            .unwrap()
12469            .pop()
12470            .unwrap();
12471        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12472            serde_json::from_slice(&reply.body).unwrap()
12473        else {
12474            panic!("not a server.describe reply");
12475        };
12476        assert!(
12477            capabilities
12478                .iter()
12479                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12480            "{capabilities:?}"
12481        );
12482        assert!(capabilities
12483            .iter()
12484            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12485    }
12486
12487    #[tokio::test]
12488    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12489        let handler = ControlHandler::new(Arc::new(Registry::default()));
12490        let (module, mut rx) = route_ctx(ConnectionId::new(156));
12491        let ack = hello_via_sink(
12492            &handler,
12493            &module,
12494            &mut rx,
12495            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12496        )
12497        .await;
12498        assert!(parse_ack(&ack)
12499            .subc_capabilities
12500            .iter()
12501            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12502        assert!(parse_ack(&ack)
12503            .subc_capabilities
12504            .iter()
12505            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12506        let (client, _rx) = route_ctx(ConnectionId::new(157));
12507        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12508        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12509        let reply = handler
12510            .handle_control_frame(&client, frame)
12511            .await
12512            .unwrap()
12513            .pop()
12514            .unwrap();
12515        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12516            serde_json::from_slice(&reply.body).unwrap()
12517        else {
12518            panic!("server.describe reply")
12519        };
12520        assert!(capabilities
12521            .iter()
12522            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12523        assert!(capabilities
12524            .iter()
12525            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12526    }
12527
12528    #[tokio::test]
12529    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12530        let handler = ControlHandler::new(Arc::new(Registry::default()));
12531        let (module, mut rx) = route_ctx(ConnectionId::new(158));
12532        let ack = hello_via_sink(
12533            &handler,
12534            &module,
12535            &mut rx,
12536            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12537        )
12538        .await;
12539        assert!(parse_ack(&ack)
12540            .subc_capabilities
12541            .iter()
12542            .any(|cap| cap == "scope-workdir/v1"));
12543        let (client, _rx) = route_ctx(ConnectionId::new(159));
12544        let frame = Frame::build(
12545            FrameType::Request,
12546            control_flags(),
12547            0,
12548            0,
12549            5,
12550            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12551        )
12552        .unwrap();
12553        let reply = handler
12554            .handle_control_frame(&client, frame)
12555            .await
12556            .unwrap()
12557            .remove(0);
12558        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12559            serde_json::from_slice(&reply.body).unwrap()
12560        else {
12561            panic!("server.describe reply")
12562        };
12563        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12564    }
12565
12566    #[tokio::test]
12567    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
12568        let handler = ControlHandler::new(Arc::new(Registry::default()));
12569        let (module, mut rx) = route_ctx(ConnectionId::new(160));
12570        let ack = hello_via_sink(
12571            &handler,
12572            &module,
12573            &mut rx,
12574            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
12575        )
12576        .await;
12577        assert!(parse_ack(&ack)
12578            .subc_capabilities
12579            .iter()
12580            .any(|cap| cap == "worker-runs/v1"));
12581        let (client, _rx) = route_ctx(ConnectionId::new(161));
12582        let frame = Frame::build(
12583            FrameType::Request,
12584            control_flags(),
12585            0,
12586            0,
12587            5,
12588            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12589        )
12590        .unwrap();
12591        let reply = handler
12592            .handle_control_frame(&client, frame)
12593            .await
12594            .unwrap()
12595            .remove(0);
12596        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12597            serde_json::from_slice(&reply.body).unwrap()
12598        else {
12599            panic!("server.describe reply")
12600        };
12601        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
12602    }
12603
12604    #[tokio::test]
12605    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12606        let handler = ControlHandler::new(Arc::new(Registry::default()));
12607        let (module, mut rx) = route_ctx(ConnectionId::new(154));
12608        let ack = hello_via_sink(
12609            &handler,
12610            &module,
12611            &mut rx,
12612            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12613        )
12614        .await;
12615        assert!(parse_ack(&ack)
12616            .subc_capabilities
12617            .iter()
12618            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12619        let (client, _rx) = route_ctx(ConnectionId::new(155));
12620        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12621        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12622        let reply = handler
12623            .handle_control_frame(&client, frame)
12624            .await
12625            .unwrap()
12626            .pop()
12627            .unwrap();
12628        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12629            serde_json::from_slice(&reply.body).unwrap()
12630        else {
12631            panic!("not a server.describe reply")
12632        };
12633        assert!(capabilities
12634            .iter()
12635            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12636    }
12637
12638    #[tokio::test]
12639    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12640        let registry = Arc::new(Registry::default());
12641        let forwarding = Arc::new(ForwardingTable::default());
12642        let handler =
12643            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12644                .with_health_probe_timeout(Duration::from_secs(5));
12645        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12646        hello_via_sink(
12647            &handler,
12648            &module_ctx,
12649            &mut module_rx,
12650            non_routable_hello_frame_with_control_ops(
12651                "mcp",
12652                300,
12653                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12654            ),
12655        )
12656        .await;
12657        assert!(registry
12658            .get_module("mcp")
12659            .unwrap()
12660            .unwrap()
12661            .manifest
12662            .provides
12663            .is_empty());
12664
12665        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12666        let route_response = handler
12667            .handle_control_frame(
12668                &route_client_ctx,
12669                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12670            )
12671            .await
12672            .unwrap();
12673        assert_eq!(route_response[0].header.ty, FrameType::Error);
12674        assert_eq!(
12675            parse_error(&route_response[0])["code"],
12676            "target_unavailable"
12677        );
12678        assert!(parse_error(&route_response[0])["message"]
12679            .as_str()
12680            .unwrap()
12681            .contains("does not provide the requested target"));
12682        assert!(module_rx.try_recv().is_err());
12683
12684        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12685        let health_handler = handler.clone();
12686        let health_task = tokio::spawn(async move {
12687            health_handler
12688                .handle_control_frame(
12689                    &health_client_ctx,
12690                    supervisor_health_probe_frame(302, "mcp"),
12691                )
12692                .await
12693                .unwrap()
12694        });
12695        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12696            .await
12697            .unwrap()
12698            .unwrap();
12699        assert_eq!(
12700            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12701            ModuleControlRequest::HealthCheck {}
12702        );
12703        handler
12704            .handle_control_frame(
12705                &module_ctx,
12706                health_response(health_frame.header.corr, HealthStatus::Ok),
12707            )
12708            .await
12709            .unwrap();
12710        let health_response = health_task.await.unwrap();
12711        assert_eq!(health_response[0].header.ty, FrameType::Response);
12712        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12713            ClientControlResponse::SupervisorHealthProbe {
12714                module_id, status, ..
12715            } => {
12716                assert_eq!(module_id, "mcp");
12717                assert_eq!(status, HealthStatus::Ok);
12718            }
12719            other => panic!("unexpected health response: {other:?}"),
12720        }
12721
12722        // Exercise the forwarding cleanup path directly while leaving the registry
12723        // advertisement in place. If cleanup leaves a stale control sink behind,
12724        // the next probe will enqueue onto it and wait for the long probe timeout
12725        // instead of returning an immediate no-connection error.
12726        forwarding
12727            .cleanup_connection(module_ctx.connection_id)
12728            .unwrap();
12729        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12730        let cleanup_response = tokio::time::timeout(
12731            Duration::from_millis(200),
12732            handler.handle_control_frame(
12733                &cleanup_probe_ctx,
12734                supervisor_health_probe_frame(303, "mcp"),
12735            ),
12736        )
12737        .await
12738        .expect("probe should fail immediately when the control lane is gone")
12739        .unwrap();
12740        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12741        assert_eq!(
12742            parse_error(&cleanup_response[0])["code"],
12743            "target_unavailable"
12744        );
12745        assert!(parse_error(&cleanup_response[0])["message"]
12746            .as_str()
12747            .unwrap()
12748            .contains("no module connection"));
12749
12750        handler
12751            .cleanup_connection(module_ctx.connection_id)
12752            .unwrap();
12753    }
12754
12755    #[tokio::test]
12756    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12757        let registry = Arc::new(Registry::default());
12758        let supervisor_handle = SupervisorHandle::new();
12759        let supervisor =
12760            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12761                .with_handle(supervisor_handle.clone())
12762                .with_connection_file_path(
12763                    std::env::temp_dir()
12764                        .join(format!("subc-route-open-warming-{}", std::process::id())),
12765                );
12766        let module = supervisor
12767            .supervise_configured(
12768                ModuleSpec {
12769                    module_id: "warming".to_string(),
12770                    program: fake_aft_stub_path(),
12771                    args: Vec::new(),
12772                    env: Vec::new(),
12773                    reserved: false,
12774                    reserved_prefixes: Vec::new(),
12775                    protocol: ModuleProtocol::Subc,
12776                    overlap: Default::default(),
12777                },
12778                true,
12779            )
12780            .unwrap();
12781        assert_eq!(module.state().unwrap(), ModuleState::Running);
12782
12783        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12784        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12785        let response = handler
12786            .handle_control_frame(
12787                &ctx,
12788                route_open_frame(304, "warming", unique_project_root("warming")),
12789            )
12790            .await
12791            .unwrap();
12792        module.stop().await.unwrap();
12793
12794        assert_eq!(response[0].header.ty, FrameType::Error);
12795        let error = parse_error(&response[0]);
12796        assert_eq!(error["code"], "module_warming");
12797        assert!(error["message"]
12798            .as_str()
12799            .unwrap()
12800            .contains("state=running, enabled=true, live=false"));
12801    }
12802
12803    #[test]
12804    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12805        let handler = ControlHandler::new(Arc::new(Registry::default()));
12806        let capture = EventCapture::default();
12807        let _subscriber =
12808            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12809        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12810        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12811        let pending = (0..limit).collect::<Vec<_>>();
12812        let response = handler
12813            .route_open_capacity_refusal(
12814                &ctx,
12815                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12816                "busy",
12817                pending.len(),
12818                limit,
12819            )
12820            .unwrap();
12821        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12822        let event = capture
12823            .events()
12824            .into_iter()
12825            .find(|event| {
12826                event.target == "control"
12827                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12828            })
12829            .expect("connection admission refusal event");
12830        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12831        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12832        // Refused before `handle_route_open` parses the request, so the caller
12833        // fields are read from the frame itself.
12834        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
12835        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
12836    }
12837
12838    #[test]
12839    fn route_open_target_cap_logs_admission_reason_and_capacity() {
12840        let handler = ControlHandler::new(Arc::new(Registry::default()));
12841        let capture = EventCapture::default();
12842        let _subscriber =
12843            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12844        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12845        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12846        let guards = (0..limit)
12847            .map(|_| {
12848                handler
12849                    .route_bind_concurrency
12850                    .try_admit("busy", limit)
12851                    .unwrap()
12852            })
12853            .collect::<Vec<_>>();
12854        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12855            Err(in_flight) => in_flight,
12856            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12857        };
12858        let response = handler
12859            .route_open_target_capacity_refusal(
12860                &ctx,
12861                &RouteOpenCaller::unknown(),
12862                &route_open_frame(397, "busy", unique_project_root("target-cap")),
12863                "busy",
12864                in_flight,
12865            )
12866            .unwrap();
12867        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12868        let event = capture
12869            .events()
12870            .into_iter()
12871            .find(|event| {
12872                event.target == "control"
12873                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12874            })
12875            .expect("target admission refusal event");
12876        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12877        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12878        drop(guards);
12879    }
12880
12881    #[cfg(unix)]
12882    #[tokio::test(start_paused = true)]
12883    async fn supervisor_list_shows_a_failed_modules_next_retry() {
12884        let registry = Arc::new(Registry::default());
12885        let handle = SupervisorHandle::new();
12886        let supervisor =
12887            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
12888                .with_handle(handle.clone());
12889        let module = supervisor
12890            .spawn(crate::ModuleSpec {
12891                module_id: "retry-list".into(),
12892                program: PathBuf::from("/bin/sh"),
12893                args: vec!["-c".into(), "exit 17".into()],
12894                env: vec![],
12895                reserved: false,
12896                reserved_prefixes: vec![],
12897                protocol: ModuleProtocol::None,
12898                overlap: Default::default(),
12899            })
12900            .unwrap();
12901        let deadline = std::time::Instant::now() + Duration::from_secs(10);
12902        while module.state().unwrap() != crate::ModuleState::Failed {
12903            assert!(
12904                std::time::Instant::now() < deadline,
12905                "waiting for budget failure"
12906            );
12907            tokio::task::yield_now().await;
12908        }
12909        let handler = ControlHandler::new(registry).with_supervisor(handle);
12910        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12911        let replies = handler.handle_supervisor_list(frame).await.unwrap();
12912        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12913        assert_eq!(body["modules"][0]["state"], "failed");
12914        let retry = body["modules"][0]["next_retry_at_ms"]
12915            .as_u64()
12916            .expect("scheduled retry on the wire");
12917        let now = std::time::SystemTime::now()
12918            .duration_since(std::time::UNIX_EPOCH)
12919            .unwrap()
12920            .as_millis() as u64;
12921        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
12922        module.set_enabled(false).await.unwrap();
12923        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12924        let replies = handler.handle_supervisor_list(frame).await.unwrap();
12925        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12926        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
12927    }
12928
12929    #[tokio::test]
12930    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12931        let registry = Arc::new(Registry::default());
12932        let handle = SupervisorHandle::new();
12933        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12934            .with_handle(handle.clone());
12935        supervisor
12936            .supervise_configured(
12937                crate::ModuleSpec {
12938                    module_id: "drift".into(),
12939                    program: PathBuf::from("disabled-module"),
12940                    args: Vec::new(),
12941                    env: Vec::new(),
12942                    reserved: false,
12943                    reserved_prefixes: Vec::new(),
12944                    protocol: ModuleProtocol::Subc,
12945                    overlap: Default::default(),
12946                },
12947                false,
12948            )
12949            .unwrap();
12950        let handler = ControlHandler::new(registry).with_supervisor(handle);
12951        for configured in [None, Some(false), Some(true)] {
12952            handler.capability_evaluator.configure(
12953                configured.map(|enabled| ("drift".to_string(), enabled)),
12954                BTreeMap::new(),
12955            );
12956            let frame =
12957                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12958            let replies = handler
12959                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12960                .await
12961                .unwrap();
12962            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12963            assert!(body.get("configured_enabled").is_none());
12964            assert_eq!(body["op"], "supervisor.ack");
12965            let frame =
12966                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12967            let replies = handler.handle_supervisor_list(frame).await.unwrap();
12968            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12969            assert_eq!(
12970                body["modules"][0].get("configured_enabled"),
12971                configured.map(Value::Bool).as_ref()
12972            );
12973        }
12974    }
12975
12976    #[tokio::test(flavor = "current_thread")]
12977    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12978        let handler = ControlHandler::new(Arc::new(Registry::default()));
12979        let capture = EventCapture::default();
12980        let _subscriber =
12981            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12982        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12983        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12984
12985        let _ = handler
12986            .handle_client_control_request(
12987                &ctx,
12988                frame,
12989                ClientControlRequest::SupervisorSetEnabled {
12990                    module_id: "broca".to_string(),
12991                    enabled: false,
12992                },
12993                &mut None,
12994            )
12995            .await
12996            .unwrap();
12997
12998        let events = capture
12999            .events()
13000            .into_iter()
13001            .filter(|event| {
13002                event.target == "control"
13003                    && event.fields.get("message").map(String::as_str)
13004                        == Some("supervisor request received")
13005            })
13006            .collect::<Vec<_>>();
13007        assert_eq!(events.len(), 1, "one supervisor request log line");
13008        let event = &events[0];
13009        assert_eq!(event.level, tracing::Level::INFO);
13010        assert_eq!(
13011            event.fields.get("op"),
13012            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13013        );
13014        assert_eq!(
13015            event.fields.get("module_id"),
13016            Some(&"\"broca\"".to_string())
13017        );
13018        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13019        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13020        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13021    }
13022
13023    #[tokio::test(flavor = "current_thread")]
13024    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13025        let registry = Arc::new(Registry::default());
13026        let module_connection = ConnectionId::new(303);
13027        registry
13028            .register_with_control_ops(
13029                manifest("aft", PROTOCOL_VERSION),
13030                PROTOCOL_VERSION,
13031                module_connection,
13032                module_baseline_control_ops(),
13033            )
13034            .unwrap();
13035        let handler = ControlHandler::new(registry);
13036        let capture = EventCapture::default();
13037        let _subscriber =
13038            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13039        let (ctx, _rx) = route_ctx(module_connection);
13040        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13041
13042        let _ = handler
13043            .handle_client_control_request(
13044                &ctx,
13045                frame,
13046                ClientControlRequest::SupervisorSetEnabled {
13047                    module_id: "broca".to_string(),
13048                    enabled: false,
13049                },
13050                &mut None,
13051            )
13052            .await
13053            .unwrap();
13054
13055        let callers = capture
13056            .events()
13057            .into_iter()
13058            .filter(|event| {
13059                event.fields.get("message").map(String::as_str)
13060                    == Some("supervisor request received")
13061            })
13062            .map(|event| event.fields.get("caller").cloned())
13063            .collect::<Vec<_>>();
13064        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13065    }
13066
13067    #[tokio::test(flavor = "current_thread")]
13068    async fn supervisor_list_does_not_log_request_received() {
13069        let handler = ControlHandler::new(Arc::new(Registry::default()));
13070        let capture = EventCapture::default();
13071        let _subscriber =
13072            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13073        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13074        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13075
13076        handler
13077            .handle_client_control_request(
13078                &ctx,
13079                frame,
13080                ClientControlRequest::SupervisorList {},
13081                &mut None,
13082            )
13083            .await
13084            .unwrap();
13085
13086        assert!(capture.events().into_iter().all(|event| {
13087            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13088        }));
13089    }
13090
13091    #[tokio::test(flavor = "current_thread")]
13092    async fn supervisor_rescan_preview_does_not_log_request_received() {
13093        let handler = ControlHandler::new(Arc::new(Registry::default()));
13094        let capture = EventCapture::default();
13095        let _subscriber =
13096            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13097        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13098        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13099
13100        handler
13101            .handle_client_control_request(
13102                &ctx,
13103                frame,
13104                ClientControlRequest::SupervisorRescan { preview: true },
13105                &mut None,
13106            )
13107            .await
13108            .unwrap();
13109
13110        assert!(capture.events().into_iter().all(|event| {
13111            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13112        }));
13113    }
13114
13115    struct ApproveOperator;
13116    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13117        fn prompt(
13118            &self,
13119            _: &str,
13120            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13121        ) -> crate::operator_confirm::ProviderResult {
13122            crate::operator_confirm::ProviderResult::Approved
13123        }
13124    }
13125
13126    #[tokio::test]
13127    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13128        assert_eq!(
13129            ControlHandler::default().privileged_operator_prompt,
13130            PrivilegedOperatorPrompt::Log
13131        );
13132        for (initial, changed, running) in [
13133            ("log", "enforce", PrivilegedOperatorPrompt::Log),
13134            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13135        ] {
13136            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13137            let path = dir.join("subc.jsonc");
13138            std::fs::write(
13139                &path,
13140                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13141            )
13142            .unwrap();
13143            let config = crate::daemon_config::load(&path).unwrap().unwrap();
13144            let registry = Arc::new(Registry::default());
13145            let handle = SupervisorHandle::new();
13146            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13147                .with_handle(handle.clone());
13148            let handler = ControlHandler::new(registry)
13149                .with_supervisor(handle)
13150                .with_privileged_operator_prompt(config.privileged_operator_prompt)
13151                .with_supervisor_rescan(supervisor, &path, config.port);
13152            handler.forwarding.operator_confirms().configure(
13153                Arc::new(ApproveOperator),
13154                crate::operator_confirm::OperatorLimits::default(),
13155            );
13156            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13157
13158            for (disk_mode, needs_restart) in [
13159                (Some(initial), false),
13160                (Some(changed), true),
13161                (Some(changed), true),
13162                (None, running == PrivilegedOperatorPrompt::Enforce),
13163                (Some(initial), false),
13164            ] {
13165                let mut doc = json!({"version":1});
13166                if let Some(mode) = disk_mode {
13167                    doc["privileged_operator_prompt"] = json!(mode);
13168                }
13169                std::fs::write(&path, doc.to_string()).unwrap();
13170                for preview in [true, false] {
13171                    let request = ClientControlRequest::SupervisorRescan { preview };
13172                    let frame = Frame::build(
13173                        FrameType::Request,
13174                        control_flags(),
13175                        0,
13176                        0,
13177                        1,
13178                        serde_json::to_vec(&request).unwrap(),
13179                    )
13180                    .unwrap();
13181                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13182                    assert_eq!(frames.len(), 1);
13183                    let ClientControlResponse::SupervisorRescan { result } =
13184                        serde_json::from_slice(&frames[0].body).unwrap()
13185                    else {
13186                        panic!("expected a rescan response");
13187                    };
13188                    let expected = if needs_restart {
13189                        vec!["privileged_operator_prompt"]
13190                    } else {
13191                        Vec::new()
13192                    };
13193                    assert_eq!(result.restart_required, expected);
13194                    assert_eq!(handler.privileged_operator_prompt, running);
13195                    assert_eq!(
13196                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13197                        running
13198                    );
13199                }
13200            }
13201        }
13202    }
13203
13204    #[tokio::test]
13205    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13206        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13207        let path = dir.join("subc.jsonc");
13208        let registry = Arc::new(Registry::default());
13209        let handle = SupervisorHandle::new();
13210        let supervisor =
13211            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13212        let handler = ControlHandler::new(registry)
13213            .with_supervisor(handle)
13214            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13215            .with_supervisor_rescan(supervisor, &path, None);
13216        handler.forwarding.operator_confirms().configure(
13217            Arc::new(ApproveOperator),
13218            crate::operator_confirm::OperatorLimits::default(),
13219        );
13220        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13221        for preview in [true, false] {
13222            std::fs::write(
13223                &path,
13224                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13225            )
13226            .unwrap();
13227            let request = ClientControlRequest::SupervisorRescan { preview };
13228            let frame = Frame::build(
13229                FrameType::Request,
13230                control_flags(),
13231                0,
13232                0,
13233                1,
13234                serde_json::to_vec(&request).unwrap(),
13235            )
13236            .unwrap();
13237            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13238            assert_eq!(frames.len(), 1);
13239            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13240            assert_eq!(
13241                handler.privileged_operator_prompt,
13242                PrivilegedOperatorPrompt::Enforce
13243            );
13244        }
13245    }
13246
13247    /// One wire code has several senders, so the refusal line names the check
13248    /// that refused. This drives the shared refusal path for ordinary refusals
13249    /// with an unregistered
13250    /// target and requires the branch label on the event.
13251    #[tokio::test]
13252    async fn route_open_refusal_names_the_check_that_refused() {
13253        let handler = ControlHandler::new(Arc::new(Registry::default()));
13254        let capture = EventCapture::default();
13255        let _subscriber =
13256            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13257        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13258        let response = handler
13259            .handle_control_frame(
13260                &ctx,
13261                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13262            )
13263            .await
13264            .unwrap();
13265
13266        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13267        let event = capture
13268            .events()
13269            .into_iter()
13270            .find(|event| {
13271                event.target == "control"
13272                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13273            })
13274            .expect("route.open refusal event");
13275        assert_eq!(
13276            event.fields.get("reason"),
13277            Some(&"\"not_registered\"".to_string())
13278        );
13279    }
13280
13281    #[tokio::test]
13282    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13283        let registry = Arc::new(Registry::default());
13284        let supervisor_handle = SupervisorHandle::new();
13285        let supervisor =
13286            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13287                .with_handle(supervisor_handle.clone())
13288                .with_connection_file_path(std::env::temp_dir().join(format!(
13289                    "subc-route-open-refusal-info-{}",
13290                    std::process::id()
13291                )));
13292        let module = supervisor
13293            .supervise_configured(
13294                ModuleSpec {
13295                    module_id: "warming".to_string(),
13296                    program: fake_aft_stub_path(),
13297                    args: Vec::new(),
13298                    env: Vec::new(),
13299                    reserved: false,
13300                    reserved_prefixes: Vec::new(),
13301                    protocol: ModuleProtocol::Subc,
13302                    overlap: Default::default(),
13303                },
13304                true,
13305            )
13306            .unwrap();
13307        assert_eq!(module.state().unwrap(), ModuleState::Running);
13308
13309        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13310        assert!(handler
13311            .counters()
13312            .snapshot()
13313            .get("route_open_refused_by_code")
13314            .is_none());
13315        let capture = EventCapture::default();
13316        let _subscriber =
13317            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13318        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13319        let response = handler
13320            .handle_control_frame(
13321                &ctx,
13322                route_open_frame(394, "warming", unique_project_root("refusal-info")),
13323            )
13324            .await
13325            .unwrap();
13326        module.stop().await.unwrap();
13327
13328        assert_eq!(parse_error(&response[0])["code"], "module_warming");
13329        let event = capture
13330            .events()
13331            .into_iter()
13332            .find(|event| {
13333                event.target == "control"
13334                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13335            })
13336            .expect("route.open refusal event");
13337        assert_eq!(
13338            event.fields.get("module_id"),
13339            Some(&"\"warming\"".to_string())
13340        );
13341        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13342        assert_eq!(
13343            event.fields.get("reason"),
13344            Some(&"\"supervised_not_registered\"".to_string())
13345        );
13346        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13347        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13348        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13349        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13350        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13351        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13352        assert_eq!(
13353            handler.counters().snapshot()["route_open_refused_by_code"],
13354            json!({ "module_warming": 1 })
13355        );
13356    }
13357
13358    const OUTAGE_START: &str = "route.open refusing module: not serving";
13359    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13360
13361    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13362        capture
13363            .events()
13364            .into_iter()
13365            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13366            .collect()
13367    }
13368
13369    fn supervise_stub(
13370        registry: &Arc<Registry>,
13371        module_id: &str,
13372        enabled: bool,
13373    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13374        let supervisor_handle = SupervisorHandle::new();
13375        let supervisor =
13376            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13377                .with_handle(supervisor_handle.clone())
13378                .with_connection_file_path(std::env::temp_dir().join(format!(
13379                    "subc-route-outage-{module_id}-{}",
13380                    std::process::id()
13381                )));
13382        let module = supervisor
13383            .supervise_configured(
13384                ModuleSpec {
13385                    module_id: module_id.to_string(),
13386                    program: fake_aft_stub_path(),
13387                    args: Vec::new(),
13388                    env: Vec::new(),
13389                    reserved: false,
13390                    reserved_prefixes: Vec::new(),
13391                    protocol: ModuleProtocol::Subc,
13392                    overlap: Default::default(),
13393                },
13394                enabled,
13395            )
13396            .unwrap();
13397        (supervisor_handle, module)
13398    }
13399
13400    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13401        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13402            module_id: module_id.to_string(),
13403            drain_timeout_ms: Some(50),
13404        })
13405        .unwrap();
13406        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13407    }
13408
13409    /// Two handlers built over one forwarding table must share one outage
13410    /// tracker; separate trackers would each log their own opening line for
13411    /// the same outage.
13412    #[test]
13413    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13414        let registry = Arc::new(Registry::default());
13415        let forwarding = Arc::new(ForwardingTable::default());
13416        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13417        let second = ControlHandler::with_forwarding(registry, forwarding);
13418        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13419    }
13420
13421    /// A client can name any module id it likes. Refusing an unknown one,
13422    /// however often, must not create outage state or outage lines, or the
13423    /// tracker would be a memory sink any client could fill.
13424    #[tokio::test(flavor = "current_thread")]
13425    async fn route_open_unknown_module_refusals_add_no_outage_state() {
13426        let handler = ControlHandler::new(Arc::new(Registry::default()));
13427        let capture = EventCapture::default();
13428        let _subscriber =
13429            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13430        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13431        for corr in 0..8 {
13432            let response = handler
13433                .handle_control_frame(
13434                    &ctx,
13435                    route_open_frame(
13436                        380 + corr,
13437                        &format!("nobody-{corr}"),
13438                        unique_project_root("outage-unknown"),
13439                    ),
13440                )
13441                .await
13442                .unwrap();
13443            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13444        }
13445
13446        assert_eq!(handler.route_outages.tracked_module_count(), 0);
13447        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13448        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13449    }
13450
13451    /// Drives the refusal path end to end: a supervised module that served
13452    /// before and stopped being registered with no instruction to stop is a
13453    /// WARN, and the same module refused after an operator `supervisor.restart`
13454    /// is an INFO.
13455    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13456    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13457        let registry = Arc::new(Registry::default());
13458        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
13459        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13460        let capture = EventCapture::default();
13461        let _subscriber =
13462            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13463        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13464        // The stub never registers, so pretend it served once: otherwise every
13465        // refusal would fall in its startup window.
13466        handler.route_outages.record_accepted("outage-restart");
13467
13468        let response = handler
13469            .handle_control_frame(
13470                &ctx,
13471                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13472            )
13473            .await
13474            .unwrap();
13475        assert_eq!(response[0].header.ty, FrameType::Error);
13476        let starts = outage_lines(&capture, OUTAGE_START);
13477        assert_eq!(starts.len(), 1, "{starts:?}");
13478        assert_eq!(starts[0].level, tracing::Level::WARN);
13479        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13480        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13481        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13482        handler.route_outages.record_accepted("outage-restart");
13483        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13484
13485        let restart = handler
13486            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13487            .await
13488            .unwrap();
13489        assert_eq!(
13490            restart[0].header.ty,
13491            FrameType::Response,
13492            "{:?}",
13493            parse_error(&restart[0])
13494        );
13495        handler
13496            .handle_control_frame(
13497                &ctx,
13498                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13499            )
13500            .await
13501            .unwrap();
13502        module.stop().await.unwrap();
13503
13504        let starts = outage_lines(&capture, OUTAGE_START);
13505        assert_eq!(starts.len(), 2, "{starts:?}");
13506        assert_eq!(starts[1].level, tracing::Level::INFO);
13507        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13508    }
13509
13510    /// A restart refused before it touched the module (here: the module is
13511    /// disabled) must clear its operator mark, so the next real outage is
13512    /// still reported as a warning.
13513    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13514    async fn failed_operator_restart_leaves_no_operator_mark() {
13515        let registry = Arc::new(Registry::default());
13516        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
13517        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13518        let capture = EventCapture::default();
13519        let _subscriber =
13520            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13521        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13522        handler.route_outages.record_accepted("outage-disabled");
13523
13524        let restart = handler
13525            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13526            .await
13527            .unwrap();
13528        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13529        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13530
13531        handler
13532            .handle_control_frame(
13533                &ctx,
13534                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13535            )
13536            .await
13537            .unwrap();
13538        let starts = outage_lines(&capture, OUTAGE_START);
13539        assert_eq!(starts.len(), 1, "{starts:?}");
13540        assert_eq!(starts[0].level, tracing::Level::WARN);
13541    }
13542
13543    #[tokio::test(flavor = "current_thread")]
13544    async fn route_open_unknown_module_escapes_target_module_id() {
13545        let handler = ControlHandler::new(Arc::new(Registry::default()));
13546        let capture = EventCapture::default();
13547        let _subscriber =
13548            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13549        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13550        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13551        let response = handler
13552            .handle_control_frame(
13553                &ctx,
13554                route_open_frame(
13555                    395,
13556                    hostile_module_id,
13557                    unique_project_root("hostile-target-module-id"),
13558                ),
13559            )
13560            .await
13561            .unwrap();
13562
13563        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13564        let event = capture
13565            .events()
13566            .into_iter()
13567            .find(|event| {
13568                event.target == "control"
13569                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13570            })
13571            .expect("route.open unknown-module refusal event");
13572        let logged = event.fields.get("module_id").expect("module_id field");
13573        assert!(!logged.bytes().any(|byte| byte < 0x20));
13574        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13575    }
13576
13577    #[tokio::test(flavor = "current_thread")]
13578    async fn route_open_module_rejection_uses_daemon_counter_key() {
13579        let registry = Arc::new(Registry::default());
13580        let forwarding = Arc::new(ForwardingTable::default());
13581        let handler =
13582            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13583        let module_connection = ConnectionId::new(95);
13584        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13585        hello_via_sink(
13586            &handler,
13587            &module_ctx,
13588            &mut module_rx,
13589            hello_frame("aft", PROTOCOL_VERSION, 395),
13590        )
13591        .await;
13592
13593        let client_connection = ConnectionId::new(96);
13594        let (client_ctx, _client_rx) = route_ctx(client_connection);
13595        let capture = EventCapture::default();
13596        let _subscriber =
13597            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13598        let (route_task, bind) = relay_route_open(
13599            &handler,
13600            client_connection,
13601            &client_ctx.egress,
13602            &mut module_rx,
13603            396,
13604            "aft",
13605            "hostile-module-code",
13606        )
13607        .await;
13608        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13609        let rejection = Frame::build(
13610            FrameType::Error,
13611            control_flags(),
13612            0,
13613            0,
13614            bind.header.corr,
13615            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13616        )
13617        .unwrap();
13618        handler
13619            .handle_control_frame(&module_ctx, rejection)
13620            .await
13621            .unwrap();
13622
13623        let response = route_task.await.unwrap();
13624        assert_eq!(parse_error(&response[0])["code"], hostile_code);
13625        let counters = handler.counters().snapshot();
13626        assert_eq!(
13627            counters["route_open_refused_by_code"],
13628            json!({ "module_rejected": 1 })
13629        );
13630        assert!(counters["route_open_refused_by_code"]
13631            .get(hostile_code)
13632            .is_none());
13633
13634        let event = capture
13635            .events()
13636            .into_iter()
13637            .find(|event| {
13638                event.target == "control"
13639                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13640            })
13641            .expect("route.open module-rejection refusal event");
13642        let logged = event.fields.get("module_code").expect("module_code field");
13643        assert!(!logged.bytes().any(|byte| byte < 0x20));
13644        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13645    }
13646
13647    #[tokio::test]
13648    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13649        let registry = Arc::new(Registry::default());
13650        let supervisor_handle = SupervisorHandle::new();
13651        let missing_program = std::env::temp_dir().join(format!(
13652            "subc-route-open-missing-program-{}",
13653            std::process::id()
13654        ));
13655        let supervisor =
13656            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13657                .with_handle(supervisor_handle.clone());
13658        let module = supervisor
13659            .supervise_configured(
13660                ModuleSpec {
13661                    module_id: "failed".to_string(),
13662                    program: missing_program,
13663                    args: Vec::new(),
13664                    env: Vec::new(),
13665                    reserved: false,
13666                    reserved_prefixes: Vec::new(),
13667                    protocol: ModuleProtocol::Subc,
13668                    overlap: Default::default(),
13669                },
13670                true,
13671            )
13672            .unwrap();
13673        assert_eq!(module.state().unwrap(), ModuleState::Failed);
13674
13675        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13676        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13677        let response = handler
13678            .handle_control_frame(
13679                &ctx,
13680                route_open_frame(305, "failed", unique_project_root("failed")),
13681            )
13682            .await
13683            .unwrap();
13684
13685        assert_eq!(response[0].header.ty, FrameType::Error);
13686        let error = parse_error(&response[0]);
13687        assert_eq!(error["code"], "target_unavailable");
13688        assert!(error["message"]
13689            .as_str()
13690            .unwrap()
13691            .contains("state=failed, enabled=true, live=false"));
13692    }
13693
13694    #[tokio::test]
13695    async fn route_open_role_mismatch_remains_target_unavailable() {
13696        let registry = Arc::new(Registry::default());
13697        let handler = ControlHandler::new(Arc::clone(&registry));
13698        handler
13699            .handle_control(
13700                ConnectionId::new(41),
13701                non_routable_hello_frame_with_control_ops("health-only", 306, None),
13702            )
13703            .unwrap();
13704
13705        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13706        let response = handler
13707            .handle_control_frame(
13708                &ctx,
13709                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13710            )
13711            .await
13712            .unwrap();
13713
13714        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13715        assert!(parse_error(&response[0])["message"]
13716            .as_str()
13717            .unwrap()
13718            .contains("does not provide the requested target"));
13719    }
13720
13721    #[tokio::test]
13722    async fn route_open_inactive_registration_remains_target_unavailable() {
13723        let registry = Arc::new(Registry::default());
13724        let handler = ControlHandler::new(Arc::clone(&registry));
13725        handler
13726            .handle_control(
13727                ConnectionId::new(43),
13728                hello_frame("inactive", PROTOCOL_VERSION, 308),
13729            )
13730            .unwrap();
13731        assert!(registry
13732            .set_module_state_for_test("inactive", ChannelState::Closed)
13733            .unwrap());
13734
13735        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13736        let response = handler
13737            .handle_control_frame(
13738                &ctx,
13739                route_open_frame(309, "inactive", unique_project_root("inactive")),
13740            )
13741            .await
13742            .unwrap();
13743
13744        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13745        assert!(parse_error(&response[0])["message"]
13746            .as_str()
13747            .unwrap()
13748            .contains("is not active"));
13749    }
13750
13751    #[tokio::test]
13752    async fn late_health_reply_is_recorded_through_the_module_response_path() {
13753        let registry = Arc::new(Registry::default());
13754        let forwarding = Arc::new(ForwardingTable::default());
13755        let supervisor_handle = SupervisorHandle::new();
13756        let supervisor =
13757            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
13758                .with_forwarding(Arc::clone(&forwarding))
13759                .with_handle(supervisor_handle.clone());
13760        let module = supervisor
13761            .supervise_configured(
13762                crate::ModuleSpec {
13763                    module_id: "late-health-response".to_string(),
13764                    program: PathBuf::from("disabled-module"),
13765                    args: Vec::new(),
13766                    env: Vec::new(),
13767                    reserved: false,
13768                    reserved_prefixes: Vec::new(),
13769                    protocol: ModuleProtocol::Subc,
13770                    overlap: Default::default(),
13771                },
13772                false,
13773            )
13774            .unwrap();
13775        let handler =
13776            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13777                .with_supervisor(supervisor_handle);
13778        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13779        handler
13780            .handle_control_frame(
13781                &module_ctx,
13782                hello_frame_with_control_ops(
13783                    "late-health-response",
13784                    PROTOCOL_VERSION,
13785                    7,
13786                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13787                ),
13788            )
13789            .await
13790            .unwrap();
13791        let probe_started_at = Instant::now() - Duration::from_millis(80);
13792        let pending = forwarding
13793            .begin_health_probe_rpc_for(
13794                "late-health-response",
13795                MODULE_CONTROL_OP_HEALTH_CHECK,
13796                probe_started_at,
13797                Instant::now() - Duration::from_millis(1),
13798            )
13799            .unwrap();
13800        assert!(forwarding
13801            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13802            .unwrap());
13803
13804        let responses = handler
13805            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13806            .await
13807            .unwrap();
13808
13809        assert!(responses.is_empty());
13810        let health = module.status().unwrap().health;
13811        assert_eq!(health.late_answer_count, 1);
13812        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13813    }
13814
13815    #[tokio::test]
13816    async fn health_probe_timeout_and_module_death_are_typed() {
13817        let registry = Arc::new(Registry::default());
13818        let forwarding = Arc::new(ForwardingTable::default());
13819        let handler =
13820            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13821                .with_health_probe_timeout(Duration::from_millis(50));
13822        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13823        hello_via_sink(
13824            &handler,
13825            &module_ctx,
13826            &mut module_rx,
13827            hello_frame_with_control_ops(
13828                "aft",
13829                PROTOCOL_VERSION,
13830                7,
13831                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13832            ),
13833        )
13834        .await;
13835
13836        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13837        let responses = handler
13838            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13839            .await
13840            .unwrap();
13841        assert_eq!(responses[0].header.ty, FrameType::Error);
13842        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13843        let _ = module_rx.try_recv();
13844
13845        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13846        let health_handler = handler.clone();
13847        let death_task = tokio::spawn(async move {
13848            health_handler
13849                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13850                .await
13851                .unwrap()
13852        });
13853        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13854            .await
13855            .unwrap()
13856            .unwrap();
13857        handler
13858            .cleanup_connection(module_ctx.connection_id)
13859            .unwrap();
13860        let responses = death_task.await.unwrap();
13861        assert_eq!(responses[0].header.ty, FrameType::Error);
13862        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13863    }
13864
13865    #[test]
13866    fn hello_requires_exact_protocol_version() {
13867        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13868            let registry = Arc::new(Registry::default());
13869            let handler = ControlHandler::new(Arc::clone(&registry));
13870            let responses = handler
13871                .handle_control(
13872                    ConnectionId::new(connection),
13873                    hello_frame("aft", offered, 9),
13874                )
13875                .unwrap();
13876
13877            assert_eq!(responses.len(), 1);
13878            assert_eq!(responses[0].header.ty, FrameType::Error);
13879            let error = parse_error(&responses[0]);
13880            assert_eq!(error["code"], "version_unsupported");
13881            assert!(registry.get_module("aft").unwrap().is_none());
13882            assert_eq!(registry.active_registration_count().unwrap(), 0);
13883        }
13884    }
13885
13886    #[test]
13887    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13888        let registry = Arc::new(Registry::default());
13889        let forwarding = Arc::new(ForwardingTable::default());
13890        let handler =
13891            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13892        let module_connection = ConnectionId::new(301);
13893        let registration = registry
13894            .register_with_control_ops(
13895                manifest("aft-push", PROTOCOL_VERSION),
13896                PROTOCOL_VERSION,
13897                module_connection,
13898                module_baseline_control_ops(),
13899            )
13900            .unwrap();
13901        let (module_tx, _module_rx) = mpsc::channel(8);
13902        let endpoint = forwarding
13903            .register_module_connection(
13904                module_connection,
13905                "aft-push".to_string(),
13906                PROTOCOL_VERSION,
13907                manifest_concurrency(&registration.manifest),
13908                FrameSink::new(module_tx),
13909            )
13910            .unwrap();
13911
13912        // A push op this version does not know is ignored (forward-compat), not errored.
13913        let unknown = Frame::build(
13914            FrameType::Push,
13915            control_flags(),
13916            0,
13917            0,
13918            5,
13919            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13920        )
13921        .unwrap();
13922        let out = handler.handle_status_update(endpoint, unknown).unwrap();
13923        assert!(
13924            out.is_empty(),
13925            "unknown push op must be ignored, got {out:?}"
13926        );
13927
13928        // A malformed body for a KNOWN op is a real error worth surfacing.
13929        let malformed = Frame::build(
13930            FrameType::Push,
13931            control_flags(),
13932            0,
13933            0,
13934            6,
13935            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13936        )
13937        .unwrap();
13938        let out = handler.handle_status_update(endpoint, malformed).unwrap();
13939        assert_eq!(out.len(), 1);
13940        assert_eq!(out[0].header.ty, FrameType::Error);
13941        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13942    }
13943
13944    #[test]
13945    fn hello_rejected_when_connection_already_owns_client_routes() {
13946        let registry = Arc::new(Registry::default());
13947        let forwarding = Arc::new(ForwardingTable::default());
13948        let handler =
13949            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13950        // Commits a client route on connection 202 (bound to a module on conn 101).
13951        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
13952        let client_connection = ConnectionId::new(202);
13953
13954        // That same connection now tries to register as a module: rejected, so one
13955        // connection never holds both client-route and module-endpoint state.
13956        let responses = handler
13957            .handle_control(
13958                client_connection,
13959                hello_frame("aft-second", PROTOCOL_VERSION, 9),
13960            )
13961            .unwrap();
13962        assert_eq!(responses[0].header.ty, FrameType::Error);
13963        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13964        assert!(registry.get_module("aft-second").unwrap().is_none());
13965    }
13966
13967    #[tokio::test]
13968    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13969        let registry = Arc::new(Registry::default());
13970        let forwarding = Arc::new(ForwardingTable::default());
13971        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13972        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13973        hello_via_sink(
13974            &handler,
13975            &module_ctx,
13976            &mut module_rx,
13977            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13978        )
13979        .await;
13980        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13981        let pending = forwarding
13982            .begin_route_bind_relay_for_test(
13983                client_ctx.connection_id,
13984                client_ctx.egress.clone(),
13985                2,
13986                "alpha",
13987            )
13988            .unwrap();
13989        forwarding
13990            .complete_pending_relay(
13991                module_ctx.connection_id,
13992                pending.corr,
13993                RouteBindRelayOutcome::Accepted,
13994            )
13995            .unwrap();
13996        client_rx.try_recv().unwrap();
13997        for module_id in ["beta", "alpha"] {
13998            let replies = handler
13999                .handle_control_frame(
14000                    &module_ctx,
14001                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14002                )
14003                .await
14004                .unwrap();
14005            assert_eq!(replies.len(), 1, "second HELLO must be refused");
14006            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14007        }
14008        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14009        assert!(registry.get_module("beta").unwrap().is_none());
14010        assert!(matches!(
14011            forwarding
14012                .lookup_data_route(
14013                    client_ctx.connection_id,
14014                    pending.client_channel,
14015                    pending.client_epoch,
14016                )
14017                .unwrap(),
14018            DataRoute::Client(DataRouteState::Bound(_))
14019        ));
14020        assert!(handler
14021            .hello_launch_nonces
14022            .lock()
14023            .unwrap()
14024            .presented(module_ctx.connection_id, Some("alpha-nonce")));
14025        assert!(module_rx.try_recv().is_err());
14026    }
14027
14028    #[test]
14029    fn reserved_module_hello_requires_matching_launch_nonce() {
14030        let registry = Arc::new(Registry::default());
14031        let supervisor = SupervisorHandle::new();
14032        // The supervisor recorded the nonce it injected when it spawned the reserved
14033        // module; the HELLO verifier checks against the same shared handle.
14034        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14035        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14036
14037        // A HELLO with NO nonce is rejected.
14038        let no_nonce = handler
14039            .handle_control(
14040                ConnectionId::new(1),
14041                hello_frame("vault", PROTOCOL_VERSION, 1),
14042            )
14043            .unwrap();
14044        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14045        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14046        assert!(registry.get_module("vault").unwrap().is_none());
14047
14048        // A HELLO with the WRONG nonce is rejected.
14049        let wrong = handler
14050            .handle_control(
14051                ConnectionId::new(2),
14052                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14053            )
14054            .unwrap();
14055        assert_eq!(wrong[0].header.ty, FrameType::Error);
14056        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14057        assert!(registry.get_module("vault").unwrap().is_none());
14058
14059        // A HELLO with the CORRECT nonce registers.
14060        let ok = handler
14061            .handle_control(
14062                ConnectionId::new(3),
14063                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14064            )
14065            .unwrap();
14066        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14067        assert!(registry.get_module("vault").unwrap().is_some());
14068    }
14069
14070    #[test]
14071    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14072        let registry = Arc::new(Registry::default());
14073        let supervisor = SupervisorHandle::new();
14074        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14075        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14076        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14077
14078        let squat = handler
14079            .handle_control(
14080                ConnectionId::new(1),
14081                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14082            )
14083            .unwrap();
14084        assert_eq!(squat[0].header.ty, FrameType::Error);
14085        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14086        assert!(parse_error(&squat[0])["message"]
14087            .as_str()
14088            .unwrap()
14089            .contains("fed:"));
14090
14091        let accepted_peer = handler
14092            .handle_control(
14093                ConnectionId::new(2),
14094                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14095            )
14096            .unwrap();
14097        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14098
14099        let accepted_short = handler
14100            .handle_control(
14101                ConnectionId::new(3),
14102                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14103            )
14104            .unwrap();
14105        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14106
14107        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14108            let response = handler
14109                .handle_control(
14110                    ConnectionId::new(conn),
14111                    hello_frame(module_id, PROTOCOL_VERSION, conn),
14112                )
14113                .unwrap();
14114            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14115        }
14116    }
14117
14118    #[test]
14119    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14120        let registry = Arc::new(Registry::default());
14121        let supervisor = SupervisorHandle::new();
14122        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14123        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14124        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14125        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14126
14127        let owner_nonce = handler
14128            .handle_control(
14129                ConnectionId::new(1),
14130                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14131            )
14132            .unwrap();
14133        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14134        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14135        assert!(registry.get_module("fed:special").unwrap().is_none());
14136
14137        let exact_nonce = handler
14138            .handle_control(
14139                ConnectionId::new(2),
14140                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14141            )
14142            .unwrap();
14143        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14144        assert!(registry.get_module("fed:special").unwrap().is_some());
14145    }
14146
14147    #[test]
14148    fn non_reserved_module_ignores_launch_nonce() {
14149        let registry = Arc::new(Registry::default());
14150        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
14151        // registration succeeds whether a spawned process echoes a nonce or not.
14152        let handler = ControlHandler::new(Arc::clone(&registry));
14153        let no_nonce = handler
14154            .handle_control(
14155                ConnectionId::new(1),
14156                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14157            )
14158            .unwrap();
14159        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14160        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14161
14162        let echoed_nonce = handler
14163            .handle_control(
14164                ConnectionId::new(2),
14165                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14166            )
14167            .unwrap();
14168        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14169        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14170    }
14171
14172    #[test]
14173    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14174        let handler = ControlHandler::default();
14175        let conn = ConnectionId::new(1);
14176        let malformed = Frame::build(
14177            FrameType::Hello,
14178            control_flags(),
14179            0,
14180            0,
14181            3,
14182            b"{not json".to_vec(),
14183        )
14184        .unwrap();
14185
14186        let error = handler.handle_control(conn, malformed).unwrap();
14187        assert_eq!(error[0].header.ty, FrameType::Error);
14188        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14189
14190        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14191        let pong = handler.handle_control(conn, ping).unwrap();
14192        assert_eq!(pong[0].header.ty, FrameType::Pong);
14193        assert_eq!(pong[0].header.corr, 4);
14194    }
14195
14196    #[test]
14197    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14198        let registry = Arc::new(Registry::default());
14199        let handler = ControlHandler::new(Arc::clone(&registry));
14200
14201        handler
14202            .handle_control(
14203                ConnectionId::new(1),
14204                hello_frame("aft", PROTOCOL_VERSION, 1),
14205            )
14206            .unwrap();
14207        let duplicate = handler
14208            .handle_control(
14209                ConnectionId::new(2),
14210                hello_frame("aft", PROTOCOL_VERSION, 2),
14211            )
14212            .unwrap();
14213
14214        assert_eq!(duplicate[0].header.ty, FrameType::Error);
14215        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14216        let registration = registry.get_module("aft").unwrap().unwrap();
14217        assert_eq!(registration.connection_id, ConnectionId::new(1));
14218    }
14219
14220    #[test]
14221    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14222        let registry = Arc::new(Registry::default());
14223        let forwarding = Arc::new(ForwardingTable::default());
14224        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14225        let handler =
14226            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14227                .with_process_liveness(process_liveness);
14228        let (ctx, route_channel, route_epoch) =
14229            bind_liveness_route(&registry, &forwarding, "aft-dead");
14230        let responses = handler
14231            .handle_route_poll(
14232                &ctx,
14233                route_poll_frame(41, PollKind::Liveness, route_channel),
14234                route_channel,
14235                route_epoch,
14236                PollKind::Liveness,
14237            )
14238            .unwrap();
14239
14240        assert_eq!(responses.len(), 1);
14241        assert_eq!(responses[0].header.ty, FrameType::Response);
14242        assert_route_poll_liveness(&responses[0], false);
14243    }
14244
14245    #[test]
14246    fn liveness_poll_without_process_source_uses_bound_route() {
14247        let registry = Arc::new(Registry::default());
14248        let forwarding = Arc::new(ForwardingTable::default());
14249        let handler =
14250            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14251        let (ctx, route_channel, route_epoch) =
14252            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
14253        let responses = handler
14254            .handle_route_poll(
14255                &ctx,
14256                route_poll_frame(42, PollKind::Liveness, route_channel),
14257                route_channel,
14258                route_epoch,
14259                PollKind::Liveness,
14260            )
14261            .unwrap();
14262
14263        assert_route_poll_liveness(&responses[0], true);
14264    }
14265
14266    #[test]
14267    fn liveness_poll_untracked_process_source_uses_bound_route() {
14268        let registry = Arc::new(Registry::default());
14269        let forwarding = Arc::new(ForwardingTable::default());
14270        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14271        let handler =
14272            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14273                .with_process_liveness(process_liveness);
14274        let (ctx, route_channel, route_epoch) =
14275            bind_liveness_route(&registry, &forwarding, "aft-untracked");
14276        let responses = handler
14277            .handle_route_poll(
14278                &ctx,
14279                route_poll_frame(43, PollKind::Liveness, route_channel),
14280                route_channel,
14281                route_epoch,
14282                PollKind::Liveness,
14283            )
14284            .unwrap();
14285
14286        assert_route_poll_liveness(&responses[0], true);
14287    }
14288
14289    #[tokio::test]
14290    async fn unknown_op_returns_unknown_control_op() {
14291        let handler = ControlHandler::default();
14292        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14293        let request = Frame::build(
14294            FrameType::Request,
14295            control_flags(),
14296            0,
14297            0,
14298            55,
14299            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14300        )
14301        .unwrap();
14302
14303        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14304
14305        assert_eq!(response.len(), 1);
14306        assert_eq!(response[0].header.ty, FrameType::Error);
14307        assert_eq!(response[0].header.corr, 55);
14308        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14309    }
14310
14311    #[tokio::test]
14312    async fn supervisor_provenance_rejects_unknown_exact_module() {
14313        let handler = ControlHandler::default();
14314        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14315        let request = Frame::build(
14316            FrameType::Request,
14317            control_flags(),
14318            0,
14319            0,
14320            57,
14321            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14322        )
14323        .unwrap();
14324
14325        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14326
14327        assert_eq!(response.len(), 1);
14328        assert_eq!(response[0].header.ty, FrameType::Error);
14329        assert_eq!(response[0].header.corr, 57);
14330        let error = parse_error(&response[0]);
14331        assert_eq!(error["code"], "unknown_module");
14332        assert_eq!(error["message"], "module_id 'missing' is not supervised");
14333    }
14334
14335    #[test]
14336    fn provenance_probe_override_keeps_handler_tests_deterministic() {
14337        let expected = subc_control::RunningImageAgreement::Unavailable {
14338            reason: subc_control::RunningImageUnavailableReason::HashFailed,
14339        };
14340        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14341        assert_eq!(handler.provenance_probe_override, Some(expected));
14342    }
14343
14344    #[test]
14345    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14346        let verdict = reload_verdict(
14347            std::path::Path::new("/bin/new"),
14348            Some(std::path::Path::new("/bin/old")),
14349            subc_control::RunningImageAgreement::Unavailable {
14350                reason: subc_control::RunningImageUnavailableReason::HashFailed,
14351            },
14352        );
14353        assert!(matches!(
14354            verdict.path,
14355            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14356                if configured == std::path::Path::new("/bin/new")
14357                    && spawned_from == std::path::Path::new("/bin/old")
14358        ));
14359    }
14360
14361    #[test]
14362    fn launch_spec_without_running_digest_is_unavailable_not_match() {
14363        use subc_control::{
14364            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14365        };
14366        let configured = crate::supervise::ModuleSpec {
14367            module_id: "legacy".into(),
14368            program: "/module".into(),
14369            args: Vec::new(),
14370            env: Vec::new(),
14371            reserved: false,
14372            reserved_prefixes: Vec::new(),
14373            protocol: subc_control::ModuleProtocol::Subc,
14374            overlap: Default::default(),
14375        };
14376        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14377        assert_eq!(
14378            launch_spec_agreement(&configured, false, Some(&digest)),
14379            Agreement::Unavailable {
14380                reason: Reason::NotRunning
14381            }
14382        );
14383        assert_eq!(
14384            launch_spec_agreement(&configured, true, None),
14385            Agreement::Unavailable {
14386                reason: Reason::SpawnedSpecUnavailable
14387            }
14388        );
14389        assert_eq!(
14390            launch_spec_agreement(&configured, true, Some(&digest)),
14391            Agreement::Match
14392        );
14393    }
14394
14395    #[cfg(windows)]
14396    #[test]
14397    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14398        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14399        let configured = dir.join("module.exe");
14400        std::fs::write(&configured, b"executable path fixture").unwrap();
14401        let spawned = std::fs::canonicalize(&configured).unwrap();
14402        assert_ne!(
14403            configured, spawned,
14404            "fixture must exercise different path spellings"
14405        );
14406        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14407        let lower_case = configured.to_string_lossy().to_lowercase();
14408        for spelling in [
14409            &configured,
14410            Path::new(&forward_slashes),
14411            Path::new(&lower_case),
14412        ] {
14413            let verdict = reload_verdict(
14414                spelling,
14415                Some(&spawned),
14416                subc_control::RunningImageAgreement::Unavailable {
14417                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14418                },
14419            );
14420            assert_eq!(
14421                verdict.path,
14422                ReloadPathAgreement::Match,
14423                "configured {spelling:?}, spawned {spawned:?}"
14424            );
14425        }
14426    }
14427
14428    #[cfg(windows)]
14429    #[test]
14430    fn windows_reload_path_keeps_different_existing_program_pending() {
14431        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14432        let old = dir.join("old.exe");
14433        let new = dir.join("new.exe");
14434        std::fs::write(&old, b"same executable content").unwrap();
14435        std::fs::copy(&old, &new).unwrap();
14436        let spawned = std::fs::canonicalize(&old).unwrap();
14437        let image = subc_control::RunningImageAgreement::Unavailable {
14438            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14439        };
14440        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14441        assert!(
14442            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14443            if configured == new && spawned_from == spawned)
14444        );
14445        let missing = dir.join("missing.exe");
14446        assert!(matches!(
14447            reload_verdict(&missing, Some(&spawned), image).path,
14448            ReloadPathAgreement::Mismatch { .. }
14449        ));
14450    }
14451
14452    #[test]
14453    fn reload_verdict_detects_replaced_image_at_same_path() {
14454        let image = subc_control::RunningImageAgreement::Mismatch {
14455            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14456                digest: "old".into(),
14457            },
14458            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14459                digest: "new".into(),
14460            },
14461        };
14462        let verdict = reload_verdict(
14463            std::path::Path::new("/bin/same"),
14464            Some(std::path::Path::new("/bin/same")),
14465            image.clone(),
14466        );
14467        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14468        assert_eq!(verdict.image, image);
14469    }
14470
14471    #[test]
14472    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14473        let image = subc_control::RunningImageAgreement::Unavailable {
14474            reason: subc_control::RunningImageUnavailableReason::NotRunning,
14475        };
14476        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14477        assert_eq!(
14478            verdict.path,
14479            subc_control::ReloadPathAgreement::Unavailable {
14480                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14481            }
14482        );
14483        assert_eq!(verdict.image, image);
14484
14485        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14486            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14487        };
14488        let verdict = reload_verdict(
14489            std::path::Path::new("/bin/same"),
14490            Some(std::path::Path::new("/bin/same")),
14491            unconfirmed.clone(),
14492        );
14493        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14494        assert_eq!(verdict.image, unconfirmed);
14495    }
14496
14497    #[test]
14498    fn reload_verdict_preserves_each_image_unavailability_reason() {
14499        use subc_control::RunningImageUnavailableReason as Reason;
14500
14501        for reason in [
14502            Reason::NotRunning,
14503            Reason::UnsupportedPlatform,
14504            Reason::RunningExecutableUnreadable,
14505            Reason::SpawnedPathUnreadable,
14506            Reason::HashFailed,
14507            Reason::ProcessIdentityUnconfirmed,
14508            Reason::Unknown("future_probe_reason".to_string()),
14509        ] {
14510            let image = subc_control::RunningImageAgreement::Unavailable {
14511                reason: reason.clone(),
14512            };
14513            let verdict = reload_verdict(
14514                std::path::Path::new("/bin/same"),
14515                Some(std::path::Path::new("/bin/same")),
14516                image.clone(),
14517            );
14518            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14519            assert_eq!(verdict.image, image, "{reason:?}");
14520        }
14521    }
14522
14523    #[tokio::test]
14524    async fn malformed_control_bodies_return_invalid_control_body() {
14525        let handler = ControlHandler::default();
14526        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14527
14528        for (corr, body) in [
14529            (56, br#"{"route_channel":1}"#.as_slice()),
14530            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14531            (
14532                58,
14533                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14534            ),
14535        ] {
14536            let request = Frame::build(
14537                FrameType::Request,
14538                control_flags(),
14539                0,
14540                0,
14541                corr,
14542                body.to_vec(),
14543            )
14544            .unwrap();
14545            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14546
14547            assert_eq!(response.len(), 1);
14548            assert_eq!(response[0].header.ty, FrameType::Error);
14549            assert_eq!(response[0].header.corr, corr);
14550            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14551        }
14552    }
14553
14554    #[tokio::test]
14555    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14556        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14557        let registry = Arc::new(Registry::default());
14558        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14559        let router = Router::with_control_handler(Arc::clone(&control));
14560        let connection = router.begin_connection();
14561        let (ctx, mut rx) = route_ctx(connection.id());
14562
14563        router
14564            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14565            .await
14566            .unwrap();
14567        let response = rx.recv().await.unwrap();
14568        let ack = parse_ack(&response);
14569        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14570        let channel = 1;
14571
14572        let goodbye =
14573            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14574        router.route_for_connection(&ctx, goodbye).await.unwrap();
14575        assert!(rx.try_recv().is_err());
14576        assert!(registry.get_module("aft").unwrap().is_none());
14577
14578        router
14579            .route_for_connection(&ctx, channel_request(channel, 13))
14580            .await
14581            .unwrap();
14582        let error_frame = rx.recv().await.unwrap();
14583        assert_eq!(error_frame.header.ty, FrameType::Error);
14584        assert_eq!(error_frame.header.channel, channel);
14585        let captured = crate::router::test_log::captured_logs(&logs);
14586        assert_eq!(
14587            captured
14588                .lines()
14589                .filter(|line| {
14590                    line.contains("module_id=aft")
14591                        && line.contains("reason=explicit_goodbye")
14592                        && line.contains("module registration ended")
14593                })
14594                .count(),
14595            1,
14596            "unexpected GOODBYE registry log: {captured}"
14597        );
14598    }
14599
14600    #[tokio::test]
14601    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14602        let registry = Arc::new(Registry::default());
14603        let handler = ControlHandler::new(registry).with_capability_config(
14604            [("prov".to_string(), true), ("cons".to_string(), true)],
14605            BTreeMap::new(),
14606        );
14607        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14608        register_capability_manifest(
14609            &handler,
14610            &provider_ctx,
14611            &mut provider_rx,
14612            capability_manifest("prov", &["thing/v1"], &[]),
14613            1,
14614        )
14615        .await;
14616        let mut consumer = capability_manifest("cons", &[], &[]);
14617        consumer.capabilities.as_mut().unwrap().requires.push(
14618            subc_protocol::manifest::CapabilityRequirement {
14619                capability: "thing/v1".to_string(),
14620                need: subc_protocol::manifest::CapabilityNeed::Required,
14621            },
14622        );
14623        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14624        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14625        assert_eq!(
14626            handler.capability_evaluator.verdict("cons", "thing/v1"),
14627            Some(CapabilityVerdict::Provided)
14628        );
14629        let (mut client_rx, _) = open_route_for_capability_test(
14630            &handler,
14631            &provider_ctx,
14632            &mut provider_rx,
14633            703,
14634            3,
14635            "prov",
14636            None,
14637        )
14638        .await;
14639        let goodbye =
14640            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14641        handler
14642            .handle_control_frame(&provider_ctx, goodbye)
14643            .await
14644            .unwrap();
14645        assert_eq!(
14646            handler.capability_evaluator.verdict("cons", "thing/v1"),
14647            Some(CapabilityVerdict::NeverProvided)
14648        );
14649        let closed = client_rx
14650            .try_recv()
14651            .expect("GOODBYE pushes route.closed before route GOODBYE");
14652        assert!(
14653            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14654            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14655        );
14656        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14657        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14658    }
14659
14660    struct RouteClosedFixture {
14661        handler: ControlHandler,
14662        #[cfg(unix)]
14663        supervisor: Supervisor,
14664        module_ctx: RouteCtx,
14665        #[cfg(unix)]
14666        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14667        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14668        route: Arc<crate::forwarding::RouteBinding>,
14669    }
14670
14671    async fn route_closed_fixture() -> RouteClosedFixture {
14672        let registry = Arc::new(Registry::default());
14673        let handler = ControlHandler::new(Arc::clone(&registry));
14674        #[cfg(unix)]
14675        let supervisor = Supervisor::new(registry, RestartPolicy::default())
14676            .with_forwarding(Arc::clone(&handler.forwarding));
14677        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14678        register_capability_manifest(
14679            &handler,
14680            &module_ctx,
14681            &mut module_rx,
14682            capability_manifest("prov", &[], &[]),
14683            1,
14684        )
14685        .await;
14686        let (client_rx, bind) = open_route_for_capability_test(
14687            &handler,
14688            &module_ctx,
14689            &mut module_rx,
14690            702,
14691            2,
14692            "prov",
14693            None,
14694        )
14695        .await;
14696        let ModuleControlRequest::RouteBind {
14697            route_channel: channel,
14698            epoch: route_epoch,
14699            ..
14700        } = bind
14701        else {
14702            panic!("expected route.bind");
14703        };
14704        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14705            handler
14706                .forwarding
14707                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14708                .unwrap()
14709        else {
14710            panic!("the acknowledged route must be bound");
14711        };
14712        RouteClosedFixture {
14713            handler,
14714            #[cfg(unix)]
14715            supervisor,
14716            module_ctx,
14717            #[cfg(unix)]
14718            module_rx,
14719            client_rx,
14720            route,
14721        }
14722    }
14723
14724    impl RouteClosedFixture {
14725        #[cfg(unix)]
14726        async fn cut(&mut self) {
14727            let notices = async {
14728                let mut draining = self.module_rx.recv().await.unwrap();
14729                assert!(matches!(
14730                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14731                        &draining.body
14732                    )
14733                    .unwrap(),
14734                    subc_protocol::session::ModuleControlCommand::Draining {
14735                        reason: RouteCloseReason::Restart,
14736                        ..
14737                    }
14738                ));
14739                draining.flushed.take().unwrap().send(()).unwrap();
14740                let mut closing = self.client_rx.recv().await.unwrap();
14741                assert!(matches!(
14742                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14743                    ClientControlPush::RouteClosing {
14744                        reason: RouteCloseReason::Restart,
14745                        ..
14746                    }
14747                ));
14748                closing.flushed.take().unwrap().send(()).unwrap();
14749            };
14750            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14751            result.unwrap();
14752        }
14753
14754        fn disconnect(&mut self) -> ClientControlPush {
14755            self.handler
14756                .cleanup_connection(self.module_ctx.connection_id)
14757                .unwrap();
14758            let closed = self.client_rx.try_recv().unwrap();
14759            assert_eq!(closed.header.ty, FrameType::Push);
14760            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14761            assert!(matches!(
14762                &push,
14763                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14764                    if channels == &[self.route.client_channel]
14765            ));
14766            assert_eq!(
14767                self.client_rx.try_recv().unwrap().header.ty,
14768                FrameType::Goodbye
14769            );
14770            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14771            push
14772        }
14773    }
14774
14775    #[cfg(unix)]
14776    #[tokio::test(start_paused = true)]
14777    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14778        let mut fixture = route_closed_fixture().await;
14779        fixture.cut().await;
14780        assert!(matches!(
14781            fixture.disconnect(),
14782            ClientControlPush::RouteClosed {
14783                reason: RouteCloseReason::Restart,
14784                drained: true,
14785                ..
14786            }
14787        ));
14788    }
14789
14790    #[cfg(unix)]
14791    #[tokio::test(start_paused = true)]
14792    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14793        let mut fixture = route_closed_fixture().await;
14794        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14795        let started = tokio::time::Instant::now();
14796        fixture.cut().await;
14797        assert!(started.elapsed() >= Duration::from_secs(2));
14798        assert_eq!(fixture.route.flow.in_flight(), 1);
14799        assert!(matches!(
14800            fixture.disconnect(),
14801            ClientControlPush::RouteClosed {
14802                reason: RouteCloseReason::Restart,
14803                drained: false,
14804                ..
14805            }
14806        ));
14807    }
14808
14809    #[tokio::test]
14810    async fn module_connection_crash_reports_crash_and_not_drained() {
14811        let mut fixture = route_closed_fixture().await;
14812        // The server calls cleanup_connection on module EOF. Without a planned
14813        // daemon shutdown, that disconnect must remain a crash with no prior
14814        // route.closing notice, even when the route has no in-flight calls.
14815        assert!(matches!(
14816            fixture.disconnect(),
14817            ClientControlPush::RouteClosed {
14818                reason: RouteCloseReason::Crash,
14819                drained: false,
14820                ..
14821            }
14822        ));
14823    }
14824
14825    #[tokio::test]
14826    async fn dropping_router_connection_releases_registration() {
14827        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14828        let registry = Arc::new(Registry::default());
14829        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14830        let router = Router::with_control_handler(Arc::clone(&control));
14831        let connection = router.begin_connection();
14832        let connection_id = connection.id();
14833        let (ctx, mut rx) = route_ctx(connection_id);
14834
14835        router
14836            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14837            .await
14838            .unwrap();
14839        let response = rx.recv().await.unwrap();
14840        let ack = parse_ack(&response);
14841        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14842        assert!(registry.get_module("aft").unwrap().is_some());
14843
14844        drop(connection);
14845
14846        assert!(registry.get_module("aft").unwrap().is_none());
14847        assert_eq!(registry.active_registration_count().unwrap(), 0);
14848
14849        control
14850            .cleanup_connection(ConnectionId::new(u64::MAX))
14851            .unwrap();
14852        let captured = crate::router::test_log::captured_logs(&logs);
14853        println!("captured registry lifecycle logs:\n{captured}");
14854        let events: Vec<_> = captured
14855            .lines()
14856            .filter(|line| {
14857                line.contains("module registered module_id=aft ")
14858                    || line.contains("module registration ended")
14859            })
14860            .collect();
14861        assert_eq!(
14862            events.len(),
14863            2,
14864            "unexpected registry lifecycle logs: {captured}"
14865        );
14866        assert!(events[0].contains("module registered module_id=aft "));
14867        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14868        assert!(events[1].contains(&format!(
14869            "module_id=aft connection_id={}",
14870            connection_id.get()
14871        )));
14872        assert!(events[1].contains("reason=connection_closed"));
14873        assert!(events[1].contains("module registration ended"));
14874        assert_eq!(
14875            captured
14876                .lines()
14877                .filter(|line| line.contains("module registered module_id=aft "))
14878                .count(),
14879            1,
14880            "legacy registration admission line must appear once: {captured}"
14881        );
14882    }
14883
14884    #[tokio::test]
14885    async fn hello_registration_keeps_legacy_module_registered_line_once() {
14886        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14887        let registry = Arc::new(Registry::default());
14888        let control = ControlHandler::new(Arc::clone(&registry));
14889        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14890        hello_via_sink(
14891            &control,
14892            &ctx,
14893            &mut rx,
14894            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14895        )
14896        .await;
14897
14898        let captured = crate::router::test_log::captured_logs(&logs);
14899        let admissions: Vec<_> = captured
14900            .lines()
14901            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14902            .collect();
14903        assert_eq!(
14904            admissions.len(),
14905            1,
14906            "legacy admission line must remain exactly once: {captured}"
14907        );
14908        assert!(admissions[0].contains("routable_provider=true"));
14909        assert!(admissions[0].contains("connection_id=777"));
14910    }
14911
14912    fn capability_manifest(
14913        module_id: &str,
14914        provides: &[&str],
14915        must_never_reach: &[&str],
14916    ) -> ModuleManifest {
14917        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14918        manifest.capabilities = Some(CapabilityDeclarations {
14919            provides: provides
14920                .iter()
14921                .map(|capability| (*capability).to_string())
14922                .collect(),
14923            requires: Vec::new(),
14924            must_never_reach: must_never_reach
14925                .iter()
14926                .map(|capability| (*capability).to_string())
14927                .collect(),
14928        });
14929        manifest
14930    }
14931
14932    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14933        Frame::build(
14934            FrameType::Hello,
14935            control_flags(),
14936            0,
14937            0,
14938            corr,
14939            serde_json::to_vec(&ModuleHelloBody {
14940                protocol_ver: manifest.protocol_ver,
14941                manifest,
14942                control_ops: None,
14943                launch_nonce: None,
14944            })
14945            .expect("capability test HELLO serializes"),
14946        )
14947        .expect("capability test HELLO frame builds")
14948    }
14949
14950    fn catalog_update_with_capabilities_frame(
14951        corr: u64,
14952        capabilities: CapabilityDeclarations,
14953    ) -> Frame {
14954        Frame::build(
14955            FrameType::Request,
14956            control_flags(),
14957            0,
14958            0,
14959            corr,
14960            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14961                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14962                capabilities: Some(capabilities),
14963                ready: None,
14964            })
14965            .expect("capability catalog.update serializes"),
14966        )
14967        .expect("capability catalog.update frame builds")
14968    }
14969
14970    async fn register_capability_manifest(
14971        handler: &ControlHandler,
14972        ctx: &RouteCtx,
14973        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14974        manifest: ModuleManifest,
14975        corr: u64,
14976    ) {
14977        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14978    }
14979
14980    async fn open_route_for_capability_test(
14981        handler: &ControlHandler,
14982        target_ctx: &RouteCtx,
14983        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14984        client_connection_id: u64,
14985        corr: u64,
14986        target_module_id: &str,
14987        consumer_identity: Option<ConsumerIdentity>,
14988    ) -> (
14989        mpsc::Receiver<crate::router::OutboundFrame>,
14990        ModuleControlRequest,
14991    ) {
14992        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14993        let route_handler = handler.clone();
14994        let target_module_id = target_module_id.to_string();
14995        let route_task = tokio::spawn(async move {
14996            route_handler
14997                .handle_control_frame(
14998                    &client_ctx,
14999                    route_open_frame_with_admission_facts(
15000                        corr,
15001                        &target_module_id,
15002                        unique_project_root("admission-facts"),
15003                        consumer_identity,
15004                        None,
15005                    ),
15006                )
15007                .await
15008                .expect("capability test route.open succeeds")
15009        });
15010        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15011            .await
15012            .expect("capability test route.open must reach route.bind")
15013            .expect("target control receiver stays open");
15014        let bind_request: ModuleControlRequest =
15015            serde_json::from_slice(&bind.body).expect("route.bind decodes");
15016        handler
15017            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15018            .await
15019            .expect("capability test route.bind ACK succeeds");
15020        assert!(route_task.await.expect("route.open task joins").is_empty());
15021        let opened = client_rx
15022            .recv()
15023            .await
15024            .expect("successful route.open publishes a response");
15025        assert!(matches!(
15026            serde_json::from_slice::<ClientControlResponse>(&opened.body),
15027            Ok(ClientControlResponse::RouteOpen { .. })
15028        ));
15029        (client_rx, bind_request)
15030    }
15031
15032    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15033        assert_eq!(frame.header.ty, FrameType::Push);
15034        assert_eq!(frame.header.channel, 0);
15035        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15036            .expect("route.closed control push decodes");
15037        let ClientControlPush::RouteClosed { channels, .. } = &push else {
15038            panic!("expected route.closed");
15039        };
15040        assert_eq!(channels.len(), 1, "exactly one violating route closed");
15041        let closed_channel = channels[0];
15042        let channels = channels.clone();
15043        assert_eq!(
15044            push,
15045            ClientControlPush::RouteClosed {
15046                module_id: target_module_id.to_string(),
15047                channels,
15048                reason: RouteCloseReason::CapabilityDenied,
15049                drained: false,
15050                abandoned: 0,
15051                excluded_subscriptions: 0,
15052                terminal: Some(false),
15053            }
15054        );
15055        closed_channel
15056    }
15057
15058    #[tokio::test]
15059    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15060        let registry = Arc::new(Registry::default());
15061        let forwarding = Arc::new(ForwardingTable::default());
15062        let supervisor = SupervisorHandle::new();
15063        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15064        let handler =
15065            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15066                .with_supervisor(supervisor);
15067        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15068        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15069        register_capability_manifest(
15070            &handler,
15071            &target_ctx,
15072            &mut target_rx,
15073            capability_manifest("target", &["credentials-provider/v1"], &[]),
15074            1,
15075        )
15076        .await;
15077        register_capability_manifest(
15078            &handler,
15079            &opener_ctx,
15080            &mut opener_rx,
15081            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15082            2,
15083        )
15084        .await;
15085
15086        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15087        let replies = handler
15088            .handle_control_frame(
15089                &client_ctx,
15090                route_open_frame_with_admission_facts(
15091                    3,
15092                    "target",
15093                    unique_project_root("admission-facts"),
15094                    Some(ConsumerIdentity {
15095                        module_id: "opener".to_string(),
15096                        launch_nonce: "opener-nonce".to_string(),
15097                    }),
15098                    None,
15099                ),
15100            )
15101            .await
15102            .expect("denied route.open returns a typed frame");
15103        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15104        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15105        assert!(
15106            target_rx.try_recv().is_err(),
15107            "forbidden route.open must not relay route.bind"
15108        );
15109    }
15110
15111    #[tokio::test]
15112    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15113        let registry = Arc::new(Registry::default());
15114        let forwarding = Arc::new(ForwardingTable::default());
15115        let supervisor = SupervisorHandle::new();
15116        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15117        let handler =
15118            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15119                .with_supervisor(supervisor);
15120        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15121        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15122        register_capability_manifest(
15123            &handler,
15124            &target_ctx,
15125            &mut target_rx,
15126            capability_manifest("target", &["credentials-provider/v1"], &[]),
15127            1,
15128        )
15129        .await;
15130        register_capability_manifest(
15131            &handler,
15132            &old_opener_ctx,
15133            &mut old_opener_rx,
15134            capability_manifest("opener", &[], &[]),
15135            2,
15136        )
15137        .await;
15138        let (mut client_rx, _) = open_route_for_capability_test(
15139            &handler,
15140            &target_ctx,
15141            &mut target_rx,
15142            712,
15143            3,
15144            "target",
15145            Some(ConsumerIdentity {
15146                module_id: "opener".to_string(),
15147                launch_nonce: "opener-nonce".to_string(),
15148            }),
15149        )
15150        .await;
15151        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15152
15153        handler
15154            .cleanup_connection(old_opener_ctx.connection_id)
15155            .expect("old opener registration cleans up");
15156        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15157        register_capability_manifest(
15158            &handler,
15159            &new_opener_ctx,
15160            &mut new_opener_rx,
15161            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15162            4,
15163        )
15164        .await;
15165
15166        let closed_channel = assert_capability_denied_push(
15167            client_rx
15168                .try_recv()
15169                .expect("HELLO deny addition must emit route.closed")
15170                .frame,
15171            "target",
15172        );
15173        let goodbye = client_rx
15174            .try_recv()
15175            .expect("HELLO deny addition must send the client a route GOODBYE")
15176            .frame;
15177        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15178        assert_eq!(goodbye.header.channel, closed_channel);
15179        assert_ne!(goodbye.header.channel, 0);
15180        assert_ne!(goodbye.header.epoch, 0);
15181        assert!(client_rx.try_recv().is_err());
15182        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15183        assert!(matches!(
15184            target_rx.try_recv(),
15185            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15186        ));
15187    }
15188
15189    #[tokio::test]
15190    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15191        let registry = Arc::new(Registry::default());
15192        let forwarding = Arc::new(ForwardingTable::default());
15193        let supervisor = SupervisorHandle::new();
15194        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15195        let handler =
15196            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15197                .with_supervisor(supervisor);
15198        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15199        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15200        register_capability_manifest(
15201            &handler,
15202            &target_ctx,
15203            &mut target_rx,
15204            capability_manifest("target", &[], &[]),
15205            1,
15206        )
15207        .await;
15208        register_capability_manifest(
15209            &handler,
15210            &opener_ctx,
15211            &mut opener_rx,
15212            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15213            2,
15214        )
15215        .await;
15216        let (mut client_rx, _) = open_route_for_capability_test(
15217            &handler,
15218            &target_ctx,
15219            &mut target_rx,
15220            722,
15221            3,
15222            "target",
15223            Some(ConsumerIdentity {
15224                module_id: "opener".to_string(),
15225                launch_nonce: "opener-nonce".to_string(),
15226            }),
15227        )
15228        .await;
15229        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15230
15231        let replies = handler
15232            .handle_control_frame(
15233                &target_ctx,
15234                catalog_update_with_capabilities_frame(
15235                    4,
15236                    CapabilityDeclarations {
15237                        provides: vec!["credentials-provider/v1".to_string()],
15238                        requires: Vec::new(),
15239                        must_never_reach: Vec::new(),
15240                    },
15241                ),
15242            )
15243            .await
15244            .expect("claim catalog.update succeeds");
15245        assert!(matches!(
15246            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15247            Ok(ModuleControlResponseToModule::CatalogUpdate {})
15248        ));
15249        let closed_channel = assert_capability_denied_push(
15250            client_rx
15251                .try_recv()
15252                .expect("claim addition must emit route.closed")
15253                .frame,
15254            "target",
15255        );
15256        let goodbye = client_rx
15257            .try_recv()
15258            .expect("claim addition must send the client a route GOODBYE")
15259            .frame;
15260        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15261        assert_eq!(goodbye.header.channel, closed_channel);
15262        assert_ne!(goodbye.header.channel, 0);
15263        assert_ne!(goodbye.header.epoch, 0);
15264        assert!(client_rx.try_recv().is_err());
15265        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15266        assert!(matches!(
15267            target_rx.try_recv(),
15268            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15269        ));
15270    }
15271
15272    #[tokio::test]
15273    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15274        let registry = Arc::new(Registry::default());
15275        let forwarding = Arc::new(ForwardingTable::default());
15276        let supervisor = SupervisorHandle::new();
15277        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15278        let handler =
15279            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15280                .with_supervisor(supervisor);
15281        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15282        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15283        register_capability_manifest(
15284            &handler,
15285            &target_ctx,
15286            &mut target_rx,
15287            capability_manifest("target", &["credentials-provider/v1"], &[]),
15288            1,
15289        )
15290        .await;
15291        register_capability_manifest(
15292            &handler,
15293            &opener_ctx,
15294            &mut opener_rx,
15295            capability_manifest("opener", &[], &[]),
15296            2,
15297        )
15298        .await;
15299        let (mut client_rx, _) = open_route_for_capability_test(
15300            &handler,
15301            &target_ctx,
15302            &mut target_rx,
15303            732,
15304            3,
15305            "target",
15306            Some(ConsumerIdentity {
15307                module_id: "opener".to_string(),
15308                launch_nonce: "opener-nonce".to_string(),
15309            }),
15310        )
15311        .await;
15312        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15313
15314        handler
15315            .handle_control_frame(
15316                &target_ctx,
15317                catalog_update_with_capabilities_frame(
15318                    4,
15319                    CapabilityDeclarations {
15320                        provides: Vec::new(),
15321                        requires: Vec::new(),
15322                        must_never_reach: Vec::new(),
15323                    },
15324                ),
15325            )
15326            .await
15327            .expect("claim removal catalog.update succeeds");
15328        assert_eq!(
15329            forwarding.active_binding_count().unwrap(),
15330            1,
15331            "removing an attested target claim must leave the route census unchanged"
15332        );
15333        assert!(
15334            client_rx.try_recv().is_err(),
15335            "claim removal must not emit route.closed or a client route GOODBYE"
15336        );
15337        assert!(
15338            target_rx.try_recv().is_err(),
15339            "claim removal must not send the target a route GOODBYE"
15340        );
15341    }
15342
15343    /// A direct client may open a route to a denied capability provider; this
15344    /// policy applies only to attested supervised module origins, not to direct clients.
15345    #[tokio::test]
15346    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15347        let registry = Arc::new(Registry::default());
15348        let forwarding = Arc::new(ForwardingTable::default());
15349        let supervisor = SupervisorHandle::new();
15350        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15351        let handler =
15352            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15353                .with_supervisor(supervisor);
15354        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15355        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15356        register_capability_manifest(
15357            &handler,
15358            &target_ctx,
15359            &mut target_rx,
15360            capability_manifest("target", &["credentials-provider/v1"], &[]),
15361            1,
15362        )
15363        .await;
15364        register_capability_manifest(
15365            &handler,
15366            &opener_ctx,
15367            &mut opener_rx,
15368            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15369            2,
15370        )
15371        .await;
15372
15373        let (_client_rx, bind) = open_route_for_capability_test(
15374            &handler,
15375            &target_ctx,
15376            &mut target_rx,
15377            742,
15378            3,
15379            "target",
15380            None,
15381        )
15382        .await;
15383        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15384            panic!("direct scope-honesty route must bind");
15385        };
15386        assert_eq!(principal, Some(Principal::Direct));
15387        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15388    }
15389
15390    /// A module that denies a capability receives no self-route exemption when it
15391    /// also attestedly provides that capability.
15392    #[tokio::test]
15393    async fn must_never_reach_self_route_is_capability_forbidden() {
15394        let registry = Arc::new(Registry::default());
15395        let forwarding = Arc::new(ForwardingTable::default());
15396        let supervisor = SupervisorHandle::new();
15397        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15398        let handler =
15399            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15400                .with_supervisor(supervisor);
15401        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15402        register_capability_manifest(
15403            &handler,
15404            &self_ctx,
15405            &mut self_rx,
15406            capability_manifest(
15407                "self-provider",
15408                &["credentials-provider/v1"],
15409                &["credentials-provider/v1"],
15410            ),
15411            1,
15412        )
15413        .await;
15414
15415        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15416        let replies = handler
15417            .handle_control_frame(
15418                &client_ctx,
15419                route_open_frame_with_admission_facts(
15420                    2,
15421                    "self-provider",
15422                    unique_project_root("admission-facts"),
15423                    Some(ConsumerIdentity {
15424                        module_id: "self-provider".to_string(),
15425                        launch_nonce: "self-nonce".to_string(),
15426                    }),
15427                    None,
15428                ),
15429            )
15430            .await
15431            .expect("self-route refusal returns a typed frame");
15432        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15433        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15434        assert!(
15435            self_rx.try_recv().is_err(),
15436            "self denial must not relay route.bind"
15437        );
15438    }
15439
15440    #[test]
15441    fn unsupported_channel_zero_frame_returns_error() {
15442        let handler = ControlHandler::default();
15443        let request = Frame::build(
15444            FrameType::Request,
15445            control_flags(),
15446            0,
15447            0,
15448            21,
15449            b"opaque".to_vec(),
15450        )
15451        .unwrap();
15452
15453        let response = handler
15454            .handle_control(ConnectionId::new(1), request)
15455            .unwrap();
15456
15457        assert_eq!(response[0].header.ty, FrameType::Error);
15458        assert_eq!(
15459            parse_error(&response[0])["code"],
15460            "unsupported_control_frame"
15461        );
15462    }
15463
15464    /// Blue/green swap at the control-plane boundary. The supervisor that opens
15465    /// a swap is not wired yet, so the candidate is registered here directly
15466    /// into the registry and forwarding candidate slots, the way the swap's
15467    /// HELLO admission will.
15468    mod swap {
15469        use super::*;
15470
15471        const INCUMBENT: ConnectionId = ConnectionId::new(30);
15472        const CANDIDATE: ConnectionId = ConnectionId::new(40);
15473
15474        struct Swap {
15475            registry: Arc<Registry>,
15476            forwarding: Arc<ForwardingTable>,
15477            handler: ControlHandler,
15478            incumbent_ctx: RouteCtx,
15479            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15480            candidate_ctx: RouteCtx,
15481            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15482        }
15483
15484        async fn swap_with_incumbent() -> Swap {
15485            let registry = Arc::new(Registry::default());
15486            let forwarding = Arc::new(ForwardingTable::default());
15487            let handler =
15488                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15489            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15490            hello_via_sink(
15491                &handler,
15492                &incumbent_ctx,
15493                &mut incumbent_rx,
15494                hello_frame("aft", PROTOCOL_VERSION, 7),
15495            )
15496            .await;
15497            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15498            Swap {
15499                registry,
15500                forwarding,
15501                handler,
15502                incumbent_ctx,
15503                incumbent_rx,
15504                candidate_ctx,
15505                candidate_rx,
15506            }
15507        }
15508
15509        fn register_candidate(swap: &Swap, ready: Option<bool>) {
15510            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15511            candidate_manifest.ready = ready;
15512            let registration = swap
15513                .registry
15514                .register_candidate_with_control_ops(
15515                    candidate_manifest,
15516                    PROTOCOL_VERSION,
15517                    CANDIDATE,
15518                    module_baseline_control_ops(),
15519                )
15520                .unwrap();
15521            swap.forwarding
15522                .register_candidate_module_connection(
15523                    CANDIDATE,
15524                    "aft".to_string(),
15525                    PROTOCOL_VERSION,
15526                    manifest_concurrency(&registration.manifest),
15527                    swap.candidate_ctx.egress.clone(),
15528                )
15529                .unwrap();
15530        }
15531
15532        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15533            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15534            swap.registry.promote_candidate("aft").unwrap().unwrap();
15535            cutover.incumbent.unwrap()
15536        }
15537
15538        /// A swap candidate's HELLO reserves its candidate registry entry
15539        /// before installing its candidate forwarding endpoint. The swap
15540        /// supervisor promotes a ready candidate as soon as it reads it as
15541        /// registered, and promotion cuts forwarding over to the candidate
15542        /// endpoint, so the lookup the supervisor reads
15543        /// (`Registry::published_registration`) must not return the candidate
15544        /// until that endpoint exists. Otherwise the cutover finds no candidate
15545        /// endpoint and the swap fails as if the candidate had disconnected.
15546        #[tokio::test]
15547        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15548            let mut swap = swap_with_incumbent().await;
15549            let hold = Arc::new(std::sync::Barrier::new(2));
15550            let handler = swap
15551                .handler
15552                .clone()
15553                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15554            handler
15555                .supervisor
15556                .open_swap("aft", "candidate-nonce".to_string());
15557            let hello = {
15558                let handler = handler.clone();
15559                let candidate_ctx = swap.candidate_ctx.clone();
15560                std::thread::spawn(move || {
15561                    handler
15562                        .handle_hello(
15563                            candidate_ctx.connection_id,
15564                            Some(candidate_ctx.egress.clone()),
15565                            hello_frame_with_nonce(
15566                                "aft",
15567                                PROTOCOL_VERSION,
15568                                8,
15569                                Some("candidate-nonce"),
15570                            ),
15571                        )
15572                        .unwrap()
15573                })
15574            };
15575
15576            hold.wait();
15577            // Parked: the candidate slot is claimed but not published, and
15578            // forwarding has no candidate endpoint.
15579            assert!(swap
15580                .registry
15581                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15582                .unwrap()
15583                .is_some());
15584            assert!(swap
15585                .registry
15586                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15587                .unwrap()
15588                .is_none());
15589            assert!(swap
15590                .forwarding
15591                .module_endpoint_for_connection(CANDIDATE)
15592                .unwrap()
15593                .is_none());
15594
15595            hold.wait();
15596            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15597            assert_eq!(
15598                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15599                FrameType::HelloAck
15600            );
15601            let published = swap
15602                .registry
15603                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15604                .unwrap()
15605                .expect("candidate is published once its HELLO completes");
15606            assert_eq!(published.connection_id, CANDIDATE);
15607            assert!(swap
15608                .forwarding
15609                .module_endpoint_for_connection(CANDIDATE)
15610                .unwrap()
15611                .is_some());
15612        }
15613
15614        fn keyed_total(counters: &Value, key: &str) -> u64 {
15615            counters[key]
15616                .as_object()
15617                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15618                .unwrap_or(0)
15619        }
15620
15621        #[tokio::test]
15622        async fn replacement_logs_old_end_and_new_admission_once() {
15623            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15624            let mut swap = swap_with_incumbent().await;
15625            swap.handler
15626                .supervisor
15627                .open_swap("aft", "candidate-nonce".to_string());
15628            hello_via_sink(
15629                &swap.handler,
15630                &swap.candidate_ctx,
15631                &mut swap.candidate_rx,
15632                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15633            )
15634            .await;
15635            cutover(&swap);
15636            swap.handler.cleanup_connection(INCUMBENT).unwrap();
15637
15638            let captured = crate::router::test_log::captured_logs(&logs);
15639            println!("captured replacement registry logs:\n{captured}");
15640            let new_admissions: Vec<_> = captured
15641                .lines()
15642                .filter(|line| {
15643                    line.contains("connection_id=40")
15644                        && line.contains("swap candidate registered; not routable until cutover")
15645                })
15646                .collect();
15647            assert_eq!(
15648                new_admissions.len(),
15649                1,
15650                "unexpected admission logs: {captured}"
15651            );
15652            assert!(new_admissions[0].contains("module_id=aft"));
15653            assert!(new_admissions[0].contains("connection_id=40"));
15654            assert!(
15655                new_admissions[0].contains("swap candidate registered; not routable until cutover")
15656            );
15657            assert!(new_admissions[0].contains("ready=true"));
15658
15659            let old_admissions: Vec<_> = captured
15660                .lines()
15661                .filter(|line| {
15662                    line.contains("module registered module_id=aft ")
15663                        && line.contains("connection_id=30")
15664                })
15665                .collect();
15666            assert_eq!(
15667                old_admissions.len(),
15668                1,
15669                "unexpected incumbent admission: {captured}"
15670            );
15671
15672            let old_ends: Vec<_> = captured
15673                .lines()
15674                .filter(|line| {
15675                    line.contains("connection_id=30") && line.contains("module registration ended")
15676                })
15677                .collect();
15678            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15679            assert!(old_ends[0].contains("module_id=aft"));
15680            assert!(old_ends[0].contains("connection_id=30"));
15681            assert!(old_ends[0].contains("reason=replaced"));
15682            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15683            assert_eq!(
15684                captured
15685                    .lines()
15686                    .filter(|line| line.contains("module registration promoted"))
15687                    .count(),
15688                1,
15689                "unexpected promotion logs: {captured}"
15690            );
15691        }
15692
15693        /// An ack from the incumbent for a bind it was sent before cutover,
15694        /// arriving before the incumbent is drained. The incumbent is the live
15695        /// connection carrying every other client's routes, so the ack must
15696        /// not end it: the waiting client is told to retry, the reservation is
15697        /// given back, and the incumbent is told to drop just that binding.
15698        #[tokio::test]
15699        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15700            let mut swap = swap_with_incumbent().await;
15701            let handler = swap.handler.clone();
15702
15703            // A co-tenant route, bound on the incumbent before the swap.
15704            let cotenant = ConnectionId::new(31);
15705            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15706            let (cotenant_task, cotenant_bind) = relay_route_open(
15707                &handler,
15708                cotenant,
15709                &cotenant_ctx.egress,
15710                &mut swap.incumbent_rx,
15711                100,
15712                "aft",
15713                "swap-cotenant",
15714            )
15715            .await;
15716            handler
15717                .handle_control_frame(
15718                    &swap.incumbent_ctx,
15719                    route_bind_ack(cotenant_bind.header.corr),
15720                )
15721                .await
15722                .unwrap();
15723            assert!(cotenant_task.await.unwrap().is_empty());
15724            let (cotenant_channel, cotenant_epoch) =
15725                published_route(&cotenant_rx.recv().await.unwrap());
15726
15727            // A second route.open, relayed to the incumbent and not yet acked.
15728            let caller = ConnectionId::new(32);
15729            let (caller_ctx, mut caller_rx) = route_ctx(caller);
15730            let (caller_task, caller_bind) = relay_route_open(
15731                &handler,
15732                caller,
15733                &caller_ctx.egress,
15734                &mut swap.incumbent_rx,
15735                101,
15736                "aft",
15737                "swap-caller",
15738            )
15739            .await;
15740            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15741
15742            register_candidate(&swap, None);
15743            cutover(&swap);
15744
15745            // The incumbent acks after promotion and before any drain.
15746            let ack = handler
15747                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15748                .await;
15749            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15750            if module_loop_error.is_some() {
15751                // What the connection loop does with an untranslated router
15752                // error: end the connection, releasing every route on it.
15753                handler.cleanup_connection(INCUMBENT).unwrap();
15754            }
15755
15756            // 1. The incumbent's other routes survive.
15757            assert!(
15758                cotenant_rx.try_recv().is_err(),
15759                "the co-tenant route on the incumbent was torn down by one late ack: \
15760                 {module_loop_error:?}"
15761            );
15762            assert!(matches!(
15763                swap.forwarding
15764                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15765                    .unwrap(),
15766                DataRoute::Client(DataRouteState::Bound(_))
15767            ));
15768            assert_eq!(module_loop_error, None);
15769            assert!(swap
15770                .registry
15771                .get_module_by_connection(INCUMBENT)
15772                .unwrap()
15773                .is_some());
15774
15775            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
15776            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15777                .await
15778                .expect("the incumbent is told to drop the abandoned binding")
15779                .unwrap()
15780                .frame;
15781            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15782            assert_eq!(goodbye.header.channel, abandoned_channel);
15783            assert_eq!(goodbye.header.epoch, abandoned_epoch);
15784            assert!(swap.incumbent_rx.try_recv().is_err());
15785
15786            // 3. The waiting client gets a retryable refusal and no route.
15787            let response = caller_task.await.unwrap();
15788            assert_eq!(response.len(), 1);
15789            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15790            assert!(caller_rx.try_recv().is_err());
15791
15792            // 4. The reservation pair is given back, and the pending bind
15793            //    settled exactly once: one accepted open (the co-tenant) and one
15794            //    refused open (the caller), nothing counted twice.
15795            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15796            let counters = handler.counters().snapshot();
15797            assert_eq!(
15798                keyed_total(&counters, "route_open_accepted_by_principal"),
15799                1
15800            );
15801            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15802            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15803        }
15804
15805        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
15806        /// id would resolve to the promoted candidate and every new route.open
15807        /// would be refused as reloading, leaving neither process routable.
15808        #[tokio::test]
15809        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15810            let mut swap = swap_with_incumbent().await;
15811            register_candidate(&swap, None);
15812            let incumbent = cutover(&swap);
15813            swap.forwarding
15814                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15815                .unwrap()
15816                .expect("the incumbent is still registered");
15817
15818            let client = ConnectionId::new(33);
15819            let (client_ctx, mut client_rx) = route_ctx(client);
15820            let route_handler = swap.handler.clone();
15821            let open_ctx = RouteCtx {
15822                connection_id: client,
15823                egress: client_ctx.egress.clone(),
15824            };
15825            let mut route_task = tokio::spawn(async move {
15826                route_handler
15827                    .handle_control_frame(
15828                        &open_ctx,
15829                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15830                    )
15831                    .await
15832                    .unwrap()
15833            });
15834            let bind = tokio::select! {
15835                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15836                response = &mut route_task => {
15837                    let response = response.unwrap();
15838                    panic!(
15839                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
15840                        parse_error(&response[0])["code"]
15841                    );
15842                }
15843            };
15844            swap.handler
15845                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15846                .await
15847                .unwrap();
15848            assert!(route_task.await.unwrap().is_empty());
15849            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15850            match swap
15851                .forwarding
15852                .lookup_data_route(client, channel, epoch)
15853                .unwrap()
15854            {
15855                DataRoute::Client(DataRouteState::Bound(route)) => {
15856                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15857                }
15858                other => panic!("expected a bound route on the candidate, got {other:?}"),
15859            }
15860            assert!(swap.incumbent_rx.try_recv().is_err());
15861        }
15862
15863        /// A candidate declares itself ready with `catalog.update` on its own
15864        /// connection. If the connection-keyed registry lookups searched only the
15865        /// active slot, this would answer `not_registered` and the candidate
15866        /// would never become ready.
15867        #[tokio::test]
15868        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15869            let swap = swap_with_incumbent().await;
15870            register_candidate(&swap, Some(false));
15871            let update = Frame::build(
15872                FrameType::Request,
15873                control_flags(),
15874                0,
15875                0,
15876                55,
15877                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15878                    provides: manifest("aft", PROTOCOL_VERSION).provides,
15879                    capabilities: None,
15880                    ready: Some(true),
15881                })
15882                .unwrap(),
15883            )
15884            .unwrap();
15885
15886            let replies = swap
15887                .handler
15888                .handle_control_frame(&swap.candidate_ctx, update)
15889                .await
15890                .unwrap();
15891
15892            assert_eq!(replies.len(), 1);
15893            assert_eq!(
15894                replies[0].header.ty,
15895                FrameType::Response,
15896                "candidate catalog.update was refused: {:?}",
15897                serde_json::from_slice::<Value>(&replies[0].body).ok()
15898            );
15899            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15900            assert_eq!(
15901                swap.registry
15902                    .get_module("aft")
15903                    .unwrap()
15904                    .unwrap()
15905                    .connection_id,
15906                INCUMBENT
15907            );
15908        }
15909    }
15910
15911    /// The HELLO gate while the supervisor has a swap open: only the nonce it
15912    /// minted for the candidate admits a second process, into the candidate
15913    /// slot, and that check runs ahead of the reserved-module gate.
15914    mod swap_admission {
15915        use super::*;
15916
15917        const INCUMBENT_NONCE: &str = "incumbent-nonce";
15918        const CANDIDATE_NONCE: &str = "candidate-nonce";
15919
15920        fn handler_with_incumbent(
15921            module_id: &str,
15922            reserved: bool,
15923        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15924            let registry = Arc::new(Registry::default());
15925            let supervisor = SupervisorHandle::new();
15926            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15927            if reserved {
15928                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15929            }
15930            let handler =
15931                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
15932            let incumbent = handler
15933                .handle_control(
15934                    ConnectionId::new(1),
15935                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15936                )
15937                .unwrap();
15938            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15939            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15940            (registry, supervisor, handler)
15941        }
15942
15943        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
15944        /// admits every nonce, so while a swap is open the swap gate is the only
15945        /// thing between a key-holder and the candidate slot. A nonce the
15946        /// supervisor did not mint, or none at all, is refused, and neither the
15947        /// incumbent's registration nor the candidate slot moves.
15948        #[test]
15949        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15950            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15951
15952            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15953                let replies = handler
15954                    .handle_control(
15955                        ConnectionId::new(connection),
15956                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15957                    )
15958                    .unwrap();
15959                assert_eq!(replies[0].header.ty, FrameType::Error);
15960                assert_eq!(
15961                    parse_error(&replies[0])["code"],
15962                    "swap_token_invalid",
15963                    "nonce {nonce:?}"
15964                );
15965            }
15966            assert!(registry.get_candidate("aft").unwrap().is_none());
15967            assert_eq!(
15968                registry.get_module("aft").unwrap().unwrap().connection_id,
15969                ConnectionId::new(1)
15970            );
15971
15972            // Control: the minted token is admitted, into the candidate slot,
15973            // and only once.
15974            let admitted = handler
15975                .handle_control(
15976                    ConnectionId::new(4),
15977                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15978                )
15979                .unwrap();
15980            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15981            assert_eq!(
15982                registry
15983                    .get_candidate("aft")
15984                    .unwrap()
15985                    .unwrap()
15986                    .connection_id,
15987                ConnectionId::new(4)
15988            );
15989            assert_eq!(
15990                registry.get_module("aft").unwrap().unwrap().connection_id,
15991                ConnectionId::new(1),
15992                "the candidate must not take the active slot"
15993            );
15994            let replayed = handler
15995                .handle_control(
15996                    ConnectionId::new(5),
15997                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15998                )
15999                .unwrap();
16000            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16001
16002            // The case only this gate covers: the incumbent has died mid-swap,
16003            // so its duplicate refusal is gone too, and without the gate a
16004            // key-holder would take the id's ACTIVE slot.
16005            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16006            let squatter = handler
16007                .handle_control(
16008                    ConnectionId::new(6),
16009                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16010                )
16011                .unwrap();
16012            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16013            assert!(
16014                registry.get_module("aft").unwrap().is_none(),
16015                "a squatter took the active slot of an id being swapped"
16016            );
16017        }
16018
16019        /// Design mutation arm (iii). A reserved module's candidate presents a
16020        /// nonce the reserved gate has never seen (that gate holds the
16021        /// incumbent's), so the swap gate must run first or the candidate is
16022        /// refused `reserved_module` and a reserved module can never be swapped.
16023        #[test]
16024        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16025            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16026
16027            let replies = handler
16028                .handle_control(
16029                    ConnectionId::new(2),
16030                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16031                )
16032                .unwrap();
16033
16034            assert_eq!(
16035                replies[0].header.ty,
16036                FrameType::HelloAck,
16037                "reserved candidate refused: {:?}",
16038                serde_json::from_slice::<Value>(&replies[0].body).ok()
16039            );
16040            assert_eq!(
16041                registry
16042                    .get_candidate("vault")
16043                    .unwrap()
16044                    .unwrap()
16045                    .connection_id,
16046                ConnectionId::new(2)
16047            );
16048        }
16049
16050        /// With no swap open the gate is inert: the incumbent's reserved gate
16051        /// and duplicate refusal behave exactly as before.
16052        #[test]
16053        fn without_an_open_swap_the_ordinary_gates_decide() {
16054            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16055            supervisor.close_swap("vault");
16056
16057            let candidate = handler
16058                .handle_control(
16059                    ConnectionId::new(2),
16060                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16061                )
16062                .unwrap();
16063            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16064            let duplicate = handler
16065                .handle_control(
16066                    ConnectionId::new(3),
16067                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16068                )
16069                .unwrap();
16070            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16071            assert!(registry.get_candidate("vault").unwrap().is_none());
16072        }
16073    }
16074
16075    /// `scope.sync` and `scope.describe` through the real control handler: who
16076    /// may sync is decided by the registration and launch nonce of the module
16077    /// connection, never by the request body.
16078    mod scopes {
16079        mod idle_tests {
16080            use super::*;
16081            include!("scope_idle_tests.rs");
16082        }
16083        mod route_open_timings {
16084            use super::*;
16085            include!("route_open_timing_tests.rs");
16086        }
16087        mod route_open_refusal_log {
16088            use super::*;
16089            include!("route_open_refusal_log_tests.rs");
16090        }
16091        mod activity {
16092            include!("activity_scope_tests.rs");
16093        }
16094        mod delta {
16095            include!("scope_handler_delta_tests.rs");
16096        }
16097        mod call_modes {
16098            include!("scope_call_mode_tests.rs");
16099        }
16100        #[tokio::test]
16101        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16102            let rig = rig().await;
16103            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16104                "s".into(),
16105                1,
16106                "tool".into(),
16107            ))
16108            .unwrap();
16109            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16110            let (direct, _rx) = wide_ctx(999);
16111            for (ctx, expected) in [
16112                (&rig.owner, FrameType::Response),
16113                (&direct, FrameType::Error),
16114            ] {
16115                let frame =
16116                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16117                        .unwrap();
16118                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16119                assert_eq!(replies[0].header.ty, expected);
16120                if expected == FrameType::Response {
16121                    let reply: subc_control::ScopeServedByReply =
16122                        serde_json::from_slice(&replies[0].body).unwrap();
16123                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16124                } else {
16125                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16126                }
16127            }
16128            let reply = call(
16129                &rig.handler,
16130                &rig.owner,
16131                &ModuleControlRequestFromModule::ScopeDescribe {
16132                    owner: Principal::Reserved {
16133                        module_id: OWNER.into(),
16134                    },
16135                    scope_ref: "s".into(),
16136                },
16137            )
16138            .await;
16139            assert!(matches!(
16140                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16141                ModuleControlResponseToModule::ScopeDescribe { .. }
16142            ));
16143        }
16144
16145        use subc_protocol::scope::{
16146            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16147            ScopeStatus,
16148        };
16149
16150        use super::*;
16151
16152        const OWNER: &str = "prefrontal-core";
16153
16154        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16155            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16156        }
16157
16158        async fn call(
16159            handler: &ControlHandler,
16160            ctx: &RouteCtx,
16161            request: &ModuleControlRequestFromModule,
16162        ) -> Frame {
16163            let body = serde_json::to_vec(request).unwrap();
16164            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16165            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16166            assert_eq!(replies.len(), 1, "{replies:?}");
16167            replies.pop().unwrap()
16168        }
16169
16170        async fn sync(
16171            handler: &ControlHandler,
16172            ctx: &RouteCtx,
16173            generation: u64,
16174            scopes: Vec<ScopeRecord>,
16175        ) -> Result<ModuleControlResponseToModule, String> {
16176            let reply = call(
16177                handler,
16178                ctx,
16179                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16180            )
16181            .await;
16182            match reply.header.ty {
16183                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16184                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16185            }
16186        }
16187
16188        async fn describe(
16189            handler: &ControlHandler,
16190            ctx: &RouteCtx,
16191            owner: &str,
16192            scope_ref: &str,
16193        ) -> ModuleControlResponseToModule {
16194            let reply = call(
16195                handler,
16196                ctx,
16197                &ModuleControlRequestFromModule::ScopeDescribe {
16198                    owner: Principal::Reserved {
16199                        module_id: owner.to_string(),
16200                    },
16201                    scope_ref: scope_ref.to_string(),
16202                },
16203            )
16204            .await;
16205            assert_eq!(
16206                reply.header.ty,
16207                FrameType::Response,
16208                "{:?}",
16209                parse_error(&reply)
16210            );
16211            serde_json::from_slice(&reply.body).unwrap()
16212        }
16213
16214        #[tokio::test]
16215        async fn scope_describe_never_echoes_bind_proof_or_hash() {
16216            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16217            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16218            let mut rig = rig().await;
16219            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16220                .await;
16221            let _bound = rig
16222                .bound(
16223                    Some(AFT),
16224                    PLEXUS,
16225                    Some(
16226                        rig.selector("s", Some(1))
16227                            .with_bind_proof(Some(PROOF.into())),
16228                    ),
16229                )
16230                .await;
16231            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16232            let output = serde_json::to_string(&reply).unwrap();
16233            assert!(output.contains("\"status\":\"live\""));
16234            assert!(!output.contains(PROOF));
16235            assert!(!output.contains(HASH));
16236        }
16237
16238        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
16239        async fn module(
16240            handler: &ControlHandler,
16241            connection: u64,
16242            module_id: &str,
16243            nonce: Option<&str>,
16244        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16245            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16246            hello_via_sink(
16247                handler,
16248                &ctx,
16249                &mut rx,
16250                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16251            )
16252            .await;
16253            (ctx, rx)
16254        }
16255
16256        /// `direct` and every other client connection has no registration, so
16257        /// it can neither sync nor own a scope.
16258        #[tokio::test]
16259        async fn a_client_connection_cannot_sync_or_describe() {
16260            let handler = ControlHandler::new(Arc::new(Registry::default()));
16261            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16262            for request in [
16263                ModuleControlRequestFromModule::ScopeSync {
16264                    generation: 1,
16265                    scopes: vec![head("s", 1)],
16266                },
16267                ModuleControlRequestFromModule::ScopeDescribe {
16268                    owner: Principal::Direct,
16269                    scope_ref: "s".to_string(),
16270                },
16271            ] {
16272                let reply = call(&handler, &ctx, &request).await;
16273                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16274            }
16275            assert!(
16276                !handler
16277                    .scopes
16278                    .read()
16279                    .unwrap()
16280                    .describe(
16281                        &Principal::Reserved {
16282                            module_id: OWNER.to_string()
16283                        },
16284                        "s"
16285                    )
16286                    .owner_synced
16287            );
16288        }
16289
16290        /// A module the supervisor did not spawn registers without a launch
16291        /// nonce, so it is never an owner's current launch.
16292        #[tokio::test]
16293        async fn a_module_without_a_supervised_launch_cannot_sync() {
16294            let handler = ControlHandler::new(Arc::new(Registry::default()));
16295            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16296            assert_eq!(
16297                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16298                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16299            );
16300        }
16301
16302        #[tokio::test]
16303        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16304            let supervisor = SupervisorHandle::new();
16305            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16306            let handler = ControlHandler::new(Arc::new(Registry::default()))
16307                .with_supervisor(supervisor.clone());
16308            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16309            sync(&handler, &incumbent, 1, vec![head("s", 1)])
16310                .await
16311                .expect("the current launch syncs");
16312
16313            // A swap candidate registers with the swap token and is refused
16314            // while the incumbent keeps syncing.
16315            supervisor.open_swap(OWNER, "n2".to_string());
16316            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16317            assert_eq!(
16318                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16319                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16320            );
16321            sync(&handler, &incumbent, 2, vec![head("s", 1)])
16322                .await
16323                .expect("the serving owner syncs during the swap");
16324
16325            // The swap fails and is rolled back. The candidate never held sync
16326            // authority, and still cannot sync.
16327            supervisor.close_swap(OWNER);
16328            assert_eq!(
16329                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16330                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16331            );
16332            sync(&handler, &incumbent, 3, vec![head("s", 1)])
16333                .await
16334                .expect("the serving owner syncs after the rollback");
16335            handler.cleanup_connection(candidate.connection_id).unwrap();
16336
16337            // A swap that cuts over. Promotion records the candidate's nonce as
16338            // the module's spawn nonce, which is what `set_spawn_nonce` does
16339            // here; the promoted connection then takes authority at any
16340            // generation and the superseded incumbent is refused.
16341            supervisor.open_swap(OWNER, "n3".to_string());
16342            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16343            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16344            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16345                .await
16346                .expect("the promoted launch takes authority");
16347            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16348                panic!("unexpected reply {reply:?}");
16349            };
16350            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16351            assert_eq!(
16352                sync(&handler, &incumbent, 4, Vec::new()).await,
16353                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16354            );
16355        }
16356
16357        /// Authority dies with its connection: the cleanup path releases it,
16358        /// so the owner's next connection takes it at any generation.
16359        #[tokio::test]
16360        async fn closing_the_authority_connection_frees_sync_authority() {
16361            let supervisor = SupervisorHandle::new();
16362            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16363            let handler = ControlHandler::new(Arc::new(Registry::default()))
16364                .with_supervisor(supervisor.clone());
16365            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16366            sync(&handler, &first, 10, vec![head("s", 1)])
16367                .await
16368                .unwrap();
16369            handler.cleanup_connection(first.connection_id).unwrap();
16370
16371            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16372            sync(&handler, &second, 1, vec![head("s", 1)])
16373                .await
16374                .expect("the next connection takes the released authority");
16375        }
16376
16377        #[tokio::test]
16378        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16379            let supervisor = SupervisorHandle::new();
16380            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16381            let handler =
16382                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16383            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16384            sync(&handler, &first, 10, vec![head("s", 1)])
16385                .await
16386                .unwrap();
16387            handler
16388                .handle_control_frame(
16389                    &first,
16390                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16391                )
16392                .await
16393                .unwrap();
16394            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16395            sync(&handler, &second, 1, vec![head("s", 1)])
16396                .await
16397                .expect("GOODBYE releases authority even if the old socket remains open");
16398        }
16399
16400        #[tokio::test]
16401        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16402            let registry = Arc::new(Registry::default());
16403            let supervisor_handle = SupervisorHandle::new();
16404            let supervisor =
16405                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16406                    .with_handle(supervisor_handle.clone())
16407                    .with_daemon_incarnation("incarnation-7".to_string());
16408            // Configured with enabled: false, so the supervisor lists the
16409            // module without spawning a process for it.
16410            supervisor
16411                .supervise_configured(
16412                    ModuleSpec {
16413                        module_id: OWNER.to_string(),
16414                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16415                        args: Vec::new(),
16416                        env: Vec::new(),
16417                        reserved: false,
16418                        reserved_prefixes: Vec::new(),
16419                        protocol: ModuleProtocol::Subc,
16420                        overlap: Default::default(),
16421                    },
16422                    false,
16423                )
16424                .unwrap();
16425            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16426            let handler =
16427                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
16428            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16429
16430            // Configured but not yet synced: a reader waits for the owner.
16431            let ModuleControlResponseToModule::ScopeDescribe {
16432                status,
16433                daemon_incarnation,
16434                owner_synced,
16435                owner_configured,
16436                scope,
16437                ..
16438            } = describe(&handler, &reader, OWNER, "s").await
16439            else {
16440                panic!("not a describe reply");
16441            };
16442            assert_eq!(status, ScopeStatus::NotLive);
16443            assert_eq!(daemon_incarnation, "incarnation-7");
16444            assert!(!owner_synced);
16445            assert!(owner_configured);
16446            assert!(scope.is_none());
16447
16448            // Not a supervised module: the owner will never sync, and a reader
16449            // refuses rather than waits.
16450            let ModuleControlResponseToModule::ScopeDescribe {
16451                status,
16452                owner_configured,
16453                ..
16454            } = describe(&handler, &reader, "ghost", "s").await
16455            else {
16456                panic!("not a describe reply");
16457            };
16458            assert_eq!(status, ScopeStatus::NotLive);
16459            assert!(!owner_configured);
16460
16461            // Live, with the stamp fields and the computed owner_authorized.
16462            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16463            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16464            let ModuleControlResponseToModule::ScopeDescribe {
16465                status,
16466                scope_epoch,
16467                owner_synced,
16468                scope,
16469                ..
16470            } = describe(&handler, &reader, OWNER, "s").await
16471            else {
16472                panic!("not a describe reply");
16473            };
16474            assert_eq!(status, ScopeStatus::Live);
16475            assert_eq!(scope_epoch, Some(4));
16476            assert!(owner_synced);
16477            let stamp = scope.expect("a live scope carries its stamp");
16478            assert!(
16479                stamp.owner_authorized,
16480                "prefrontal-core is the default authority"
16481            );
16482            assert_eq!(stamp.kind, ScopeKind::Head);
16483        }
16484
16485        #[tokio::test]
16486        async fn scope_authority_owners_decides_owner_authorized() {
16487            let supervisor = SupervisorHandle::new();
16488            supervisor.set_spawn_nonce("broca", "b1".to_string());
16489            let handler = ControlHandler::new(Arc::new(Registry::default()))
16490                .with_supervisor(supervisor)
16491                .with_scope_authority_owners(vec!["broca".to_string()]);
16492            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16493            let mut gated = head("s", 1);
16494            gated.attributes.agent_id = Some("agent".to_string());
16495            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16496            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16497                describe(&handler, &broca, "broca", "s").await
16498            else {
16499                panic!("not a describe reply");
16500            };
16501            assert!(scope.unwrap().owner_authorized);
16502        }
16503
16504        /// With route admission, the stamp, the commit re-check and drains in
16505        /// place, the feature is advertised: the module ops in HELLO_ACK, and
16506        /// `scopes/v1` in HELLO_ACK and `server.describe`.
16507        #[tokio::test]
16508        async fn scope_ops_and_the_scopes_capability_are_advertised() {
16509            let handler = ControlHandler::new(Arc::new(Registry::default()));
16510            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16511            let ack = hello_via_sink(
16512                &handler,
16513                &ctx,
16514                &mut rx,
16515                hello_frame("m", PROTOCOL_VERSION, 1),
16516            )
16517            .await;
16518            let ack = parse_ack(&ack);
16519            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16520                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16521            }
16522            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16523
16524            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16525            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16526            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16527            let reply = handler
16528                .handle_control_frame(&client, frame)
16529                .await
16530                .unwrap()
16531                .pop()
16532                .unwrap();
16533            let ClientControlResponse::ServerDescribe { capabilities, .. } =
16534                serde_json::from_slice(&reply.body).unwrap()
16535            else {
16536                panic!("not a server.describe reply");
16537            };
16538            assert!(
16539                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16540                "{capabilities:?}"
16541            );
16542        }
16543
16544        // ---- route admission, stamps, commit re-check and drains ----------
16545
16546        const PLEXUS: &str = "plexus";
16547        const OTHER: &str = "other";
16548        const AFT: &str = "aft";
16549        const BROCA: &str = "broca";
16550        const MAGIC: &str = "magic-context";
16551
16552        fn nonce(module_id: &str) -> String {
16553            format!("nonce-{module_id}")
16554        }
16555
16556        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16557            let (tx, rx) = mpsc::channel(64);
16558            (
16559                RouteCtx {
16560                    connection_id: ConnectionId::new(connection),
16561                    egress: FrameSink::new(tx),
16562                },
16563                rx,
16564            )
16565        }
16566
16567        /// A daemon with a configured owner (prefrontal-core) registered on its
16568        /// own module connection, two routable targets (plexus, other), and
16569        /// launch nonces minted for the modules that open routes as carriers.
16570        struct Rig {
16571            handler: ControlHandler,
16572            forwarding: Arc<ForwardingTable>,
16573            owner: RouteCtx,
16574            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16575            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16576            generation: u64,
16577            next_connection: u64,
16578            _supervisor: Supervisor,
16579        }
16580
16581        async fn rig() -> Rig {
16582            rig_with_flow_support(true).await
16583        }
16584
16585        async fn rig_with_flow_support(flow_support: bool) -> Rig {
16586            rig_with_scope_capabilities(flow_support, false).await
16587        }
16588
16589        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16590            let mut provides = Vec::new();
16591            if flow_support {
16592                provides.push("flow-scopes/v1");
16593            }
16594            if run_support {
16595                provides.push("agent-run-scopes/v1");
16596            }
16597            rig_with_target_capabilities(&provides).await
16598        }
16599
16600        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16601            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16602            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16603            // Decoder versions alone cannot admit scoped calls: only a capability
16604            // declaration promises the required run or flow behaviour.
16605            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16606            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16607            Frame::build(
16608                FrameType::Hello,
16609                control_flags(),
16610                0,
16611                0,
16612                connection,
16613                serde_json::to_vec(&body).unwrap(),
16614            )
16615            .unwrap()
16616        }
16617
16618        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16619            let registry = Arc::new(Registry::default());
16620            let forwarding = Arc::new(ForwardingTable::default());
16621            let supervisor_handle = SupervisorHandle::new();
16622            let supervisor =
16623                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16624                    .with_handle(supervisor_handle.clone());
16625            supervisor
16626                .supervise_configured(
16627                    ModuleSpec {
16628                        module_id: OWNER.to_string(),
16629                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16630                        args: Vec::new(),
16631                        env: Vec::new(),
16632                        reserved: false,
16633                        reserved_prefixes: Vec::new(),
16634                        protocol: ModuleProtocol::Subc,
16635                        overlap: Default::default(),
16636                    },
16637                    false,
16638                )
16639                .unwrap();
16640            for module_id in [OWNER, AFT, BROCA, MAGIC] {
16641                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16642            }
16643            let handler =
16644                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16645                    .with_supervisor(supervisor_handle);
16646            let (owner, mut owner_rx) = wide_ctx(1);
16647            hello_via_sink(
16648                &handler,
16649                &owner,
16650                &mut owner_rx,
16651                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16652            )
16653            .await;
16654            let mut modules = BTreeMap::new();
16655            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16656                let (ctx, mut rx) = wide_ctx(connection);
16657                let hello = hello_with_capabilities(module_id, connection, provides);
16658                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16659                modules.insert(module_id.to_string(), (ctx, rx));
16660            }
16661            Rig {
16662                handler,
16663                forwarding,
16664                owner,
16665                _owner_rx: owner_rx,
16666                modules,
16667                generation: 0,
16668                next_connection: 100,
16669                _supervisor: supervisor,
16670            }
16671        }
16672
16673        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16674            ScopeCarrier::new(Principal::Reserved {
16675                module_id: module_id.to_string(),
16676            })
16677            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16678        }
16679
16680        /// The scope most tests open under: aft carries to any module, broca
16681        /// only to plexus and other, and the owner delegates as agent-1.
16682        fn session(scope_epoch: u64) -> ScopeRecord {
16683            let mut record = head("s", scope_epoch);
16684            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16685            record.attributes.agent_id = Some("agent-1".to_string());
16686            record.attributes.delegates = true;
16687            record
16688        }
16689
16690        impl Rig {
16691            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16692                self.generation += 1;
16693                sync(&self.handler, &self.owner, self.generation, scopes)
16694                    .await
16695                    .expect("the owner's sync is accepted");
16696            }
16697
16698            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16699                ScopeSelector::new(
16700                    Principal::Reserved {
16701                        module_id: OWNER.to_string(),
16702                    },
16703                    scope_ref.to_string(),
16704                    scope_epoch,
16705                )
16706            }
16707
16708            fn open_frame(
16709                &mut self,
16710                opener: Option<&str>,
16711                target: &str,
16712                scope: Option<ScopeSelector>,
16713            ) -> (
16714                RouteCtx,
16715                mpsc::Receiver<crate::router::OutboundFrame>,
16716                Frame,
16717            ) {
16718                self.next_connection += 1;
16719                let (ctx, rx) = wide_ctx(self.next_connection);
16720                let root = unique_project_root("scoped-open");
16721                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16722                    target: RouteTarget::ToolProvider {
16723                        module_id: target.to_string(),
16724                    },
16725                    identity: BindIdentity::new(
16726                        root.path().to_path_buf(),
16727                        "unit".to_string(),
16728                        "session".to_string(),
16729                    ),
16730                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
16731                        module_id: module_id.to_string(),
16732                        launch_nonce: nonce(module_id),
16733                    }),
16734                    consumer_capabilities: None,
16735                    role_versions: None,
16736                    admission_facts: None,
16737                    scope,
16738                })
16739                .unwrap();
16740                let frame = Frame::build(
16741                    FrameType::Request,
16742                    control_flags(),
16743                    0,
16744                    0,
16745                    self.next_connection,
16746                    body,
16747                )
16748                .unwrap();
16749                (ctx, rx, frame)
16750            }
16751
16752            /// Open and expect a refusal before anything is relayed.
16753            async fn refused(
16754                &mut self,
16755                opener: Option<&str>,
16756                target: &str,
16757                scope: Option<ScopeSelector>,
16758            ) -> String {
16759                self.refusal_body(opener, target, scope).await["code"]
16760                    .as_str()
16761                    .unwrap()
16762                    .to_string()
16763            }
16764
16765            async fn refusal_body(
16766                &mut self,
16767                opener: Option<&str>,
16768                target: &str,
16769                scope: Option<ScopeSelector>,
16770            ) -> Value {
16771                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16772                let replies = tokio::time::timeout(
16773                    Duration::from_secs(2),
16774                    self.handler.handle_control_frame(&ctx, frame),
16775                )
16776                .await
16777                .expect("the open must be refused before waiting for a bind ack")
16778                .unwrap();
16779                assert_eq!(replies.len(), 1, "{replies:?}");
16780                assert_eq!(replies[0].header.ty, FrameType::Error);
16781                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16782                assert!(
16783                    module_rx.try_recv().is_err(),
16784                    "a refused open relays nothing"
16785                );
16786                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16787                parse_error(&replies[0])
16788            }
16789
16790            /// Start an open and return its task and the bind the target got.
16791            async fn relayed(
16792                &mut self,
16793                opener: Option<&str>,
16794                target: &str,
16795                scope: Option<ScopeSelector>,
16796            ) -> Relayed {
16797                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16798                let handler = self.handler.clone();
16799                let task_ctx = ctx.clone();
16800                let task = tokio::spawn(async move {
16801                    handler
16802                        .handle_control_frame(&task_ctx, frame)
16803                        .await
16804                        .unwrap()
16805                });
16806                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16807                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16808                    .await
16809                    .expect("the target receives the relayed route.bind")
16810                    .unwrap()
16811                    .frame;
16812                Relayed {
16813                    target: target.to_string(),
16814                    client: ctx,
16815                    client_rx: rx,
16816                    task,
16817                    bind,
16818                }
16819            }
16820
16821            async fn ack(&self, relayed: &Relayed) {
16822                let (module, _) = &self.modules[&relayed.target];
16823                self.handler
16824                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16825                    .await
16826                    .unwrap();
16827            }
16828
16829            /// Open, ack and return the bound route.
16830            async fn bound(
16831                &mut self,
16832                opener: Option<&str>,
16833                target: &str,
16834                scope: Option<ScopeSelector>,
16835            ) -> Bound {
16836                let relayed = self.relayed(opener, target, scope).await;
16837                self.ack(&relayed).await;
16838                let Relayed {
16839                    target,
16840                    client,
16841                    mut client_rx,
16842                    task,
16843                    bind,
16844                } = relayed;
16845                assert!(
16846                    task.await.unwrap().is_empty(),
16847                    "the open is answered by commit"
16848                );
16849                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16850                Bound {
16851                    target,
16852                    client,
16853                    client_rx,
16854                    channel,
16855                    epoch,
16856                    bind,
16857                }
16858            }
16859
16860            fn live(&self, route: &Bound) -> bool {
16861                matches!(
16862                    self.forwarding
16863                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16864                        .unwrap(),
16865                    DataRoute::Client(DataRouteState::Bound(_))
16866                )
16867            }
16868        }
16869
16870        struct Relayed {
16871            target: String,
16872            client: RouteCtx,
16873            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16874            task: tokio::task::JoinHandle<Vec<Frame>>,
16875            bind: Frame,
16876        }
16877
16878        struct Bound {
16879            target: String,
16880            client: RouteCtx,
16881            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16882            channel: u16,
16883            epoch: u32,
16884            bind: Frame,
16885        }
16886
16887        impl Bound {
16888            /// The reason of the `route.closed` this client was sent, after
16889            /// checking it also got a GOODBYE on exactly this route.
16890            fn closed_reason(&mut self) -> RouteCloseReason {
16891                let mut reason = None;
16892                let mut goodbye = false;
16893                while let Ok(outbound) = self.client_rx.try_recv() {
16894                    let frame = outbound.frame;
16895                    match frame.header.ty {
16896                        FrameType::Goodbye => {
16897                            assert_eq!(
16898                                (frame.header.channel, frame.header.epoch),
16899                                (self.channel, self.epoch)
16900                            );
16901                            goodbye = true;
16902                        }
16903                        FrameType::Push => {
16904                            let ClientControlPush::RouteClosed {
16905                                reason: r,
16906                                module_id,
16907                                ..
16908                            } = serde_json::from_slice(&frame.body).unwrap()
16909                            else {
16910                                panic!("unexpected push");
16911                            };
16912                            assert_eq!(module_id, self.target);
16913                            reason = Some(r);
16914                        }
16915                        other => panic!("unexpected frame {other:?}"),
16916                    }
16917                }
16918                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16919                reason.expect("the client is told why the route closed")
16920            }
16921
16922            fn untouched(&mut self) -> bool {
16923                self.client_rx.try_recv().is_err()
16924            }
16925
16926            fn stamp(&self) -> Option<ScopeStamp> {
16927                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16928                    ModuleControlRequest::RouteBind { scope, .. } => scope,
16929                    other => panic!("expected a route.bind, got {other:?}"),
16930                }
16931            }
16932        }
16933
16934        #[tokio::test]
16935        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16936        ) {
16937            let mut rig = rig().await;
16938            let mut record = session(1);
16939            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16940            record.child_owners = vec![Principal::Reserved {
16941                module_id: MAGIC.to_string(),
16942            }];
16943            rig.sync(vec![record]).await;
16944            let scope = || Some(rig_selector("s", Some(1)));
16945
16946            // Admitted: the owner, a bare carrier to any module, a targeted
16947            // carrier to its listed module.
16948            rig.bound(Some(OWNER), PLEXUS, scope()).await;
16949            rig.bound(Some(AFT), OTHER, scope()).await;
16950            rig.bound(Some(BROCA), PLEXUS, scope()).await;
16951
16952            // Refused scope_not_carrier: a targeted carrier to an unlisted
16953            // module, a module that is not listed at all (a child owner is not
16954            // a carrier), and a direct key-holder.
16955            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16956                assert_eq!(
16957                    rig.refused(opener, target, scope()).await,
16958                    error_codes::SCOPE_NOT_CARRIER,
16959                    "{opener:?} -> {target}"
16960                );
16961            }
16962        }
16963
16964        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16965            ScopeSelector::new(
16966                Principal::Reserved {
16967                    module_id: OWNER.to_string(),
16968                },
16969                scope_ref.to_string(),
16970                scope_epoch,
16971            )
16972        }
16973
16974        #[tokio::test]
16975        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16976            let mut rig = rig().await;
16977            rig.sync(vec![session(1)]).await;
16978            for opener in [OWNER, AFT] {
16979                assert_eq!(
16980                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16981                        .await,
16982                    error_codes::SCOPE_EPOCH_REQUIRED,
16983                    "{opener}"
16984                );
16985            }
16986        }
16987
16988        #[tokio::test]
16989        async fn admission_separates_not_synced_not_live_and_ended() {
16990            let mut rig = rig().await;
16991            // Before the configured owner's first sync: retryable.
16992            let code = rig
16993                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16994                .await;
16995            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16996            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16997
16998            // An owner that is not configured will never sync: terminal.
16999            let ghost = ScopeSelector::new(
17000                Principal::Reserved {
17001                    module_id: "ghost".to_string(),
17002                },
17003                "s".to_string(),
17004                Some(1),
17005            );
17006            assert_eq!(
17007                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17008                error_codes::SCOPE_NOT_LIVE
17009            );
17010
17011            rig.sync(vec![session(2)]).await;
17012            assert_eq!(
17013                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17014                    .await,
17015                error_codes::SCOPE_NOT_LIVE
17016            );
17017            for epoch in [1, 3] {
17018                assert_eq!(
17019                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17020                        .await,
17021                    error_codes::SCOPE_ENDED,
17022                    "epoch {epoch}"
17023                );
17024            }
17025            // Control: the live epoch is admitted.
17026            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17027                .await;
17028        }
17029
17030        #[tokio::test]
17031        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17032            let mut rig = rig().await;
17033            rig.sync(vec![session(1)]).await;
17034            let route = rig
17035                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17036                .await;
17037            let stamp = route.stamp().expect("a scoped bind carries the stamp");
17038            assert_eq!(stamp.scope_ref, "s");
17039            assert_eq!(stamp.scope_epoch, 1);
17040            assert_eq!(stamp.kind, ScopeKind::Head);
17041            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17042            assert!(stamp.attributes.delegates);
17043            assert!(stamp.owner_authorized);
17044            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17045            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17046
17047            // broca owns a scope of its own on its own module connection; it is
17048            // not in scope_authority_owners, so its stamp is not authorized.
17049            let (broca, mut broca_rx) = wide_ctx(50);
17050            hello_via_sink(
17051                &rig.handler,
17052                &broca,
17053                &mut broca_rx,
17054                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17055            )
17056            .await;
17057            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17058                .await
17059                .unwrap();
17060            let own = ScopeSelector::new(
17061                Principal::Reserved {
17062                    module_id: BROCA.to_string(),
17063                },
17064                "b".to_string(),
17065                Some(1),
17066            );
17067            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17068            assert!(!route.stamp().unwrap().owner_authorized);
17069        }
17070
17071        #[tokio::test]
17072        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17073            let mut rig = rig().await;
17074            let mut record = head("s", 1);
17075            record.carriers = vec![carrier(AFT, None)];
17076            let flow_id = "Flow:run-7/step_2!~";
17077            record.attributes.flow_id = Some(flow_id.to_string());
17078            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17079                .await
17080                .unwrap();
17081            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17082                panic!("not a sync reply");
17083            };
17084            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17085            let route = rig
17086                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17087                .await;
17088            assert!(rig.live(&route), "the stamped bind committed");
17089            let stamp = route.stamp().expect("a flow scope carries a stamp");
17090            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17091            assert_eq!(stamp.attributes.agent_id, None);
17092            assert!(!stamp.attributes.delegates);
17093            assert!(stamp.owner_authorized);
17094            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17095            assert_eq!(unscoped.stamp(), None);
17096        }
17097
17098        #[tokio::test]
17099        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17100            let mut rig = rig_with_flow_support(false).await;
17101            let mut record = session(1);
17102            record.attributes.flow_id = Some("flow:7".to_string());
17103            rig.sync(vec![record]).await;
17104            for opener in [OWNER, AFT] {
17105                let body = rig
17106                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17107                    .await;
17108                assert_eq!(body["code"], "target_flow_unsupported");
17109                let message = body["message"].as_str().unwrap();
17110                for required in [PLEXUS, "flow-scopes/v1"] {
17111                    assert!(message.contains(required), "{message}");
17112                }
17113            }
17114        }
17115
17116        #[tokio::test]
17117        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17118            let mut rig = rig_with_flow_support(true).await;
17119            let mut record = session(1);
17120            record.attributes.flow_id = Some("flow:7".to_string());
17121            rig.sync(vec![record]).await;
17122            for opener in [OWNER, AFT] {
17123                let route = rig
17124                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17125                    .await;
17126                assert!(rig.live(&route));
17127                assert_eq!(
17128                    route.stamp().unwrap().attributes.flow_id.as_deref(),
17129                    Some("flow:7")
17130                );
17131            }
17132        }
17133
17134        #[tokio::test]
17135        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17136            use std::future::Future;
17137
17138            let mut rig = rig_with_flow_support(true).await;
17139            let mut record = session(1);
17140            record.attributes.flow_id = Some("flow:7".to_string());
17141            rig.sync(vec![record]).await;
17142            let (client, mut client_rx, frame) =
17143                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17144            // Hold the route-open response permit so admission sees the first
17145            // target but relay reservation cannot capture an endpoint yet.
17146            for _ in 0..64 {
17147                client.egress.try_send(route_bind_ack(1)).unwrap();
17148            }
17149            let handler = rig.handler.clone();
17150            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17151            std::future::poll_fn(|cx| {
17152                assert!(open.as_mut().poll(cx).is_pending());
17153                std::task::Poll::Ready(())
17154            })
17155            .await;
17156            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17157
17158            let old_connection = rig.modules[PLEXUS].0.connection_id;
17159            rig.handler.cleanup_connection(old_connection).unwrap();
17160            let (replacement, mut replacement_rx) = wide_ctx(200);
17161            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17162            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17163            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17164            let hello = Frame::build(
17165                FrameType::Hello,
17166                control_flags(),
17167                0,
17168                0,
17169                200,
17170                serde_json::to_vec(&body).unwrap(),
17171            )
17172            .unwrap();
17173            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17174
17175            client_rx.try_recv().unwrap();
17176            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17177                .await
17178                .expect("the replacement is refused without waiting for a bind ack")
17179                .unwrap();
17180            assert_eq!(replies.len(), 1);
17181            let body = parse_error(&replies[0]);
17182            assert_eq!(body["code"], "target_flow_unsupported");
17183            for required in [PLEXUS, "flow-scopes/v1"] {
17184                assert!(body["message"].as_str().unwrap().contains(required));
17185            }
17186            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17187            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17188            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17189            assert!(rig
17190                .handler
17191                .registry
17192                .get_module_by_connection(replacement.connection_id)
17193                .unwrap()
17194                .is_some());
17195        }
17196
17197        #[tokio::test]
17198        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17199            use std::future::Future;
17200            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17201            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17202            const NEXT_HASH: &str =
17203                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17204            let mut rig = rig().await;
17205            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17206            record.carriers.push(ScopeCarrier::new(Principal::Direct));
17207            rig.sync(vec![record.clone()]).await;
17208            let (client, mut rx, frame) = rig.open_frame(
17209                None,
17210                PLEXUS,
17211                Some(
17212                    rig.selector("s", Some(1))
17213                        .with_bind_proof(Some(PROOF.into())),
17214                ),
17215            );
17216            // Hold egress reservation while the scope owner changes its bind-proof
17217            // hash, so a proof valid at admission must be checked again at relay.
17218            for _ in 0..64 {
17219                client.egress.try_send(route_bind_ack(1)).unwrap();
17220            }
17221            let handler = rig.handler.clone();
17222            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17223            std::future::poll_fn(|cx| {
17224                assert!(open.as_mut().poll(cx).is_pending());
17225                std::task::Poll::Ready(())
17226            })
17227            .await;
17228            record.bind_proof_sha256 = Some(NEXT_HASH.into());
17229            rig.sync(vec![record]).await;
17230            rx.try_recv().unwrap();
17231            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17232                .await
17233                .unwrap()
17234                .unwrap();
17235            assert_eq!(
17236                parse_error(&replies[0])["code"],
17237                error_codes::SCOPE_BIND_PROOF_MISMATCH
17238            );
17239            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17240            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17241            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17242        }
17243
17244        #[tokio::test]
17245        async fn scope_bind_proof_reserved_openers_ignore_proof() {
17246            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17247            let mut rig = rig().await;
17248            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17249                .await;
17250            for opener in [OWNER, AFT] {
17251                for proof in [None, Some("malformed+proof=")] {
17252                    let route = rig
17253                        .bound(
17254                            Some(opener),
17255                            PLEXUS,
17256                            Some(
17257                                rig.selector("s", Some(1))
17258                                    .with_bind_proof(proof.map(str::to_string)),
17259                            ),
17260                        )
17261                        .await;
17262                    assert!(rig.live(&route));
17263                }
17264            }
17265        }
17266
17267        #[tokio::test]
17268        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17269            use std::future::Future;
17270            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17271            const NEXT_HASH: &str =
17272                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17273            let mut rig = rig().await;
17274            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17275                .await;
17276            let (client, mut rx, frame) =
17277                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17278            for _ in 0..64 {
17279                client.egress.try_send(route_bind_ack(1)).unwrap();
17280            }
17281            let handler = rig.handler.clone();
17282            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17283            std::future::poll_fn(|cx| {
17284                assert!(open.as_mut().poll(cx).is_pending());
17285                std::task::Poll::Ready(())
17286            })
17287            .await;
17288            rig.sync(vec![
17289                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17290            ])
17291            .await;
17292            rx.try_recv().unwrap();
17293            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17294            let bind = std::future::poll_fn(|cx| {
17295                assert!(open.as_mut().poll(cx).is_pending());
17296                module_rx.poll_recv(cx)
17297            })
17298            .await
17299            .unwrap()
17300            .frame;
17301            handler
17302                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17303                .await
17304                .unwrap();
17305            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17306                .await
17307                .unwrap()
17308                .unwrap();
17309            assert_eq!(replies.len(), 1);
17310            // Proof rechecking allowed the relay. Independent tag settlement
17311            // still asks the opener to retry after any same-epoch content change.
17312            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17313            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17314            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17315            let route = rig
17316                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17317                .await;
17318            assert!(rig.live(&route));
17319        }
17320
17321        #[tokio::test]
17322        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17323        {
17324            let mut rig = rig_with_flow_support(false).await;
17325            rig.sync(vec![session(1)]).await;
17326            let route = rig
17327                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17328                .await;
17329            assert!(rig.live(&route));
17330            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17331            let mut record = session(1);
17332            record.attributes.flow_id = Some("flow:7".to_string());
17333            rig.sync(vec![record]).await;
17334            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17335            assert!(rig.live(&unscoped));
17336            assert_eq!(unscoped.stamp(), None);
17337        }
17338
17339        #[tokio::test]
17340        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17341            let mut rig = rig().await;
17342            let mut record = session(1);
17343            record.attributes.flow_id = Some("flow:7".to_string());
17344            rig.sync(vec![record.clone()]).await;
17345            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17346            let mut owner_route = rig
17347                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17348                .await;
17349            let mut carrier_route = rig
17350                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17351                .await;
17352            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17353            rig.sync(vec![record.clone()]).await;
17354            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17355            assert!(rig.live(&owner_route) && owner_route.untouched());
17356            assert!(rig.live(&carrier_route) && carrier_route.untouched());
17357
17358            record.attributes.flow_id = Some("flow:8".to_string());
17359            rig.sync(vec![record]).await;
17360            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17361            let before = before.unwrap();
17362            assert_eq!(after.scope_epoch, before.scope_epoch);
17363            assert!(after.version > before.version);
17364            for route in [&mut owner_route, &mut carrier_route] {
17365                assert!(!rig.live(route));
17366                assert_eq!(
17367                    route.closed_reason(),
17368                    RouteCloseReason::ScopeDelegationChanged
17369                );
17370            }
17371            assert!(rig.live(&unscoped) && unscoped.untouched());
17372            // Each provider also receives a GOODBYE for its drained route;
17373            // consume it before expecting the next route.bind on that sink.
17374            for target in [PLEXUS, OTHER] {
17375                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17376                let goodbye = module_rx
17377                    .try_recv()
17378                    .expect("the provider sees the drain")
17379                    .frame;
17380                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17381                assert!(module_rx.try_recv().is_err());
17382            }
17383            let rebound = rig
17384                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17385                .await;
17386            assert_eq!(
17387                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17388                Some("flow:8")
17389            );
17390        }
17391
17392        /// The owner's sync lands between admission and the module's ack. The
17393        /// open is refused by name, the module's other routes stay up, and the
17394        /// reserved pair is released. Changed content is retryable; an ended
17395        /// scope is not.
17396        #[tokio::test]
17397        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17398            let mut rig = rig().await;
17399            rig.sync(vec![session(1)]).await;
17400            let mut cotenant = rig
17401                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17402                .await;
17403
17404            let mut changed = session(1);
17405            changed.child_owners.push(Principal::Reserved {
17406                module_id: MAGIC.to_string(),
17407            });
17408            let mut ended = None;
17409            for (code, next) in [
17410                (error_codes::SCOPE_CHANGED, vec![changed]),
17411                (error_codes::SCOPE_ENDED, Vec::new()),
17412            ] {
17413                let relayed = rig
17414                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17415                    .await;
17416                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17417                ended = Some(next.is_empty());
17418                rig.sync(next).await;
17419                rig.ack(&relayed).await;
17420                let replies = relayed.task.await.unwrap();
17421                assert_eq!(replies.len(), 1, "{replies:?}");
17422                assert_eq!(parse_error(&replies[0])["code"], code);
17423                assert_eq!(
17424                    subc_protocol::error_codes::is_retryable_route_open(code),
17425                    code == error_codes::SCOPE_CHANGED
17426                );
17427                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17428                // The module is told to drop just the binding it created.
17429                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17430                // Collected, because ending the scope also closes the co-tenant
17431                // route, whose GOODBYE comes first.
17432                let mut goodbyes = Vec::new();
17433                while let Ok(outbound) = plexus_rx.try_recv() {
17434                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17435                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17436                }
17437                assert!(
17438                    goodbyes.contains(&(bind_channel, bind_epoch)),
17439                    "{goodbyes:?}"
17440                );
17441                assert!(rig
17442                    .handler
17443                    .registry
17444                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17445                    .unwrap()
17446                    .is_some());
17447            }
17448            assert_eq!(ended, Some(true));
17449            // The co-tenant stayed up through the change, and closed only when
17450            // the scope ended, by the drain rule rather than by the commit.
17451            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17452        }
17453
17454        /// Each row of the drain table on one set of routes: the owner's, a
17455        /// bare carrier's, and a targeted carrier's to each of its targets.
17456        #[tokio::test]
17457        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17458            struct Case {
17459                name: &'static str,
17460                change: fn(&mut ScopeRecord),
17461                /// Closed routes by index: owner->plexus, aft->plexus,
17462                /// broca->plexus, broca->other.
17463                closed: [Option<RouteCloseReason>; 4],
17464            }
17465            use RouteCloseReason::*;
17466            let cases = [
17467                Case {
17468                    name: "a carrier entry removed",
17469                    change: |r| {
17470                        r.carriers.retain(|c| {
17471                            c.principal
17472                                != Principal::Reserved {
17473                                    module_id: AFT.to_string(),
17474                                }
17475                        })
17476                    },
17477                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17478                },
17479                Case {
17480                    name: "a target removed from a carrier",
17481                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17482                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
17483                },
17484                Case {
17485                    name: "a bare carrier narrowed to targets",
17486                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17487                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17488                },
17489                Case {
17490                    name: "delegates turned off",
17491                    change: |r| r.attributes.delegates = false,
17492                    closed: [Some(ScopeDelegationChanged); 4],
17493                },
17494                Case {
17495                    name: "agent_id changed",
17496                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17497                    closed: [Some(ScopeDelegationChanged); 4],
17498                },
17499                Case {
17500                    name: "a carrier added, child owners changed, the record re-sent",
17501                    change: |r| {
17502                        r.carriers.push(carrier(MAGIC, None));
17503                        r.child_owners.push(Principal::Reserved {
17504                            module_id: MAGIC.to_string(),
17505                        });
17506                    },
17507                    closed: [None; 4],
17508                },
17509                Case {
17510                    name: "a target added",
17511                    change: |r| {
17512                        r.carriers[1]
17513                            .targets
17514                            .as_mut()
17515                            .unwrap()
17516                            .push("third".to_string())
17517                    },
17518                    closed: [None; 4],
17519                },
17520                Case {
17521                    name: "delegates turned on",
17522                    change: |r| r.attributes.delegates = true,
17523                    closed: [None; 4],
17524                },
17525            ];
17526            for case in cases {
17527                let mut rig = rig().await;
17528                rig.sync(vec![session(1)]).await;
17529                let scope = || Some(rig_selector("s", Some(1)));
17530                let mut routes = [
17531                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
17532                    rig.bound(Some(AFT), PLEXUS, scope()).await,
17533                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
17534                    rig.bound(Some(BROCA), OTHER, scope()).await,
17535                ];
17536                let mut record = session(1);
17537                (case.change)(&mut record);
17538                rig.sync(vec![record]).await;
17539                for (index, expected) in case.closed.iter().enumerate() {
17540                    let route = &mut routes[index];
17541                    match expected {
17542                        Some(reason) => {
17543                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
17544                            assert_eq!(
17545                                route.closed_reason(),
17546                                *reason,
17547                                "{}: route {index}",
17548                                case.name
17549                            );
17550                        }
17551                        None => {
17552                            assert!(rig.live(route), "{}: route {index} closed", case.name);
17553                            assert!(
17554                                route.untouched(),
17555                                "{}: route {index} was told something",
17556                                case.name
17557                            );
17558                        }
17559                    }
17560                }
17561            }
17562        }
17563
17564        #[tokio::test]
17565        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17566            // Removed, and replaced by a higher epoch.
17567            for next in [Vec::new(), vec![session(2)]] {
17568                let mut rig = rig().await;
17569                rig.sync(vec![session(1)]).await;
17570                let mut route = rig
17571                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17572                    .await;
17573                rig.sync(next).await;
17574                assert!(!rig.live(&route));
17575                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17576            }
17577
17578            // A child whose parent ends: its routes close as parent-ended, the
17579            // child stays live, and routes under the parent close as ended.
17580            let mut rig = rig().await;
17581            let mut child = session(1);
17582            child.scope_ref = "child".to_string();
17583            child.kind = ScopeKind::Worker;
17584            child.parent = Some(ScopeParent::new(
17585                Principal::Reserved {
17586                    module_id: OWNER.to_string(),
17587                },
17588                "s".to_string(),
17589                1,
17590            ));
17591            rig.sync(vec![session(1), child.clone()]).await;
17592            let mut child_route = rig
17593                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17594                .await;
17595            assert_eq!(
17596                child_route.stamp().unwrap().parent_state,
17597                Some(ParentState::Linked)
17598            );
17599            rig.sync(vec![child]).await;
17600            assert!(!rig.live(&child_route));
17601            assert_eq!(
17602                child_route.closed_reason(),
17603                RouteCloseReason::ScopeParentEnded
17604            );
17605        }
17606
17607        #[tokio::test]
17608        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17609        ) {
17610            let mut rig = rig().await;
17611            rig.sync(vec![session(1)]).await;
17612            let mut route = rig
17613                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17614                .await;
17615            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17616            rig.sync(vec![session(1)]).await;
17617            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17618            assert!(rig.live(&route) && route.untouched());
17619
17620            // A call in flight on the route when another carrier is added. A
17621            // forwarded REQUEST holds one credit on the route's flow until the
17622            // module answers; the router takes it exactly like this.
17623            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17624                .forwarding
17625                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17626                .unwrap()
17627            else {
17628                panic!("the route is bound");
17629            };
17630            binding.flow.acquire_tagged(9, false).await.unwrap();
17631            let mut widened = session(1);
17632            widened.carriers.push(carrier(MAGIC, None));
17633            rig.sync(vec![widened]).await;
17634            assert!(rig.live(&route) && route.untouched());
17635            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17636            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17637            // The call's credit is still held on an open flow, so its answer
17638            // will be delivered: closing the route would have closed the flow.
17639            assert_eq!(binding.flow.in_flight(), 1);
17640            binding
17641                .flow
17642                .acquire_tagged(10, false)
17643                .await
17644                .expect("the flow is still open");
17645        }
17646
17647        /// A swap's superseded endpoint keeps its routes until drained; ending
17648        /// the scope closes them there too.
17649        #[tokio::test]
17650        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17651            let mut rig = rig().await;
17652            rig.sync(vec![session(1)]).await;
17653            let mut on_incumbent = rig
17654                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17655                .await;
17656
17657            // Swap plexus: register a candidate and cut over, leaving the
17658            // incumbent superseded with the route still on it.
17659            let (candidate, _candidate_rx) = wide_ctx(9);
17660            let registration = rig
17661                .handler
17662                .registry
17663                .register_candidate_with_control_ops(
17664                    manifest(PLEXUS, PROTOCOL_VERSION),
17665                    PROTOCOL_VERSION,
17666                    candidate.connection_id,
17667                    module_baseline_control_ops(),
17668                )
17669                .unwrap();
17670            rig.forwarding
17671                .register_candidate_module_connection(
17672                    candidate.connection_id,
17673                    PLEXUS.to_string(),
17674                    PROTOCOL_VERSION,
17675                    manifest_concurrency(&registration.manifest),
17676                    candidate.egress.clone(),
17677                )
17678                .unwrap();
17679            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17680            rig.handler
17681                .registry
17682                .promote_candidate(PLEXUS)
17683                .unwrap()
17684                .unwrap();
17685            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17686
17687            rig.sync(Vec::new()).await;
17688            assert!(!rig.live(&on_incumbent));
17689            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17690            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17691            let goodbye = incumbent_rx
17692                .try_recv()
17693                .expect("the superseded endpoint is told")
17694                .frame;
17695            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17696        }
17697    }
17698}
17699
17700#[cfg(test)]
17701mod concurrency_default_exposure_tests {
17702    use super::*;
17703
17704    fn hello_body(role_json: &str) -> Vec<u8> {
17705        format!(
17706            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17707        )
17708        .into_bytes()
17709    }
17710
17711    fn manifest_from(body: &[u8]) -> ModuleManifest {
17712        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17713        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17714            .expect("manifest parses")
17715    }
17716
17717    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17718
17719    #[test]
17720    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17721        let body = hello_body(&format!(
17722            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17723        ));
17724        let manifest = manifest_from(&body);
17725        // Precondition: serde really resolved it to the default, so the typed
17726        // manifest alone cannot answer the question this probe exists for.
17727        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17728        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17729    }
17730
17731    #[test]
17732    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17733        let body = hello_body(&format!(
17734            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17735        ));
17736        let manifest = manifest_from(&body);
17737        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17738        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17739    }
17740
17741    #[test]
17742    fn non_management_roles_are_never_reported() {
17743        let body = hello_body(
17744            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17745        );
17746        let manifest = manifest_from(&body);
17747        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17748    }
17749}