Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
26        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
27        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
28    },
29    scope::{
30        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32        SCOPE_SYNC_OP,
33    },
34    session::{
35        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38        ROLE_VERSIONS_FIELD,
39    },
40    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41    Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47    caller_identity::CallerIdentity,
48    capability_requirements::{
49        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51        RuntimeModule,
52    },
53    child_roster::DaemonShutdownFlag,
54    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55    forwarding::{
56        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59    },
60    observability::{
61        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62    },
63    privileged_prompt::{prompt_text, PrivilegedOperation},
64    provenance::{
65        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66    },
67    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68    router::{RouteCtx, RouterError},
69    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71    stderr_tail::{CaptureState, TailEntry},
72    supervise::{
73        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74        SupervisorHandle, SwapHelloAdmission,
75    },
76    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79/// Lowest envelope version this subc build will negotiate.
80///
81/// Module HELLO negotiation is exact: peers must use the daemon's locked
82/// protocol version. Older and newer peers receive `version_unsupported` and
83/// are not registered.
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93    ops::ACTIVITY_RECORD,
94    ops::ACTIVITY_QUERY,
95    ops::ACTIVITY_SUBSCRIBE,
96    ops::SERVER_DESCRIBE,
97    ops::CATALOG_LIST,
98    ops::ROUTE_OPEN,
99    ops::ROUTE_POLL,
100    ops::ROUTE_CLOSING,
101    ops::ROUTE_CLOSED,
102    ops::SUPERVISOR_LIST,
103    ops::SUPERVISOR_RESTART,
104    ops::SUPERVISOR_SWAP,
105    ops::SUPERVISOR_RELOAD,
106    ops::SUPERVISOR_RESCAN,
107    ops::SUPERVISOR_RELEASE_RESERVED,
108    ops::SUPERVISOR_SET_ENABLED,
109    ops::SUPERVISOR_HEALTH_PROBE,
110    ops::SUPERVISOR_HEALTH,
111    ops::SUPERVISOR_STDERR_TAIL,
112    ops::SUPERVISOR_TERMINALS,
113    ops::SUPERVISOR_ROUTES,
114    ops::SUPERVISOR_PROVENANCE,
115    ops::SUPERVISOR_SPAWN_SNAPSHOT,
116    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
120/// announced by a capability in `server.describe` instead, and clients check
121/// that capability before sending it. `ck daemon --json` prints `subc_ops`
122/// verbatim and scripts read that output, so a new name there would change it.
123/// A malformed body for one of these is still refused `invalid_control_body`
124/// rather than as an unknown op.
125const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132    "supervisor.live_roots",
133    SCOPE_SYNC_OP,
134    SCOPE_APPLY_OP,
135    SCOPE_DESCRIBE_OP,
136    "operator.confirm",
137];
138
139/// Module-originated ops the daemon answers but does not advertise in
140/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
141/// belongs to is incomplete, so no module is told it works before it does.
142const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146/// How long subc waits for a module to ack a relayed route.bind before returning
147/// `module_timeout`. The ack waits on the module's own configure, which for AFT
148/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
149/// and DB-open work — on a cold page cache or a large repo that legitimately
150/// exceeds a couple of seconds. The default is generous because rejecting a VALID
151/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
152/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
153pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155/// How many CONSECUTIVE full-budget relay timeouts against one target module
156/// open that module's bind-relay breaker.
157///
158/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
159/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
160/// so three consecutive full-budget timeouts take ~36s to observe: every client
161/// whose open contributed to opening the breaker had already given up on its
162/// own. That is what makes opening the breaker unable to turn a call that would
163/// have succeeded into a refusal — it can only make an already-failing module
164/// fail faster.
165///
166/// Two would be reachable inside one default deadline. One would convict a
167/// module on a single cold-cache bind, which is exactly the valid-but-slow case
168/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
169pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171/// How long a module's bind-relay breaker stays open before exactly one
172/// `route.open` is let through as a probe.
173///
174/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
175/// re-pays a full-budget stall almost continuously, and the breaker stops being
176/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
177/// deadline: a client that starts retrying after the module recovers has to get
178/// a probe opportunity inside its own deadline, or the breaker converts a
179/// recovered module into a failed call — the failure it exists to prevent,
180/// pointed the other way.
181///
182/// 20s sits between those with room on both sides, and it caps what a wedged
183/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
184/// rather than one per `route.open` per connection. The stall that motivated
185/// this, with its measurements, is written up in
186/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
187/// waited the whole budget out.
188pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194    configured: &Path,
195    spawned_from: Option<&Path>,
196    image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198    let path = match spawned_from {
199        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200            ReloadPathAgreement::Match
201        }
202        Some(spawned_from) => ReloadPathAgreement::Mismatch {
203            configured: configured.to_path_buf(),
204            spawned_from: spawned_from.to_path_buf(),
205        },
206        None => ReloadPathAgreement::Unavailable {
207            reason: if matches!(
208                image,
209                subc_control::RunningImageAgreement::Unavailable {
210                    reason: subc_control::RunningImageUnavailableReason::NotRunning
211                }
212            ) {
213                ReloadPathUnavailableReason::NotRunning
214            } else {
215                ReloadPathUnavailableReason::SpawnedPathUnavailable
216            },
217        },
218    };
219    PendingReloadVerdict {
220        path,
221        image,
222        spec: None,
223    }
224}
225
226fn launch_spec_agreement(
227    configured: &crate::supervise::ModuleSpec,
228    running: bool,
229    spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232    if !running {
233        return Agreement::Unavailable {
234            reason: Reason::NotRunning,
235        };
236    }
237    match spawned {
238        None => Agreement::Unavailable {
239            reason: Reason::SpawnedSpecUnavailable,
240        },
241        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242            Agreement::Match
243        }
244        Some(_) => Agreement::Mismatch,
245    }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249    if configured == spawned_from {
250        return true;
251    }
252    #[cfg(windows)]
253    {
254        // Windows spawn records the resolved verbatim path (\\?\...), while
255        // configuration keeps its original spelling. Resolve both with the OS
256        // instead of stripping prefixes or folding case, which could conflate
257        // distinct names in case-sensitive directories. Different resolved
258        // paths still differ even if their files have identical content.
259        match (
260            std::fs::canonicalize(configured),
261            std::fs::canonicalize(spawned_from),
262        ) {
263            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264            _ => false,
265        }
266    }
267    #[cfg(not(windows))]
268    {
269        false
270    }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275    build: DaemonBuildProvenance,
276    pid: Option<u32>,
277    started_at_ms: Option<u64>,
278    start_clock: Option<crate::clock::StartClock>,
279    executable_path: Option<PathBuf>,
280    executable_identity: Option<SpawnedFileIdentity>,
281    process_start_time: Option<u64>,
282    probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286    fn default() -> Self {
287        Self {
288            build: DaemonBuildProvenance {
289                build_git_sha: None,
290                build_lock_digest: None,
291            },
292            pid: None,
293            started_at_ms: None,
294            start_clock: None,
295            executable_path: None,
296            executable_identity: None,
297            process_start_time: None,
298            probe: ExecutableIdentityProbe::default(),
299        }
300    }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305    supervisor: Supervisor,
306    config_path: PathBuf,
307    configured_port: Option<u16>,
308    storage_config: Option<crate::daemon_config::StorageConfig>,
309    admission_facts_carrier_module_id: Option<String>,
310    admission_facts_targets: Option<Vec<String>>,
311    scope_authority_owners: Vec<String>,
312    privileged_operator_prompt: PrivilegedOperatorPrompt,
313    /// Counts executed rescans that set out to change the supervised modules'
314    /// staged or applied configuration, counted before the first change so a
315    /// rescan that fails partway still counts. With the daemon incarnation it
316    /// names the state a `supervisor.rescan_checked` preview was computed
317    /// against.
318    generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321/// Refusal labels passed to `observe_route_open_refusal` that mean the target
322/// module is not serving right now, and so open or extend an outage in the
323/// route outage tracker. Every one of them is only reachable after the target
324/// was found in the registry, which is what keeps an arbitrary client-chosen
325/// id from ever creating tracker state.
326///
327/// Deliberately absent: `not_registered` and `removed` (the id may be
328/// anything a client sent, and a removed module is gone on purpose),
329/// `protocol_none` (such a module never serves routes, so nothing is out),
330/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
331/// capability and admission-facts refusals (they refuse the caller, not a
332/// module outage), and `relay_reservation_failed` (its code ranges over
333/// capacity limits as well as a vanished connection). Capacity, breaker,
334/// relay-timeout and module-rejection refusals do not pass through that
335/// function at all; the breaker logs its own transitions.
336///
337/// The two not-serving refusals that bypass that function record themselves
338/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
339/// `required_capability_unprovided` is not tracked: the module itself is up,
340/// and the outage belongs to the missing provider.
341const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342    "reloading",
343    "supervisor_not_live",
344    "registration_not_active",
345    "no_forwarding_connection",
346    "relay_send_failed",
347];
348
349/// Real channel-0 control handler for subc itself.
350#[derive(Clone)]
351pub struct ControlHandler {
352    registry: Arc<Registry>,
353    activity: Option<crate::activity::ActivityLedger>,
354    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355    forwarding: Arc<ForwardingTable>,
356    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357    supervisor: SupervisorHandle,
358    subc_capabilities: Arc<[String]>,
359    /// Daemon-wide route.bind relay budget. Used as the fallback when the
360    /// target module has no per-module override in
361    /// `route_bind_relay_timeouts`.
362    route_bind_relay_timeout: Duration,
363    /// Per-module route.bind relay budget overrides, keyed by module id. When
364    /// `handle_route_open` resolves the deadline for a target module, a
365    /// per-module entry wins over the daemon-wide value above.
366    route_bind_relay_timeouts: BTreeMap<String, Duration>,
367    /// Per-target-module bind-relay breaker state. Shared with the forwarding
368    /// table, which is where a new module connection resets it.
369    route_bind_breakers: RouteBindBreakers,
370    /// Live relay admissions keyed by target module. Shared through the
371    /// forwarding table so cloned or separately built handlers enforce one cap.
372    route_bind_concurrency: RouteBindConcurrency,
373    /// Start and end of each module's not-serving period as seen by
374    /// `route.open`, so an outage gets one line at each edge instead of only
375    /// the per-refusal INFO lines. Taken from the forwarding table, so every
376    /// handler built over one table shares it.
377    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378    /// Consecutive relay timeouts that open a module's breaker.
379    route_bind_breaker_threshold: u32,
380    /// How long a breaker stays open before one probe is admitted.
381    route_bind_breaker_cooldown: Duration,
382    health_probe_timeout: Duration,
383    /// Central storage policy. When set, each registering module receives its
384    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
385    storage_config: Option<crate::daemon_config::StorageConfig>,
386    /// The machine id established at boot, served on every HELLO_ACK and on
387    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
388    /// changes the file, never this value. `None` serves no id.
389    machine_id: Option<crate::machine_id::MachineId>,
390    admission_facts_carrier_module_id: Option<String>,
391    admission_facts_targets: Option<Vec<String>>,
392    /// Scope records with their sync authorities and tombstones; see
393    /// `crate::scopes`. Shared by clones of this handler, so every connection
394    /// reads and writes one table.
395    scopes: Arc<RwLock<ScopeTable>>,
396    /// Wakes the expiry loop when an accepted scope change may have added a
397    /// deadline, so the loop can wait without ticking while no live scope has
398    /// one. A permit is stored when nothing waits, so no wake is lost.
399    scope_deadline_added: Arc<tokio::sync::Notify>,
400    /// The configured `scope_authority_owners`, kept so a rescan can report a
401    /// changed value as needing a daemon restart; rescan never applies it.
402    scope_authority_owners: Vec<String>,
403    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
404    privileged_operator_prompt: PrivilegedOperatorPrompt,
405    /// The local and peer address of each accepted TCP client connection, read
406    /// from the daemon's own socket, so a privileged request can name its caller.
407    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408    #[cfg(test)]
409    caller_lookup: Option<TestCallerLookup>,
410    /// The launch nonce each module connection presented at HELLO, which is how
411    /// a `scope.sync` is matched to the owner's current launch.
412    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413    rescan: Option<SupervisorRescanContext>,
414    connected_clients: ConnectedClients,
415    counters: DaemonCounters,
416    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
418    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
419    /// their usual severity (see [`log_requirement_events`]). A handler built
420    /// without one has a flag that is never set.
421    daemon_shutdown: DaemonShutdownFlag,
422    daemon_provenance: DaemonProvenanceFacts,
423    #[cfg(test)]
424    control_dispatch_delay: Option<Duration>,
425    #[cfg(test)]
426    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427    /// Holds `route.open` after the route.bind is queued to the module and
428    /// before it is marked enqueued, so a test can make the module's reply (or
429    /// a client teardown) land exactly inside that window.
430    #[cfg(test)]
431    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432    /// Holds a HELLO after its registry entry is reserved and before its
433    /// forwarding endpoint is installed, so a test can read the catalog and
434    /// open routes while the module is registered but not yet routable. A
435    /// blocking barrier because the HELLO handler is synchronous; the test
436    /// runs the HELLO on its own thread.
437    #[cfg(test)]
438    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443        f.debug_struct("ControlHandler")
444            .field("registry", &self.registry)
445            .field("forwarding", &self.forwarding)
446            .field("process_liveness", &self.process_liveness.is_some())
447            .field("supervisor", &self.supervisor)
448            .field("subc_capabilities", &self.subc_capabilities)
449            .finish()
450    }
451}
452
453struct RouteOpenRequest {
454    target: RouteTarget,
455    identity: BindIdentity,
456    consumer_identity: Option<ConsumerIdentity>,
457    consumer_capabilities: Option<Vec<String>>,
458    role_versions: Option<BTreeMap<String, String>>,
459    admission_facts: Option<serde_json::Value>,
460    scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465    Admission,
466    ProjectRoot,
467    EgressReservation,
468    BindSend,
469    BindAckWait,
470}
471
472/// Store elapsed time for each route-open phase, including attempts that end
473/// in an early refusal or backend error. Admission also includes rechecks and
474/// commit bookkeeping performed between the other timed phases.
475struct RouteOpenTimings {
476    module_id: String,
477    principal: String,
478    admission: Duration,
479    project_root: Duration,
480    egress_reservation: Duration,
481    bind_send: Duration,
482    bind_ack_wait: Duration,
483    phase: Option<RouteOpenPhase>,
484    phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488    fn new() -> Self {
489        Self {
490            module_id: String::new(),
491            // Refusals before attestation must not claim a verified principal.
492            principal: "unresolved".to_string(),
493            admission: Duration::ZERO,
494            project_root: Duration::ZERO,
495            egress_reservation: Duration::ZERO,
496            bind_send: Duration::ZERO,
497            bind_ack_wait: Duration::ZERO,
498            phase: Some(RouteOpenPhase::Admission),
499            phase_started_at: Instant::now(),
500        }
501    }
502
503    fn finish(&mut self) {
504        let elapsed = self.phase_started_at.elapsed();
505        match self.phase.take() {
506            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511            None => {}
512        }
513    }
514
515    fn enter(&mut self, phase: RouteOpenPhase) {
516        self.finish();
517        self.phase = Some(phase);
518        self.phase_started_at = Instant::now();
519    }
520}
521
522struct RouteBindReservationGuard {
523    forwarding: Arc<ForwardingTable>,
524    endpoint: ModuleEndpointId,
525    relay_corr: u64,
526    armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530    forwarding: Arc<ForwardingTable>,
531    endpoint: ModuleEndpointId,
532    corr: u64,
533    armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538        Self {
539            forwarding,
540            endpoint,
541            corr,
542            armed: true,
543        }
544    }
545
546    fn disarm(&mut self) {
547        self.armed = false;
548    }
549}
550
551impl Drop for ModuleControlRpcGuard {
552    fn drop(&mut self) {
553        if self.armed {
554            let _ = self
555                .forwarding
556                .cancel_module_control_rpc(self.endpoint, self.corr);
557        }
558    }
559}
560
561impl RouteBindReservationGuard {
562    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563        Self {
564            forwarding,
565            endpoint,
566            relay_corr,
567            armed: true,
568        }
569    }
570
571    fn release_and_disarm(&mut self) {
572        if !self.armed {
573            return;
574        }
575        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576            self.endpoint,
577            self.relay_corr,
578            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579        ) {
580            send_goodbye_target_best_effort(
581                &self.forwarding.counters(),
582                &target,
583                "canceled route.bind",
584            );
585        }
586        self.armed = false;
587    }
588
589    fn disarm(&mut self) {
590        self.armed = false;
591    }
592}
593
594impl Drop for RouteBindReservationGuard {
595    fn drop(&mut self) {
596        self.release_and_disarm();
597    }
598}
599
600/// Per-target-module circuit breaker around the `route.bind` relay.
601///
602/// The connection reader is serial per connection, so a module whose `on_bind`
603/// sits on the ack blocks every LATER frame on the connections that call it,
604/// including calls to unrelated modules. This does not make any module's bind
605/// fast; it stops the daemon paying the full budget again and again for a
606/// condition it has already observed.
607///
608/// State is keyed by TARGET MODULE and shared by every connection: a wedged
609/// module wedges everyone, so what one connection learned should protect the
610/// rest.
611///
612/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
613/// relay to that module has actually timed out, and is removed again when a
614/// relay is accepted or the module reconnects, so it cannot grow with traffic
615/// or with modules that behave.
616///
617/// # Why a `std` mutex here is not the head-of-line defect again
618///
619/// Acquisition never awaits. The critical section is a hash lookup plus a few
620/// integer updates, with no I/O and no `.await` inside it, so a reader task
621/// cannot be descheduled behind it the way it can behind
622/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
623/// primitive, held for the same kind of work, as the refusal counter this very
624/// path already increments.
625///
626/// It is also NOT on the data-plane splice path: only `route.open` and module
627/// registration touch it, so bound-route frames gain no state check and no
628/// contention.
629#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636    modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640    concurrency: RouteBindConcurrency,
641    module_id: String,
642}
643
644impl RouteBindConcurrency {
645    /// Admit without waiting. Waiting here would move the bind stall from the
646    /// module reply to a semaphore and restore reader head-of-line blocking.
647    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648        let mut modules = self
649            .modules
650            .lock()
651            .expect("route.bind concurrency mutex poisoned");
652        let in_flight = modules.entry(module_id.to_string()).or_default();
653        if *in_flight >= limit {
654            return Err(*in_flight);
655        }
656        *in_flight += 1;
657        Ok(RouteBindConcurrencyGuard {
658            concurrency: self.clone(),
659            module_id: module_id.to_string(),
660        })
661    }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665    fn drop(&mut self) {
666        let mut modules = self
667            .concurrency
668            .modules
669            .lock()
670            .expect("route.bind concurrency mutex poisoned");
671        let remove = {
672            let in_flight = modules
673                .get_mut(&self.module_id)
674                .expect("admitted route.bind has a concurrency entry");
675            *in_flight -= 1;
676            *in_flight == 0
677        };
678        if remove {
679            modules.remove(&self.module_id);
680        }
681    }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686    /// Relay timeouts observed with no accepted relay in between.
687    consecutive_timeouts: u32,
688    /// `Some` while the breaker is open: the instant the cooldown expires and
689    /// the next arrival may probe. `None` means closed.
690    cooldown_until: Option<Instant>,
691    /// A half-open probe has been admitted and has not settled yet. This is
692    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
693    /// that read the cooldown, so concurrent opens arriving at the moment the
694    /// cooldown expires cannot all decide that they are the probe.
695    probe_in_flight: Option<Arc<()>>,
696}
697
698/// What the breaker decided for one `route.open`, before any relay work.
699enum RouteBindAdmission<'a> {
700    Admitted {
701        guard: RouteBindBreakerGuard<'a>,
702        /// This open is the single half-open probe, so the transition is worth
703        /// one log line.
704        probe: bool,
705    },
706    Refused {
707        consecutive_timeouts: u32,
708        /// What is left of the cooldown. Zero when the refusal is because the
709        /// one probe is already in flight rather than because the cooldown has
710        /// not elapsed.
711        retry_in: Duration,
712        probe_in_flight: bool,
713    },
714}
715
716/// An outstanding admission, which must be told how its relay settled.
717///
718/// `Drop` settles it as inconclusive, so an early return between admission and
719/// the relay -- or the whole handler being cancelled when the client
720/// disconnects -- releases a half-open probe slot instead of leaving the
721/// breaker wedged half-open with no further probes.
722struct RouteBindBreakerGuard<'a> {
723    breakers: RouteBindBreakers,
724    module_id: &'a str,
725    probe_token: Option<Arc<()>>,
726    settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730    /// The module answered within the budget and took the bind. THE ONLY
731    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
732    /// breaker, which is a transition worth logging.
733    fn record_accepted(&mut self) -> bool {
734        self.settled = true;
735        self.breakers.record_accepted(self.module_id)
736    }
737
738    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
739    /// COUNTS TOWARD OPENING.
740    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741        self.settled = true;
742        self.breakers.record_timeout(
743            self.module_id,
744            self.probe_token.as_ref(),
745            threshold,
746            cooldown,
747        )
748    }
749
750    /// Everything else: the module REJECTED the bind, its connection went away
751    /// mid-relay, or the waiter was cancelled.
752    ///
753    /// None of these is evidence that a module is slow, and each already has
754    /// its own refusal with its own code. A module that rejects a bind in
755    /// microseconds is healthy and must never be convicted for it; a module
756    /// that died has said nothing about the module that replaces it. So these
757    /// neither increment nor reset the count -- they only release a probe slot.
758    fn record_inconclusive(&mut self) {
759        self.settled = true;
760        self.breakers
761            .record_inconclusive(self.module_id, self.probe_token.as_ref());
762    }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766    fn drop(&mut self) {
767        if !self.settled {
768            self.breakers
769                .record_inconclusive(self.module_id, self.probe_token.as_ref());
770        }
771    }
772}
773
774/// The breaker moved to open, reported so the caller can log it outside the
775/// lock. Opening is rare and load-bearing; the refusals that follow are
776/// frequent and are counted rather than logged.
777struct BreakerOpened {
778    consecutive_timeouts: u32,
779    /// True when a failed probe re-opened an already-open breaker, which reads
780    /// very differently in a log from a first opening.
781    reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786        self.modules
787            .lock()
788            .expect("route.bind breaker mutex poisoned")
789    }
790
791    /// Decide whether this `route.open` may attempt its relay. Takes the map
792    /// lock and nothing else, and never awaits.
793    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795            probe: probe_token.is_some(),
796            guard: RouteBindBreakerGuard {
797                breakers: self.clone(),
798                module_id,
799                probe_token,
800                settled: false,
801            },
802        };
803
804        let mut modules = self.lock();
805        let Some(state) = modules.get_mut(module_id) else {
806            return admitted(None);
807        };
808        let Some(cooldown_until) = state.cooldown_until else {
809            return admitted(None);
810        };
811        if state.probe_in_flight.is_some() {
812            return RouteBindAdmission::Refused {
813                consecutive_timeouts: state.consecutive_timeouts,
814                retry_in: Duration::ZERO,
815                probe_in_flight: true,
816            };
817        }
818        let now = Instant::now();
819        if now < cooldown_until {
820            return RouteBindAdmission::Refused {
821                consecutive_timeouts: state.consecutive_timeouts,
822                retry_in: cooldown_until - now,
823                probe_in_flight: false,
824            };
825        }
826        let token = Arc::new(());
827        state.probe_in_flight = Some(Arc::clone(&token));
828        admitted(Some(token))
829    }
830
831    fn record_accepted(&self, module_id: &str) -> bool {
832        self.lock()
833            .remove(module_id)
834            .is_some_and(|state| state.cooldown_until.is_some())
835    }
836
837    fn record_timeout(
838        &self,
839        module_id: &str,
840        probe_token: Option<&Arc<()>>,
841        threshold: u32,
842        cooldown: Duration,
843    ) -> Option<BreakerOpened> {
844        let mut modules = self.lock();
845        let state = modules.entry(module_id.to_string()).or_default();
846        let was_open = state.cooldown_until.is_some();
847        let was_probe = Self::owns_probe(state, probe_token);
848        if was_probe {
849            state.probe_in_flight = None;
850        }
851        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852        if state.consecutive_timeouts < threshold {
853            return None;
854        }
855        state.cooldown_until = Some(Instant::now() + cooldown);
856        Some(BreakerOpened {
857            consecutive_timeouts: state.consecutive_timeouts,
858            reopened_after_probe: was_open && was_probe,
859        })
860    }
861
862    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863        // A relay can finish after the breaker was reset or after it opened
864        // again and started a new probe. Only the guard whose token matches the
865        // active probe may release it, so a late relay never frees a newer probe.
866        state
867            .probe_in_flight
868            .as_ref()
869            .zip(token)
870            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871    }
872
873    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874        if let Some(state) = self.lock().get_mut(module_id) {
875            if Self::owns_probe(state, probe_token) {
876                state.probe_in_flight = None;
877            }
878        }
879    }
880
881    /// Discard what was learned about a module, because the process it was
882    /// learned about is gone. Returns the discarded count when it was non-zero.
883    ///
884    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
885    /// `module_id` is a configuration identity that outlives any particular
886    /// child; what the breaker observed was the process behind the module
887    /// connection of the moment. When a new connection registers under that id
888    /// the verdict's subject no longer exists, so the verdict is stale by
889    /// construction rather than merely likely to be wrong. Keeping it would
890    /// apply a dead process's record to a live one, which is the same defect
891    /// class this breaker exists to stop the daemon committing.
892    ///
893    /// A half-open probe in flight is discarded with the rest: it was a
894    /// question about the old process.
895    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896        self.lock()
897            .remove(module_id)
898            .map(|state| state.consecutive_timeouts)
899            .filter(|discarded| *discarded > 0)
900    }
901
902    /// Open breakers, for the `server.describe` counters object. `None` when
903    /// none is open, so the key stays absent rather than present-and-empty.
904    ///
905    /// This is the operator's answer to "is this module refusing instantly or
906    /// is it fine?", which look identical from a client that retries and then
907    /// succeeds.
908    fn open_snapshot(&self) -> Option<serde_json::Value> {
909        let now = Instant::now();
910        let modules = self.lock();
911        let open = modules
912            .iter()
913            .filter_map(|(module_id, state)| {
914                let cooldown_until = state.cooldown_until?;
915                Some((
916                    module_id.clone(),
917                    serde_json::json!({
918                        "consecutive_timeouts": state.consecutive_timeouts,
919                        "cooldown_remaining_ms":
920                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
921                        "probe_in_flight": state.probe_in_flight.is_some(),
922                    }),
923                ))
924            })
925            .collect::<serde_json::Map<String, serde_json::Value>>();
926        (!open.is_empty()).then_some(serde_json::Value::Object(open))
927    }
928}
929
930impl ControlHandler {
931    pub fn new(registry: Arc<Registry>) -> Self {
932        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933    }
934
935    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936        let counters = forwarding.counters();
937        // Taken from the forwarding table rather than created here, so that the
938        // breaker a `route.open` consults is the same one a module's
939        // registration resets, however many handlers are built over one table.
940        let route_bind_breakers = forwarding.route_bind_breakers();
941        let route_bind_concurrency = forwarding.route_bind_concurrency();
942        let route_outages = forwarding.route_outages();
943        Self {
944            registry,
945            activity: None,
946            activity_streams: Arc::new(Mutex::new(HashMap::new())),
947            forwarding,
948            process_liveness: None,
949            supervisor: SupervisorHandle::new(),
950            subc_capabilities: Arc::from([
951                CAP_MANIFEST_REGISTRATION.to_string(),
952                CAP_CHANNEL_LIFECYCLE.to_string(),
953                CAP_PING_PONG.to_string(),
954                CAP_SESSION_ATTACH.to_string(),
955                CAP_ADMISSION_FACTS_RELAY.to_string(),
956                CAP_SCOPES_V1.to_string(),
957                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
965                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
966                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
967            ]),
968            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
969            route_bind_relay_timeouts: BTreeMap::new(),
970            route_bind_breakers,
971            route_bind_concurrency,
972            route_outages,
973            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
974            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
975            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
976            storage_config: None,
977            machine_id: None,
978            admission_facts_carrier_module_id: None,
979            admission_facts_targets: None,
980            scopes: Arc::new(RwLock::new(ScopeTable::new(
981                crate::daemon_config::default_scope_authority_owners(),
982            ))),
983            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
984            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
985            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
986            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
987            #[cfg(test)]
988            caller_lookup: None,
989            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
990            rescan: None,
991            connected_clients: ConnectedClients::new(),
992            counters,
993            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
994            daemon_shutdown: DaemonShutdownFlag::default(),
995            daemon_provenance: DaemonProvenanceFacts::default(),
996            #[cfg(test)]
997            control_dispatch_delay: None,
998            #[cfg(test)]
999            provenance_probe_override: None,
1000            #[cfg(test)]
1001            route_bind_relay_sent_hold: None,
1002            #[cfg(test)]
1003            hello_registry_to_forwarding_hold: None,
1004        }
1005    }
1006
1007    /// Set the central storage policy: registering modules then receive their
1008    /// resolved storage descriptor in HELLO_ACK.
1009    pub fn with_storage_config(
1010        mut self,
1011        storage_config: Option<crate::daemon_config::StorageConfig>,
1012    ) -> Self {
1013        self.storage_config = storage_config;
1014        self
1015    }
1016
1017    /// Open the ledger on its own writer thread; an unavailable sink does not
1018    /// prevent the daemon from serving routing or supervisor requests.
1019    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1020        self.activity = Some(crate::activity::ActivityLedger::open(
1021            run_dir.join("activity.jsonl"),
1022            incarnation,
1023        ));
1024        self.supervisor.set_activity(self.activity.clone());
1025        self
1026    }
1027
1028    /// Set the machine id served to every registering module (HELLO_ACK) and on
1029    /// `server.describe`.
1030    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1031        self.machine_id = machine_id;
1032        self
1033    }
1034
1035    /// Configure the exact reserved module and target ids permitted to relay
1036    /// opaque admission facts. Config-file loading validates this authority;
1037    /// this builder keeps the same policy available to embedded test daemons.
1038    pub fn with_admission_facts_config(
1039        mut self,
1040        carrier_module_id: Option<String>,
1041        targets: Option<Vec<String>>,
1042    ) -> Self {
1043        self.admission_facts_carrier_module_id = carrier_module_id;
1044        self.admission_facts_targets = targets;
1045        self
1046    }
1047
1048    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1049    /// Replaces the scope table with an empty one under the new list, so call it
1050    /// while building the handler, before any module can sync.
1051    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1052        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1053        self.scope_authority_owners = owners;
1054        self
1055    }
1056
1057    /// Set the startup policy before installing the rescan context or serving requests.
1058    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1059        self.privileged_operator_prompt = mode;
1060        self
1061    }
1062
1063    /// Override the route.bind relay timeout. Used by tests that assert the
1064    /// timeout path so they don't block on the production-safe default.
1065    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1066        self.route_bind_relay_timeout = timeout;
1067        self
1068    }
1069
1070    /// Install per-module route.bind relay budget overrides. A module id
1071    /// listed here wins over the daemon-wide default set via
1072    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1073    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1074    /// the same `Duration` the bind path will use.
1075    pub fn with_route_bind_relay_timeouts(
1076        mut self,
1077        timeouts: impl IntoIterator<Item = (String, Duration)>,
1078    ) -> Self {
1079        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1080        self
1081    }
1082
1083    /// Resolve the route.bind relay budget for a specific target module id.
1084    /// Per-module overrides win; the daemon-wide value (set via
1085    /// `with_route_bind_relay_timeout` or the built-in default) is the
1086    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1087    /// the same resolution `handle_route_open` will use.
1088    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1089        self.route_bind_relay_timeouts
1090            .get(module_id)
1091            .copied()
1092            .unwrap_or(self.route_bind_relay_timeout)
1093    }
1094
1095    /// Override the per-module bind-relay breaker policy.
1096    ///
1097    /// Used by tests, which cannot spend three production budgets opening a
1098    /// breaker or twenty seconds waiting for its cooldown. The production
1099    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1100    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1101    /// reasoning for the numbers.
1102    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1103        self.route_bind_breaker_threshold = threshold.max(1);
1104        self.route_bind_breaker_cooldown = cooldown;
1105        self
1106    }
1107
1108    #[cfg(test)]
1109    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1110        self.health_probe_timeout = timeout;
1111        self
1112    }
1113
1114    #[cfg(test)]
1115    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1116        self.control_dispatch_delay = Some(delay);
1117        self
1118    }
1119
1120    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1121    /// announce it is parked, once to be released. Build the barrier for two
1122    /// parties, the test being the other one.
1123    #[cfg(test)]
1124    pub(crate) fn with_route_bind_relay_sent_hold(
1125        mut self,
1126        hold: Arc<tokio::sync::Barrier>,
1127    ) -> Self {
1128        self.route_bind_relay_sent_hold = Some(hold);
1129        self
1130    }
1131
1132    /// A HELLO waits on `hold` twice between reserving its registry entry and
1133    /// installing its forwarding endpoint: once to announce it is parked, once
1134    /// to be released. Build the barrier for two parties, the test being the
1135    /// other one, and run the HELLO on a thread of its own.
1136    #[cfg(test)]
1137    pub(crate) fn with_hello_registry_to_forwarding_hold(
1138        mut self,
1139        hold: Arc<std::sync::Barrier>,
1140    ) -> Self {
1141        self.hello_registry_to_forwarding_hold = Some(hold);
1142        self
1143    }
1144
1145    #[cfg(test)]
1146    fn hold_between_registry_and_forwarding(&self) {
1147        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1148            hold.wait();
1149            hold.wait();
1150        }
1151    }
1152
1153    pub fn with_process_liveness(
1154        mut self,
1155        process_liveness: Arc<dyn ModuleProcessLiveness>,
1156    ) -> Self {
1157        self.process_liveness = Some(process_liveness);
1158        self
1159    }
1160
1161    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1162        self.supervisor = supervisor;
1163        self.supervisor.set_activity(self.activity.clone());
1164        self
1165    }
1166
1167    pub fn with_daemon_provenance(
1168        mut self,
1169        pid: u32,
1170        started_at_ms: u64,
1171        executable_path: Option<PathBuf>,
1172        build_git_sha: Option<String>,
1173        build_lock_digest: Option<String>,
1174    ) -> Self {
1175        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1176        let process_start_time = process_start_time(pid);
1177        self.daemon_provenance = DaemonProvenanceFacts {
1178            build: DaemonBuildProvenance {
1179                build_git_sha,
1180                build_lock_digest,
1181            },
1182            pid: Some(pid),
1183            started_at_ms: Some(started_at_ms),
1184            start_clock: None,
1185            executable_path,
1186            executable_identity,
1187            process_start_time,
1188            probe: ExecutableIdentityProbe::default(),
1189        };
1190        self
1191    }
1192
1193    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1194        self.daemon_provenance.start_clock = Some(clock);
1195        self
1196    }
1197
1198    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1199    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1200        self.scopes
1201            .write()
1202            .unwrap_or_else(|poisoned| poisoned.into_inner())
1203            .set_wall_clock(Arc::new(clock));
1204        self
1205    }
1206
1207    #[cfg(test)]
1208    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1209        self.provenance_probe_override = Some(result);
1210        self
1211    }
1212
1213    /// Install the configured module set and its reserved capability bindings.
1214    /// Bindings are configuration-scoped and may point at a provider that has not
1215    /// been installed yet, so this does not require the bound module to exist.
1216    pub fn with_capability_config(
1217        self,
1218        modules: impl IntoIterator<Item = (String, bool)>,
1219        reserved_capabilities: BTreeMap<String, String>,
1220    ) -> Self {
1221        self.capability_evaluator
1222            .configure(modules, reserved_capabilities);
1223        self
1224    }
1225
1226    /// Share the supervisor's daemon-shutdown flag with this handler, so
1227    /// requirement changes after the daemon has begun its announced shutdown
1228    /// are not reported as faults.
1229    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1230        self.daemon_shutdown = flag;
1231        self
1232    }
1233
1234    pub fn with_supervisor_rescan(
1235        mut self,
1236        supervisor: Supervisor,
1237        config_path: impl Into<PathBuf>,
1238        configured_port: Option<u16>,
1239    ) -> Self {
1240        self.rescan = Some(SupervisorRescanContext {
1241            supervisor,
1242            config_path: config_path.into(),
1243            configured_port,
1244            storage_config: self.storage_config.clone(),
1245            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1246            admission_facts_targets: self.admission_facts_targets.clone(),
1247            scope_authority_owners: self.scope_authority_owners.clone(),
1248            privileged_operator_prompt: self.privileged_operator_prompt,
1249            generation: Arc::default(),
1250        });
1251        self
1252    }
1253
1254    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1255        self.connected_clients = connected_clients;
1256        self
1257    }
1258
1259    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1260        Arc::clone(&self.forwarding)
1261    }
1262
1263    pub(crate) fn counters(&self) -> DaemonCounters {
1264        self.counters.clone()
1265    }
1266
1267    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1268    /// requirement event without depending on an operator polling a status command.
1269    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1270        tokio::spawn(async move {
1271            loop {
1272                self.capability_evaluator
1273                    .wait_for_change_or_deadline()
1274                    .await;
1275                self.refresh_capability_requirements();
1276            }
1277        });
1278    }
1279
1280    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1281    /// deadline passed during sleep is enforced on the first tick after wake.
1282    /// While no live scope carries a deadline the loop waits for an accepted
1283    /// scope change instead of ticking, so a daemon without deadlines never
1284    /// wakes for expiry.
1285    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1286        tokio::spawn(async move {
1287            loop {
1288                let has_deadlines = self
1289                    .scopes
1290                    .read()
1291                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1292                    .has_deadlines();
1293                if !has_deadlines {
1294                    self.scope_deadline_added.notified().await;
1295                    continue;
1296                }
1297                tokio::time::sleep(Duration::from_secs(1)).await;
1298                if let Err(error) = self.sweep_expired_scopes() {
1299                    warn!(%error, "scope expiry sweep failed");
1300                }
1301            }
1302        });
1303    }
1304
1305    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1306        let mut table = self
1307            .scopes
1308            .write()
1309            .unwrap_or_else(|poisoned| poisoned.into_inner());
1310        let swept = table.sweep_expired();
1311        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1312        drop(table);
1313        self.close_scope_drained_routes(drained);
1314        Ok(())
1315    }
1316
1317    fn publish_scope_changes(
1318        &self,
1319        changes: &[crate::scopes::ScopeTagChange],
1320        expired: &[crate::scopes::ScopeExpired],
1321    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1322        if changes.is_empty() {
1323            return Ok(Vec::new());
1324        }
1325        let drained = self
1326            .forwarding
1327            .publish_scope_changes(changes)
1328            .map_err(RouterError::Forwarding)?;
1329        let mut counts = HashMap::new();
1330        for route in &drained {
1331            *counts
1332                .entry((
1333                    route.scope.owner.as_str(),
1334                    route.scope.scope_ref.as_str(),
1335                    route.scope.tag.scope_epoch,
1336                ))
1337                .or_insert(0usize) += 1;
1338        }
1339        for scope in expired {
1340            let routes_closed = counts
1341                .get(&(
1342                    scope.owner.as_str(),
1343                    scope.scope_ref.as_str(),
1344                    scope.scope_epoch,
1345                ))
1346                .copied()
1347                .unwrap_or(0);
1348            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1349                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1350                routes_closed, "scope expired");
1351        }
1352        Ok(drained)
1353    }
1354
1355    fn runtime_capability_snapshot(
1356        &self,
1357    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1358        let runtime = self
1359            .supervisor
1360            .list()
1361            .into_iter()
1362            .map(|module| {
1363                let status = module.status().map_err(|err| {
1364                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1365                })?;
1366                Ok(RuntimeModule {
1367                    module_id: status.module_id,
1368                    state: status.state,
1369                    enabled: status.enabled,
1370                })
1371            })
1372            .collect::<Result<Vec<_>, RouterError>>()?;
1373        // Include every registration, published or not. The capability
1374        // evaluator counts a module as providing its capabilities as soon as it
1375        // has registered, even before `catalog.list` shows it, so a HELLO in
1376        // progress does not change other modules' required-capability
1377        // verdicts. See `Registry::list_registered_modules`.
1378        let registrations = self.registry.list_registered_modules().map_err(|err| {
1379            RouterError::backend(
1380                0,
1381                0,
1382                format!("failed to list capability registrations: {err}"),
1383            )
1384        })?;
1385        let registrations = registrations
1386            .into_iter()
1387            .map(|registration| RegisteredModule {
1388                module_id: registration.manifest.module_id,
1389                module_version: registration.manifest.module_version,
1390                capabilities: registration.manifest.capabilities,
1391            })
1392            .collect();
1393        Ok((runtime, registrations))
1394    }
1395
1396    /// The capability side effects of a module becoming the active registration
1397    /// for its id: cache its manifest (warning if its claims drifted), run the
1398    /// deny census when its declarations call for one, and recompute the
1399    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1400    /// candidate's does not, and the supervisor does it at promotion instead,
1401    /// through [`crate::supervise::SwapPromotionObserver`].
1402    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1403        let cached_registration = RegisteredModule {
1404            module_id: registration.manifest.module_id.clone(),
1405            module_version: registration.manifest.module_version.clone(),
1406            capabilities: registration.manifest.capabilities.clone(),
1407        };
1408        if self.capability_evaluator.record_hello(&cached_registration) {
1409            warn!(
1410                module_id = %cached_registration.module_id,
1411                "capability claims drifted from the cached manifest"
1412            );
1413        }
1414        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1415            self.enforce_capability_denies();
1416        }
1417        self.refresh_capability_requirements();
1418    }
1419
1420    /// Point the shared supervisor handle at this handler for swap promotions.
1421    /// Called wherever a handler is put behind the `Arc` the router serves, so
1422    /// it can be held weakly.
1423    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1424        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1425            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1426        self.supervisor.set_swap_promotion_observer(observer);
1427    }
1428
1429    pub fn refresh_capability_requirements(&self) {
1430        match self.runtime_capability_snapshot() {
1431            Ok((runtime, registrations)) => {
1432                log_requirement_events(
1433                    self.capability_evaluator
1434                        .evaluate_now(&runtime, &registrations),
1435                    &self.daemon_shutdown,
1436                );
1437            }
1438            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1439        }
1440    }
1441
1442    /// Reconcile only live, attested route bindings after a capability deny edge
1443    /// or target claim was added. This is deliberately a control-plane census:
1444    /// the opaque forwarding hot path must not grow a per-frame capability check.
1445    fn enforce_capability_denies(&self) {
1446        let registrations = match self.registry.list_registered_modules() {
1447            Ok(snapshot) => snapshot,
1448            Err(err) => {
1449                warn!(error = %err, "failed to read registrations for capability deny census");
1450                return;
1451            }
1452        };
1453        let manifests = registrations
1454            .into_iter()
1455            .map(|registration| {
1456                (
1457                    registration.manifest.module_id.clone(),
1458                    registration.manifest,
1459                )
1460            })
1461            .collect::<BTreeMap<_, _>>();
1462        let census = match self.forwarding.route_census(None) {
1463            Ok(census) => census,
1464            Err(err) => {
1465                warn!(error = %err, "failed to read route census for capability deny enforcement");
1466                return;
1467            }
1468        };
1469
1470        for (target_module_id, routes) in census {
1471            let Some(target_manifest) = manifests.get(&target_module_id) else {
1472                continue;
1473            };
1474            let mut closed_routes = Vec::new();
1475            let mut goodbyes = Vec::new();
1476            for route in routes {
1477                let Principal::Reserved {
1478                    module_id: opening_module_id,
1479                } = &route.principal
1480                else {
1481                    continue;
1482                };
1483                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1484                    continue;
1485                };
1486                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1487                    continue;
1488                };
1489
1490                match self.forwarding.release_client_route(
1491                    route.goodbye_target.connection_id,
1492                    route.goodbye_target.channel,
1493                    route.goodbye_target.epoch,
1494                ) {
1495                    Ok(RouteRelease::Removed(module_goodbye)) => {
1496                        warn!(
1497                            opening_module_id,
1498                            target_module_id,
1499                            capability,
1500                            "force-closing route because an attested capability deny edge now matches"
1501                        );
1502                        goodbyes.push(module_goodbye);
1503                        goodbyes.push(route.goodbye_target.clone());
1504                        closed_routes.push(route);
1505                    }
1506                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1507                    Err(err) => warn!(
1508                        opening_module_id,
1509                        target_module_id,
1510                        capability,
1511                        error = %err,
1512                        "failed to force-close capability-denied route"
1513                    ),
1514                }
1515            }
1516
1517            if closed_routes.is_empty() {
1518                continue;
1519            }
1520            send_route_control_pushes(
1521                &self.forwarding,
1522                closed_routes,
1523                ClientControlPush::RouteClosed {
1524                    module_id: target_module_id,
1525                    channels: Vec::new(),
1526                    reason: RouteCloseReason::CapabilityDenied,
1527                    drained: false,
1528                    abandoned: 0,
1529                    excluded_subscriptions: 0,
1530                    terminal: Some(false),
1531                },
1532            );
1533            // route.closed records the reason; the route GOODBYE ends the
1534            // client's handle and settles any calls still pending on it.
1535            self.emit_route_goodbyes(goodbyes);
1536        }
1537    }
1538
1539    /// Why a registered module is not accepting new route binds, or `None` when
1540    /// it is. This is the module's effective readiness: its declared readiness
1541    /// first, then every `need: required` capability it declares evaluating to
1542    /// `provided`. `route.open` and `catalog.list` both read it here so the
1543    /// catalog never reports a module routable that `route.open` would refuse.
1544    fn not_ready_reason(
1545        &self,
1546        registration: &crate::registry::ModuleRegistration,
1547    ) -> Option<NotReadyReason> {
1548        if !registration.ready {
1549            return Some(NotReadyReason {
1550                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1551                capability: None,
1552            });
1553        }
1554        self.first_unprovided_required_capability(registration)
1555            .map(|capability| NotReadyReason {
1556                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1557                capability: Some(capability),
1558            })
1559    }
1560
1561    /// The lexicographically first capability this registration declares
1562    /// `need: required` whose evaluator verdict is not `provided`.
1563    ///
1564    /// The verdicts are the capability evaluator's own; nothing here decides
1565    /// what "provided" means. The evaluator counts a capability provided as
1566    /// soon as a module claiming it has REGISTERED, not once that module is
1567    /// ready. That distinction is what keeps two modules that require each
1568    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1569    /// is ready", each would wait for the other to become ready first and
1570    /// neither ever would. Do not tighten it to readiness.
1571    ///
1572    /// A required capability with no verdict at all means this registration's
1573    /// HELLO or catalog.update landed after the last recompute; recompute once
1574    /// rather than let a missing verdict read as either answer. If it is still
1575    /// missing (the recompute itself failed) the capability counts as
1576    /// unprovided: the refusal is retryable, and routing a module whose
1577    /// required provider is unknown is the outcome this check exists to stop.
1578    fn first_unprovided_required_capability(
1579        &self,
1580        registration: &crate::registry::ModuleRegistration,
1581    ) -> Option<String> {
1582        let required = registration
1583            .manifest
1584            .capabilities
1585            .iter()
1586            .flat_map(|declarations| declarations.requires.iter())
1587            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1588            .map(|requirement| requirement.capability.as_str())
1589            .collect::<BTreeSet<_>>();
1590        if required.is_empty() {
1591            return None;
1592        }
1593        let module_id = registration.manifest.module_id.as_str();
1594        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1595        if required
1596            .iter()
1597            .any(|capability| verdict(capability).is_none())
1598        {
1599            self.refresh_capability_requirements();
1600        }
1601        required
1602            .into_iter()
1603            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1604            .map(str::to_string)
1605    }
1606
1607    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1608        self.capability_evaluator
1609            .statuses()
1610            .into_iter()
1611            .map(capability_requirement_status)
1612            .collect()
1613    }
1614
1615    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1616    /// registration-release watch. The signal is what the supervisor waits on
1617    /// before spawning a replacement, so it must only fire once forwarding
1618    /// teardown is also done (see [`Self::cleanup_connection`] /
1619    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1620    /// admitted but its forwarding endpoint could not be installed.
1621    fn deregister_connection(
1622        &self,
1623        connection_id: ConnectionId,
1624        reason: RegistrationEndReason,
1625    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1626        self.registry
1627            .deregister_connection_with_reason(connection_id, reason)
1628    }
1629
1630    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1631        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1632            return None;
1633        }
1634        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1635            parse_client_control_request(&frame.body)
1636        else {
1637            return None;
1638        };
1639        Some(target_module_id(&target).to_string())
1640    }
1641
1642    pub(crate) fn route_open_capacity_refusal(
1643        &self,
1644        ctx: &RouteCtx,
1645        frame: &Frame,
1646        target_module_id: &str,
1647        in_flight: usize,
1648        limit: usize,
1649    ) -> Result<Frame, RouterError> {
1650        // Sent before the request reaches `handle_route_open`, so the caller
1651        // fields come from the raw frame here.
1652        let caller = self.route_open_caller_from_frame(frame);
1653        self.route_open_admission_refusal_frame(
1654            ctx,
1655            &caller,
1656            frame,
1657            target_module_id,
1658            "open_admission_full",
1659            (in_flight, limit),
1660            format!(
1661                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1662            ),
1663        )
1664    }
1665
1666    fn route_open_target_capacity_refusal(
1667        &self,
1668        ctx: &RouteCtx,
1669        caller: &RouteOpenCaller,
1670        frame: &Frame,
1671        target_module_id: &str,
1672        in_flight: usize,
1673    ) -> Result<Frame, RouterError> {
1674        self.route_open_admission_refusal_frame(
1675            ctx,
1676            caller,
1677            frame,
1678            target_module_id,
1679            "target_binds_full",
1680            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1681            format!(
1682                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1683            ),
1684        )
1685    }
1686
1687    /// Admission pressure clears as existing binds settle, so its refusal must
1688    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1689    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1690    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1691    /// cannot currently reach its target; `module_timeout` would falsely claim
1692    /// that a wait expired. A new, cleaner code would be terminal to deployed
1693    /// clients, so it requires a client-tolerance rollout before daemon emission.
1694    #[allow(clippy::too_many_arguments)]
1695    fn route_open_admission_refusal_frame(
1696        &self,
1697        ctx: &RouteCtx,
1698        caller: &RouteOpenCaller,
1699        frame: &Frame,
1700        target_module_id: &str,
1701        reason: &'static str,
1702        (in_flight, limit): (usize, usize),
1703        message: impl Into<String>,
1704    ) -> Result<Frame, RouterError> {
1705        let code = error_codes::TARGET_UNAVAILABLE;
1706        self.counters.increment_route_open_refused(code);
1707        info!(
1708            target: "control",
1709            code,
1710            reason,
1711            module_id = ?target_module_id,
1712            connection_id = ctx.connection_id.get(),
1713            principal = %caller.principal,
1714            scope = %caller.scope,
1715            harness = %caller.harness,
1716            in_flight,
1717            limit,
1718            "route.open refused"
1719        );
1720        control_error_frame(frame, code, message.into())
1721    }
1722
1723    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1724    ///
1725    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1726    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1727    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1728    #[cfg(test)]
1729    pub fn handle_control(
1730        &self,
1731        connection_id: ConnectionId,
1732        frame: Frame,
1733    ) -> Result<Vec<Frame>, RouterError> {
1734        match frame.header.ty {
1735            FrameType::Ping => Ok(vec![pong(&frame)?]),
1736            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1737            FrameType::Goodbye => self.handle_goodbye(connection_id),
1738            ty => Ok(vec![control_error_frame(
1739                &frame,
1740                "unsupported_control_frame",
1741                format!("unsupported channel-0 frame {ty:?}"),
1742            )?]),
1743        }
1744    }
1745
1746    pub async fn handle_control_frame(
1747        &self,
1748        ctx: &RouteCtx,
1749        frame: Frame,
1750    ) -> Result<Vec<Frame>, RouterError> {
1751        self.handle_control_frame_timed(ctx, frame, None).await
1752    }
1753
1754    pub(crate) fn record_caller_endpoints(
1755        &self,
1756        connection: ConnectionId,
1757        endpoints: Option<(SocketAddr, SocketAddr)>,
1758    ) {
1759        let mut callers = self
1760            .caller_endpoints
1761            .lock()
1762            .unwrap_or_else(|p| p.into_inner());
1763        callers.remove(&connection);
1764        if let Some(endpoints) = endpoints {
1765            callers.insert(connection, endpoints);
1766        }
1767    }
1768
1769    #[cfg(test)]
1770    pub(crate) fn with_caller_lookup_for_test(
1771        mut self,
1772        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1773    ) -> Self {
1774        self.caller_lookup = Some(Arc::new(lookup));
1775        self
1776    }
1777
1778    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1779        let endpoints = self
1780            .caller_endpoints
1781            .lock()
1782            .unwrap_or_else(|p| p.into_inner())
1783            .get(&connection)
1784            .copied();
1785        #[cfg(test)]
1786        let lookup = self.caller_lookup.clone();
1787        tokio::task::spawn_blocking(move || {
1788            #[cfg(test)]
1789            if let Some(lookup) = lookup {
1790                return lookup(endpoints);
1791            }
1792            endpoints
1793                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1794                .unwrap_or_default()
1795        })
1796        .await
1797        .unwrap_or_default()
1798    }
1799
1800    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1801        frame.header.channel == 0
1802            && frame.header.ty == FrameType::Request
1803            && self
1804                .forwarding
1805                .module_endpoint_for_connection(ctx.connection_id)
1806                .is_ok_and(|endpoint| endpoint.is_none())
1807            && parse_client_control_request(&frame.body)
1808                .ok()
1809                .is_some_and(|request| privileged_operation(&request).is_some())
1810    }
1811
1812    pub(crate) async fn handle_control_frame_timed(
1813        &self,
1814        ctx: &RouteCtx,
1815        frame: Frame,
1816        dispatch_started_at: Option<StdInstant>,
1817    ) -> Result<Vec<Frame>, RouterError> {
1818        match frame.header.ty {
1819            FrameType::Ping => Ok(vec![pong(&frame)?]),
1820            FrameType::Hello => {
1821                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1822            }
1823            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1824            FrameType::Cancel => {
1825                // A Cancel on channel 0 names a waiting operator.confirm or a
1826                // subscription. Successful cancellation has no ordinary reply.
1827                if self
1828                    .forwarding
1829                    .operator_confirms()
1830                    .cancel(ctx.connection_id, frame.header.corr)
1831                    || self
1832                        .supervisor
1833                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1834                    || self.cancel_activity_subscription(ctx, &frame)?
1835                {
1836                    Ok(Vec::new())
1837                } else {
1838                    Ok(vec![control_error_frame(
1839                        &frame,
1840                        "unknown_subscription",
1841                        "no subscription has this correlation id",
1842                    )?])
1843                }
1844            }
1845            FrameType::Request => {
1846                // This additive operation is not part of the existing exhaustive
1847                // module-control enum. Probe the op before decoding that enum.
1848                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1849                if op
1850                    .as_ref()
1851                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1852                {
1853                    return self.handle_activity_record(ctx, frame).await;
1854                }
1855                if op.as_ref().is_some_and(|probe| {
1856                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1857                }) {
1858                    return self.handle_activity_read(ctx, frame).await;
1859                }
1860                if op
1861                    .as_ref()
1862                    .is_some_and(|probe| probe.op == "operator.confirm")
1863                {
1864                    return self.handle_operator_confirm(ctx, frame);
1865                }
1866                if op
1867                    .as_ref()
1868                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1869                {
1870                    return self.handle_scope_served_by(ctx.connection_id, frame);
1871                }
1872                if self
1873                    .forwarding
1874                    .module_endpoint_for_connection(ctx.connection_id)
1875                    .map_err(RouterError::Forwarding)?
1876                    .is_some()
1877                {
1878                    if !is_known_module_request_op(&frame.body) {
1879                        return Ok(vec![control_error_frame(
1880                            &frame,
1881                            "unsupported_control_frame",
1882                            "module-originated channel-0 REQUEST is not supported",
1883                        )?]);
1884                    }
1885                    let request = match parse_module_control_request_from_module(&frame.body) {
1886                        Ok(request) => request,
1887                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1888                            return Ok(vec![control_error_frame(
1889                                &frame,
1890                                "unsupported_control_frame",
1891                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1892                            )?])
1893                        }
1894                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1895                            return Ok(vec![control_error_frame(
1896                                &frame,
1897                                "invalid_control_body",
1898                                format!("malformed module control body: {err}"),
1899                            )?])
1900                        }
1901                    };
1902                    let op = module_control_request_op(&request);
1903                    let corr = frame.header.corr;
1904                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1905                    let result =
1906                        self.handle_module_control_request(ctx.connection_id, frame, request);
1907                    log_slow_control_dispatch(
1908                        dispatch_started_at,
1909                        op,
1910                        ctx.connection_id,
1911                        corr,
1912                        None,
1913                    );
1914                    return result;
1915                }
1916
1917                if is_known_module_request_op(&frame.body) {
1918                    return Ok(vec![control_error_frame(
1919                        &frame,
1920                        "not_registered",
1921                        "catalog.update requires an active module registration owned by this connection",
1922                    )?]);
1923                }
1924
1925                let request = match parse_client_control_request(&frame.body) {
1926                    Ok(request) => request,
1927                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1928                        return Ok(vec![control_error_frame(
1929                            &frame,
1930                            "unknown_control_op",
1931                            format!("unknown client control op: {err}"),
1932                        )?])
1933                    }
1934                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1935                        return Ok(vec![control_error_frame(
1936                            &frame,
1937                            "invalid_control_body",
1938                            format!("malformed client control body: {err}"),
1939                        )?])
1940                    }
1941                };
1942                let op = client_control_request_op(&request);
1943                let corr = frame.header.corr;
1944                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1945                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1946                #[cfg(test)]
1947                if let Some(delay) = self.control_dispatch_delay {
1948                    tokio::time::sleep(delay).await;
1949                }
1950                let result = self
1951                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1952                    .await;
1953                if let Some(timings) = route_open_timings.as_mut() {
1954                    timings.finish();
1955                }
1956                log_slow_control_dispatch(
1957                    dispatch_started_at,
1958                    op,
1959                    ctx.connection_id,
1960                    corr,
1961                    route_open_timings.as_ref(),
1962                );
1963                result
1964            }
1965            FrameType::Push => {
1966                let Some(endpoint) = self
1967                    .forwarding
1968                    .module_endpoint_for_connection(ctx.connection_id)
1969                    .map_err(RouterError::Forwarding)?
1970                else {
1971                    return Ok(vec![control_error_frame(
1972                        &frame,
1973                        "unsupported_control_frame",
1974                        "client-originated channel-0 PUSH is not supported",
1975                    )?]);
1976                };
1977                self.handle_status_update(endpoint, frame)
1978            }
1979            FrameType::Response | FrameType::Error
1980                if self
1981                    .forwarding
1982                    .module_endpoint_for_connection(ctx.connection_id)
1983                    .map_err(RouterError::Forwarding)?
1984                    .is_some() =>
1985            {
1986                self.handle_module_relay_response(ctx.connection_id, frame)
1987            }
1988            ty => Ok(vec![control_error_frame(
1989                &frame,
1990                "unsupported_control_frame",
1991                format!("unsupported channel-0 frame {ty:?}"),
1992            )?]),
1993        }
1994    }
1995
1996    pub fn cleanup_connection(
1997        &self,
1998        connection_id: ConnectionId,
1999    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2000        self.caller_endpoints
2001            .lock()
2002            .unwrap_or_else(|p| p.into_inner())
2003            .remove(&connection_id);
2004        self.forwarding
2005            .operator_confirms()
2006            .connection_closed(connection_id);
2007        self.cleanup_connection_with_end_reason(
2008            connection_id,
2009            RegistrationEndReason::ConnectionClosed,
2010        )
2011    }
2012
2013    fn cleanup_connection_with_end_reason(
2014        &self,
2015        connection_id: ConnectionId,
2016        requested_reason: RegistrationEndReason,
2017    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2018        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2019            self.registry
2020                .get_module_by_connection(connection_id)?
2021                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2022                .and_then(|module| module.registration_end_reason().ok().flatten())
2023                .unwrap_or(requested_reason)
2024        } else {
2025            requested_reason
2026        };
2027        let crash_closed = self
2028            .registry
2029            .get_module_by_connection(connection_id)?
2030            .and_then(|registration| {
2031                self.forwarding
2032                    .module_endpoint_for_connection(connection_id)
2033                    .ok()
2034                    .flatten()
2035                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2036                    .map(|routes| (registration.manifest.module_id, routes))
2037            });
2038        let crash_closed = crash_closed.map(|(module_id, routes)| {
2039            let terminal = match self.supervisor.get(&module_id) {
2040                None => false,
2041                Some(module) => match module.will_recover_after_connection_loss() {
2042                    Ok(will_recover) => !will_recover,
2043                    Err(err) => {
2044                        warn!(
2045                            %module_id,
2046                            error = %err,
2047                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2048                        );
2049                        false
2050                    }
2051                },
2052            };
2053            // The forwarding table gates all providers at the start of daemon
2054            // shutdown, before their connections are closed. An ordinary
2055            // module disconnect still reports crash if that gate is not set.
2056            let (reason, drained) = match self
2057                .forwarding
2058                .daemon_drain_result_for_connection(connection_id)
2059            {
2060                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2061                Ok(None) => (RouteCloseReason::Crash, false),
2062                Err(err) => {
2063                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2064                    (RouteCloseReason::Crash, false)
2065                }
2066            };
2067            (module_id, routes, reason, drained, terminal)
2068        });
2069        let registrations = self.deregister_connection(connection_id, end_reason);
2070        let cleanup = if crash_closed.is_some() {
2071            self.forwarding.cleanup_connection_counted(connection_id)
2072        } else {
2073            // A module connection's teardown needs the count of abandoned
2074            // route.bind relays for its route.closed notice below. Any other
2075            // connection, such as a client's, sends no such notice and needs
2076            // only its routes released, so it uses the route-only wrapper and
2077            // reports zero.
2078            self.forwarding
2079                .cleanup_connection(connection_id)
2080                .map(|released| crate::forwarding::ConnectionCleanup {
2081                    released,
2082                    abandoned_relays: 0,
2083                })
2084        };
2085        // The route.closed push waits for forwarding teardown because only
2086        // teardown knows how many pending route.bind relays it aborted. It still
2087        // goes out before the GOODBYEs for the released routes, and its targets
2088        // were captured above, before teardown removed those routes.
2089        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2090            let abandoned = cleanup
2091                .as_ref()
2092                .map_or(0, |cleanup| cleanup.abandoned_relays);
2093            send_route_control_pushes(
2094                &self.forwarding,
2095                routes,
2096                ClientControlPush::RouteClosed {
2097                    module_id,
2098                    channels: Vec::new(),
2099                    reason,
2100                    drained,
2101                    abandoned,
2102                    excluded_subscriptions: 0,
2103                    terminal: Some(terminal),
2104                },
2105            );
2106        }
2107        if let Ok(cleanup) = cleanup {
2108            self.emit_route_goodbyes(cleanup.released);
2109        }
2110        // Signal the registration-release watch only now that BOTH registry and
2111        // forwarding teardown are done, so a supervisor waiting to spawn a
2112        // replacement never observes release while old routes still exist.
2113        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2114            crate::supervise::notify_registration_release();
2115            self.capability_evaluator.wake_deadline_loop();
2116            self.refresh_capability_requirements();
2117        }
2118        self.supervisor.remove_spawn_subscribers(connection_id);
2119        self.activity_streams
2120            .lock()
2121            .unwrap_or_else(|p| p.into_inner())
2122            .retain(|(id, _), task| {
2123                if *id == connection_id {
2124                    task.abort();
2125                    false
2126                } else {
2127                    true
2128                }
2129            });
2130        // Sync authority dies with its connection, so the owner's next
2131        // connection can take it; the owner's scopes stay as they are.
2132        self.hello_launch_nonces
2133            .lock()
2134            .unwrap_or_else(|poisoned| poisoned.into_inner())
2135            .forget(connection_id);
2136        self.scopes
2137            .write()
2138            .unwrap_or_else(|poisoned| poisoned.into_inner())
2139            .release_connection(connection_id);
2140        registrations
2141    }
2142
2143    pub(crate) fn handle_route_goodbye(
2144        &self,
2145        connection_id: ConnectionId,
2146        route_channel: u16,
2147        route_epoch: u32,
2148    ) -> Result<bool, RouterError> {
2149        debug!(
2150            connection_id = connection_id.get(),
2151            route_channel, route_epoch, "handling route GOODBYE"
2152        );
2153        let RouteRelease::Removed(released_route) = self
2154            .forwarding
2155            .release_client_route(connection_id, route_channel, route_epoch)
2156            .map_err(RouterError::Forwarding)?
2157        else {
2158            return Ok(false);
2159        };
2160        self.emit_route_goodbyes(vec![released_route]);
2161        Ok(true)
2162    }
2163
2164    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2165        for released in released_routes {
2166            let frame = match Frame::build_with_version(
2167                released.negotiated_ver,
2168                FrameType::Goodbye,
2169                control_flags(),
2170                released.channel,
2171                released.epoch,
2172                0,
2173                Vec::new(),
2174            ) {
2175                Ok(frame) => frame,
2176                Err(err) => {
2177                    warn!(
2178                        route_channel = released.channel,
2179                        error = %err,
2180                        "failed to build route GOODBYE frame"
2181                    );
2182                    continue;
2183                }
2184            };
2185            if !released.close_on_delivery_failure() {
2186                crate::forwarding::send_module_route_goodbye(
2187                    &self.counters,
2188                    &released.sink,
2189                    frame,
2190                    released.module_id.as_deref(),
2191                    "client route released",
2192                );
2193                continue;
2194            }
2195            if let Err(err) = released.sink.try_send(frame) {
2196                warn!(
2197                    target_connection_id = released.connection_id.get(),
2198                    route_channel = released.channel,
2199                    error = %err,
2200                    "route GOODBYE was not delivered to client; closing target connection"
2201                );
2202                if self
2203                    .forwarding
2204                    .escalate_client_delivery_failure(
2205                        released.connection_id,
2206                        released.channel,
2207                        released.epoch,
2208                        CloseReason::new(
2209                            "route_goodbye_delivery_failed",
2210                            format!(
2211                                "failed to enqueue route GOODBYE for channel {}: {err}",
2212                                released.channel
2213                            ),
2214                        ),
2215                        crate::forwarding::UndeliveredFrame {
2216                            module_id: released.module_id.as_deref(),
2217                            sink: &released.sink,
2218                        },
2219                    )
2220                    .unwrap_or(false)
2221                {
2222                    self.counters.increment_goodbye_relay_client_failed();
2223                }
2224            }
2225        }
2226    }
2227
2228    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2229    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2230    /// own commit failed after the module had already accepted). Without this, a
2231    /// module that accepts late keeps a binding subc has torn down, so a later
2232    /// frame on that module channel could misdeliver if the channel is reused.
2233    ///
2234    /// Never closes the shared module connection on failure: a dropped notification
2235    /// only wastes a bounded amount of warm module-side state, which the module's
2236    /// own idle reaper reclaims. Only call this once the route.bind relay was
2237    /// actually enqueued to the module — if the relay send itself failed, the
2238    /// module never created a binding and there is nothing to tear down.
2239    fn send_abandoned_route_bind_goodbye(
2240        &self,
2241        module_sink: &crate::FrameSink,
2242        negotiated_ver: u8,
2243        module_channel: u16,
2244        module_epoch: u32,
2245    ) {
2246        let frame = match Frame::build_with_version(
2247            negotiated_ver,
2248            FrameType::Goodbye,
2249            control_flags(),
2250            module_channel,
2251            module_epoch,
2252            0,
2253            Vec::new(),
2254        ) {
2255            Ok(frame) => frame,
2256            Err(err) => {
2257                warn!(
2258                    route_channel = module_channel,
2259                    error = %err,
2260                    "failed to build GOODBYE for abandoned route.bind"
2261                );
2262                return;
2263            }
2264        };
2265        crate::forwarding::send_module_route_goodbye(
2266            &self.counters,
2267            module_sink,
2268            frame,
2269            None,
2270            "abandoned route.bind",
2271        );
2272    }
2273
2274    fn handle_hello(
2275        &self,
2276        connection_id: ConnectionId,
2277        sink: Option<crate::FrameSink>,
2278        frame: Frame,
2279    ) -> Result<Vec<Frame>, RouterError> {
2280        debug!(
2281            connection_id = connection_id.get(),
2282            corr = frame.header.corr,
2283            "handling HELLO"
2284        );
2285        // A module connection has one identity for its entire lifetime. A second
2286        // registration would leave the old registry owner behind while replacing
2287        // its forwarding endpoint and launch nonce.
2288        if self
2289            .registry
2290            .get_module_by_connection(connection_id)
2291            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2292            .is_some()
2293        {
2294            return Ok(vec![control_error_frame(
2295                &frame,
2296                "invalid_hello",
2297                "connection is already registered as a module",
2298            )?]);
2299        }
2300        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2301            Ok(value) => value,
2302            Err(err) => {
2303                return Ok(vec![control_error_frame(
2304                    &frame,
2305                    "invalid_hello",
2306                    format!("malformed HELLO body: {err}"),
2307                )?])
2308            }
2309        };
2310        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2311            return Ok(vec![control_error_frame(
2312                &frame,
2313                "invalid_capability_grammar",
2314                err.to_string(),
2315            )?]);
2316        }
2317        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2318            return Ok(vec![control_error_frame(
2319                &frame,
2320                "invalid_manifest",
2321                err.to_string(),
2322            )?]);
2323        }
2324        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2325            return Ok(vec![control_error_frame(
2326                &frame,
2327                "invalid_event_declaration",
2328                err.to_string(),
2329            )?]);
2330        }
2331        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2332            return Ok(vec![control_error_frame(
2333                &frame,
2334                "invalid_manifest",
2335                err.to_string(),
2336            )?]);
2337        }
2338        if let Some(provenance) = hello_value
2339            .get("manifest")
2340            .and_then(|manifest| manifest.get("provenance"))
2341        {
2342            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2343                return Ok(vec![control_error_frame(
2344                    &frame,
2345                    "invalid_manifest",
2346                    format!("malformed manifest provenance: {err}"),
2347                )?]);
2348            }
2349        }
2350        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2351            let module_id = hello_value["manifest"]["module_id"]
2352                .as_str()
2353                .unwrap_or("<unknown>");
2354            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2355        }
2356        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2357            Ok(hello) => hello,
2358            Err(err) => {
2359                return Ok(vec![control_error_frame(
2360                    &frame,
2361                    "invalid_hello",
2362                    format!("malformed HELLO body: {err}"),
2363                )?])
2364            }
2365        };
2366
2367        if hello.protocol_ver != hello.manifest.protocol_ver {
2368            return Ok(vec![control_error_frame(
2369                &frame,
2370                "invalid_manifest",
2371                format!(
2372                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2373                    hello.protocol_ver, hello.manifest.protocol_ver
2374                ),
2375            )?]);
2376        }
2377
2378        if hello.manifest.module_id.trim().is_empty() {
2379            return Ok(vec![control_error_frame(
2380                &frame,
2381                "invalid_manifest",
2382                "manifest module_id must not be empty",
2383            )?]);
2384        }
2385
2386        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2387            Ok(negotiated_ver) => negotiated_ver,
2388            Err(message) => {
2389                return Ok(vec![control_error_frame(
2390                    &frame,
2391                    "version_unsupported",
2392                    message,
2393                )?])
2394            }
2395        };
2396
2397        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2398        // swap is open for this id, the only HELLO admitted as a second process
2399        // is the one carrying the candidate's launch nonce (the swap token), and
2400        // it registers into the candidate slot rather than being refused as a
2401        // duplicate. Run after the reserved gate, a reserved module's candidate
2402        // would be refused `reserved_module` for presenting a nonce that gate
2403        // does not know. See `SupervisorHandle::swap_hello_admission`.
2404        let swap_admission = self
2405            .supervisor
2406            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2407        if swap_admission == SwapHelloAdmission::Refused {
2408            warn!(
2409                module_id = %hello.manifest.module_id,
2410                connection_id = connection_id.get(),
2411                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2412            );
2413            return Ok(vec![control_error_frame(
2414                &frame,
2415                "swap_token_invalid",
2416                format!(
2417                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2418                    hello.manifest.module_id
2419                ),
2420            )?]);
2421        }
2422        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2423
2424        // A swap candidate proves its identity with the candidate nonce above.
2425        // Other registrations receive this catalog fact only when reserved-module
2426        // policy required and accepted a nonce; unreserved registrations require
2427        // no nonce check.
2428        let nonce_verified = if swap_candidate {
2429            true
2430        } else {
2431            match self.supervisor.reserved_hello_verification(
2432                &hello.manifest.module_id,
2433                hello.launch_nonce.as_deref(),
2434            ) {
2435                Ok(verified) => verified,
2436                Err(rejection) => {
2437                    let message = match rejection {
2438                        ReservedHelloRejection::Exact { module_id } => format!(
2439                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2440                        ),
2441                        ReservedHelloRejection::Prefix {
2442                            prefix,
2443                            owner_module_id,
2444                        } => format!(
2445                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2446                            hello.manifest.module_id
2447                        ),
2448                    };
2449                    return Ok(vec![control_error_frame(
2450                        &frame,
2451                        "reserved_module",
2452                        message,
2453                    )?]);
2454                }
2455            }
2456        };
2457
2458        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2459            &hello.manifest.module_id,
2460            hello.manifest.capabilities.as_ref(),
2461        );
2462        if let Some(refusal) = reserved_capability_refusals.first() {
2463            let capability = refusal.capability.clone();
2464            let bound_module = refusal.claimants[0].clone();
2465            log_duplicate_claim_events(reserved_capability_refusals);
2466            return Ok(vec![control_error_frame(
2467                &frame,
2468                "reserved_capability",
2469                format!(
2470                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2471                    capability, bound_module, hello.manifest.module_id
2472                ),
2473            )?]);
2474        }
2475
2476        // A connection that already opened client routes must not also register as
2477        // a module: cleanup would then release only one side and leak the other.
2478        if self
2479            .forwarding
2480            .connection_has_client_routes(connection_id)
2481            .map_err(RouterError::Forwarding)?
2482        {
2483            return Ok(vec![control_error_frame(
2484                &frame,
2485                "invalid_hello",
2486                "connection has open client routes and cannot also register as a module",
2487            )?]);
2488        }
2489
2490        // Kept for scope sync authority, which goes only to the connection that
2491        // presented the module's current launch nonce. Recorded before the
2492        // registration is attempted: a connection whose registration then fails
2493        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2494        self.hello_launch_nonces
2495            .lock()
2496            .unwrap_or_else(|poisoned| poisoned.into_inner())
2497            .record(connection_id, hello.launch_nonce.as_deref());
2498        let control_ops = effective_module_control_ops(hello.control_ops);
2499        // Built before anything is registered so an encoding failure leaves no
2500        // registry or forwarding state behind.
2501        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2502        if swap_candidate {
2503            return self.register_swap_candidate(
2504                connection_id,
2505                sink,
2506                &frame,
2507                hello.manifest,
2508                negotiated_ver,
2509                control_ops,
2510                hello_ack,
2511            );
2512        }
2513        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2514        // is refused, but `catalog.list` does not show the module and the
2515        // catalog generation does not move until it is published below, once
2516        // its forwarding endpoint exists. Listing it now would advertise a
2517        // module that `route.open` still refuses for having no forwarding
2518        // connection.
2519        let registration = match self
2520            .registry
2521            .register_unpublished_verified_with_control_ops(
2522                hello.manifest,
2523                negotiated_ver,
2524                connection_id,
2525                control_ops,
2526                nonce_verified,
2527            ) {
2528            Ok(registration) => registration,
2529            Err(RegistryError::DuplicateModuleId { module_id }) => {
2530                return Ok(vec![control_error_frame(
2531                    &frame,
2532                    "duplicate_module_id",
2533                    format!(
2534                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2535                    ),
2536                )?])
2537            }
2538            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2539                return Ok(vec![control_error_frame(
2540                    &frame,
2541                    "invalid_module_id",
2542                    err.to_string(),
2543                )?])
2544            }
2545            Err(err) => {
2546                return Ok(vec![control_error_frame(
2547                    &frame,
2548                    "registry_error",
2549                    err.to_string(),
2550                )?])
2551            }
2552        };
2553        #[cfg(test)]
2554        self.hold_between_registry_and_forwarding();
2555
2556        let reply = if let Some(sink) = sink {
2557            // The forwarding table's module store is also the daemon-to-module
2558            // control-RPC lane, so every HELLO gets a live endpoint even when the
2559            // manifest has no routable provider role. Non-routable modules still
2560            // cannot receive route.bind in production: `handle_route_open` checks
2561            // the registry manifest with `target_has_required_role` before the
2562            // only production call to `begin_route_bind_relay_for` below that
2563            // route.open path. The remaining direct relay callers are unit tests
2564            // and benchmark harnesses that construct forwarding state explicitly.
2565            //
2566            // The HELLO_ACK is queued by the forwarding table itself, before the
2567            // endpoint becomes visible, and is NOT returned as a reply. A module
2568            // reads HELLO_ACK first and exits on anything else; a reply is only
2569            // written after this handler returns, by which time a route.open on
2570            // another connection could already have queued a route.bind request
2571            // for this module ahead of it.
2572            let concurrency = manifest_concurrency(&registration.manifest);
2573            if let Err(err) = self.forwarding.register_module_connection_acked(
2574                connection_id,
2575                registration.manifest.module_id.clone(),
2576                negotiated_ver,
2577                concurrency,
2578                sink,
2579                hello_ack,
2580            ) {
2581                // Forwarding registration failed, so there is no forwarding
2582                // state to tear down. Remove the registry entry, which was never
2583                // published and so never listed or counted in the catalog
2584                // generation, and signal the release watch directly.
2585                if matches!(
2586                    self.deregister_connection(
2587                        connection_id,
2588                        RegistrationEndReason::RegistrationFailed,
2589                    ),
2590                    Ok(r) if !r.is_empty()
2591                ) {
2592                    crate::supervise::notify_registration_release();
2593                }
2594                return Ok(vec![control_error_frame(
2595                    &frame,
2596                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2597                        "invalid_hello"
2598                    } else {
2599                        forwarding_error_code(&err)
2600                    },
2601                    err.to_string(),
2602                )?]);
2603            }
2604            Vec::new()
2605        } else {
2606            // No sink means no forwarding endpoint, so nothing can be routed
2607            // ahead of the ack; it goes out as the reply.
2608            vec![hello_ack]
2609        };
2610
2611        // The module is routable now (or, without a sink, as routable as it
2612        // will ever be), so the catalog may list it. Publishing bumps the
2613        // catalog generation in the same registry critical section that makes
2614        // the module listed.
2615        if !self
2616            .registry
2617            .publish_registration(connection_id)
2618            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2619        {
2620            debug!(
2621                module_id = %registration.manifest.module_id,
2622                connection_id = connection_id.get(),
2623                "registration was removed before it could be published"
2624            );
2625        }
2626
2627        // Exposure over assumption: Concurrency's serde default is pinned to the
2628        // pre-field behavior (ModuleManaged), so a management surface that is
2629        // genuinely Serial and just never declared it inherits concurrent
2630        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2631        // turns "no module has been bitten yet" into the checkable claim "no
2632        // module is exposed" -- one read of the boot log instead of a fleet
2633        // audit. Detected from the raw HELLO bytes because the serde default
2634        // deliberately erases the absent/declared distinction from the type.
2635        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2636            info!(
2637                module_id = %registration.manifest.module_id,
2638                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2639            );
2640        }
2641
2642        self.apply_registration_capabilities(&registration);
2643
2644        self.supervisor
2645            .record_activity_registration(&registration.manifest.module_id);
2646
2647        info!(
2648            module_id = %registration.manifest.module_id,
2649            module_version = %registration.manifest.module_version,
2650            negotiated_ver,
2651            routable_provider = manifest_provides_routable_role(&registration.manifest),
2652            connection_id = connection_id.get(),
2653            "module registered"
2654        );
2655
2656        Ok(reply)
2657    }
2658
2659    /// Register a HELLO the swap gate admitted into the candidate slot of the
2660    /// registry and of forwarding, where it is reachable over its own
2661    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2662    /// nothing routes to it until the supervisor cuts over.
2663    ///
2664    /// Registry first (unpublished), then forwarding, then publication, the
2665    /// same order as an ordinary HELLO; a forwarding failure removes the
2666    /// registry entry again. The swap supervisor treats a published, ready
2667    /// candidate as registered and promotes it, and promotion needs the
2668    /// candidate's forwarding endpoint, so the candidate is published only
2669    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2670    /// recompute capability verdicts or run the capability deny census
2671    /// (`enforce_capability_denies`): both are about the module in the active
2672    /// slot, which `route.open` can target. A candidate sits in the candidate
2673    /// slot, so it neither provides capabilities to other modules nor can be
2674    /// routed to until promotion moves it into the active slot.
2675    #[allow(clippy::too_many_arguments)]
2676    fn register_swap_candidate(
2677        &self,
2678        connection_id: ConnectionId,
2679        sink: Option<crate::FrameSink>,
2680        frame: &Frame,
2681        manifest: ModuleManifest,
2682        negotiated_ver: u8,
2683        control_ops: Vec<String>,
2684        hello_ack: Frame,
2685    ) -> Result<Vec<Frame>, RouterError> {
2686        let module_id = manifest.module_id.clone();
2687        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2688            manifest,
2689            negotiated_ver,
2690            connection_id,
2691            control_ops,
2692        ) {
2693            Ok(registration) => registration,
2694            Err(RegistryError::DuplicateModuleId { module_id }) => {
2695                return Ok(vec![control_error_frame(
2696                    frame,
2697                    "duplicate_module_id",
2698                    format!(
2699                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2700                    ),
2701                )?])
2702            }
2703            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2704                return Ok(vec![control_error_frame(
2705                    frame,
2706                    "invalid_module_id",
2707                    err.to_string(),
2708                )?])
2709            }
2710            Err(err) => {
2711                return Ok(vec![control_error_frame(
2712                    frame,
2713                    "registry_error",
2714                    err.to_string(),
2715                )?])
2716            }
2717        };
2718        #[cfg(test)]
2719        self.hold_between_registry_and_forwarding();
2720        let reply = if let Some(sink) = sink {
2721            // Same ordering as an ordinary HELLO: the forwarding table queues
2722            // the HELLO_ACK before the candidate endpoint is inserted, because
2723            // a module exits if its first frame after HELLO is anything else.
2724            let concurrency = manifest_concurrency(&registration.manifest);
2725            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2726                connection_id,
2727                module_id.clone(),
2728                negotiated_ver,
2729                concurrency,
2730                sink,
2731                hello_ack,
2732            ) {
2733                if matches!(
2734                    self.deregister_connection(
2735                        connection_id,
2736                        RegistrationEndReason::RegistrationFailed,
2737                    ),
2738                    Ok(r) if !r.is_empty()
2739                ) {
2740                    crate::supervise::notify_registration_release();
2741                }
2742                return Ok(vec![control_error_frame(
2743                    frame,
2744                    forwarding_error_code(&err),
2745                    err.to_string(),
2746                )?]);
2747            }
2748            Vec::new()
2749        } else {
2750            vec![hello_ack]
2751        };
2752        if !self
2753            .registry
2754            .publish_registration(connection_id)
2755            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2756        {
2757            debug!(
2758                module_id = %module_id,
2759                connection_id = connection_id.get(),
2760                "swap candidate registration was removed before it could be published"
2761            );
2762        }
2763        self.supervisor.mark_swap_candidate_admitted(&module_id);
2764        info!(
2765            module_id = %module_id,
2766            module_version = %registration.manifest.module_version,
2767            negotiated_ver,
2768            ready = registration.ready,
2769            connection_id = connection_id.get(),
2770            "swap candidate registered; not routable until cutover"
2771        );
2772        Ok(reply)
2773    }
2774
2775    fn build_hello_ack(
2776        &self,
2777        frame: &Frame,
2778        negotiated_ver: u8,
2779        module_id: &str,
2780    ) -> Result<Frame, RouterError> {
2781        let ack = ModuleHelloAckBody {
2782            negotiated_ver,
2783            subc_ops: module_subc_ops(),
2784            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2785            storage: self
2786                .storage_config
2787                .as_ref()
2788                .map(|cfg| cfg.descriptor_for(module_id)),
2789            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2790        };
2791        let body = serde_json::to_vec(&ack).map_err(|err| {
2792            RouterError::backend(
2793                0,
2794                frame.header.corr,
2795                format!("failed to encode HELLO_ACK: {err}"),
2796            )
2797        })?;
2798
2799        Frame::build_with_version(
2800            negotiated_ver,
2801            FrameType::HelloAck,
2802            control_flags(),
2803            0,
2804            0,
2805            frame.header.corr,
2806            body,
2807        )
2808        .map_err(RouterError::FrameBuild)
2809    }
2810
2811    async fn handle_client_control_request(
2812        &self,
2813        ctx: &RouteCtx,
2814        frame: Frame,
2815        request: ClientControlRequest,
2816        route_open_timings: &mut Option<RouteOpenTimings>,
2817    ) -> Result<Vec<Frame>, RouterError> {
2818        if let Some(operation) = privileged_operation(&request) {
2819            let caller = self.resolve_caller(ctx.connection_id).await;
2820            let op = client_control_request_op(&request);
2821            let text = prompt_text(operation, &caller);
2822            match self.privileged_operator_prompt {
2823                PrivilegedOperatorPrompt::Log => {
2824                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2825                        target = ?operation, caller = %caller.description(), summary = %text,
2826                        "privileged request would prompt the operator");
2827                }
2828                PrivilegedOperatorPrompt::Enforce => {
2829                    let key =
2830                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2831                    let outcome = self
2832                        .forwarding
2833                        .operator_confirms()
2834                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2835                        .await;
2836                    if outcome.code != "confirmed" {
2837                        return Ok(vec![outcome.frame(&frame)]);
2838                    }
2839                }
2840            }
2841        }
2842        match request {
2843            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2844            ClientControlRequest::CatalogList { module_id } => {
2845                self.handle_catalog_list(frame, module_id)
2846            }
2847            ClientControlRequest::RouteOpen {
2848                target,
2849                identity,
2850                consumer_identity,
2851                consumer_capabilities,
2852                role_versions,
2853                admission_facts,
2854                scope,
2855            } => {
2856                self.handle_route_open(
2857                    ctx,
2858                    frame,
2859                    RouteOpenRequest {
2860                        target,
2861                        identity,
2862                        consumer_identity,
2863                        consumer_capabilities,
2864                        role_versions,
2865                        admission_facts,
2866                        scope,
2867                    },
2868                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2869                )
2870                .await
2871            }
2872            ClientControlRequest::RoutePoll {
2873                route_channel,
2874                route_epoch,
2875                kind,
2876            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2877            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2878            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2879                self.handle_supervisor_spawn_snapshot(frame)
2880            }
2881            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2882                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2883            }
2884            ClientControlRequest::SupervisorRestart {
2885                module_id,
2886                drain_timeout_ms,
2887            } => {
2888                self.log_supervisor_request_received(
2889                    ctx,
2890                    frame.header.corr,
2891                    ops::SUPERVISOR_RESTART,
2892                    Some(&module_id),
2893                    None,
2894                )?;
2895                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2896                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2897                    .await
2898            }
2899            ClientControlRequest::SupervisorSwap {
2900                module_id,
2901                ready_timeout_ms,
2902            } => {
2903                self.log_supervisor_request_received(
2904                    ctx,
2905                    frame.header.corr,
2906                    ops::SUPERVISOR_SWAP,
2907                    Some(&module_id),
2908                    None,
2909                )?;
2910                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2911                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2912                    .await
2913            }
2914            ClientControlRequest::SupervisorReload { module_id } => {
2915                self.log_supervisor_request_received(
2916                    ctx,
2917                    frame.header.corr,
2918                    ops::SUPERVISOR_RELOAD,
2919                    Some(&module_id),
2920                    None,
2921                )?;
2922                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2923                self.handle_supervisor_reload(frame, module_id, requester)
2924                    .await
2925            }
2926            ClientControlRequest::SupervisorRescan { preview } => {
2927                if !preview {
2928                    self.log_supervisor_request_received(
2929                        ctx,
2930                        frame.header.corr,
2931                        ops::SUPERVISOR_RESCAN,
2932                        None,
2933                        None,
2934                    )?;
2935                }
2936                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2937                self.handle_supervisor_rescan(
2938                    frame,
2939                    if preview {
2940                        RescanMode::Preview
2941                    } else {
2942                        RescanMode::Execute
2943                    },
2944                    requester,
2945                )
2946                .await
2947            }
2948            ClientControlRequest::SupervisorRescanChecked {
2949                preview,
2950                candidate,
2951                expect,
2952            } => {
2953                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2954                    .await
2955            }
2956            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2957                self.log_supervisor_request_received(
2958                    ctx,
2959                    frame.header.corr,
2960                    ops::SUPERVISOR_RELEASE_RESERVED,
2961                    Some(&module_id),
2962                    None,
2963                )?;
2964                self.handle_supervisor_release_reserved(frame, module_id)
2965                    .await
2966            }
2967            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2968                self.log_supervisor_request_received(
2969                    ctx,
2970                    frame.header.corr,
2971                    ops::SUPERVISOR_SET_ENABLED,
2972                    Some(&module_id),
2973                    Some(enabled),
2974                )?;
2975                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2976                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2977                    .await
2978            }
2979            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2980                self.handle_supervisor_health_probe(frame, module_id).await
2981            }
2982            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2983            ClientControlRequest::SupervisorRoutes { module_id } => {
2984                self.handle_supervisor_routes(frame, module_id)
2985            }
2986            ClientControlRequest::SupervisorProvenance { module_id } => {
2987                self.handle_supervisor_provenance(frame, module_id).await
2988            }
2989            ClientControlRequest::SupervisorStderrTail {
2990                module_id,
2991                max_lines,
2992                max_bytes,
2993            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2994            ClientControlRequest::SupervisorTerminals { module_id } => {
2995                self.handle_supervisor_terminals(frame, module_id).await
2996            }
2997        }
2998    }
2999
3000    fn log_supervisor_request_received(
3001        &self,
3002        ctx: &RouteCtx,
3003        corr: u64,
3004        op: &'static str,
3005        module_id: Option<&str>,
3006        enabled: Option<bool>,
3007    ) -> Result<(), RouterError> {
3008        let caller = self.supervisor_requester(ctx, corr)?;
3009        let caller = principal_label(&caller);
3010        // `Option` fields are recorded only when present, so a request with no
3011        // module id (rescan) or no enabled flag simply omits that field.
3012        info!(
3013            target: "control",
3014            op,
3015            module_id,
3016            enabled,
3017            connection_id = ctx.connection_id.get(),
3018            caller = %caller,
3019            "supervisor request received"
3020        );
3021        Ok(())
3022    }
3023
3024    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3025        Ok(self
3026            .registry
3027            .get_module_by_connection(ctx.connection_id)
3028            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3029            .map(|registration| Principal::Reserved {
3030                module_id: registration.manifest.module_id,
3031            })
3032            .unwrap_or(Principal::Direct))
3033    }
3034
3035    fn handle_module_control_request(
3036        &self,
3037        connection_id: ConnectionId,
3038        frame: Frame,
3039        request: ModuleControlRequestFromModule,
3040    ) -> Result<Vec<Frame>, RouterError> {
3041        match request {
3042            ModuleControlRequestFromModule::CatalogUpdate {
3043                provides,
3044                capabilities,
3045                ready,
3046            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3047            ModuleControlRequestFromModule::LiveRoots {} => {
3048                let registered = self
3049                    .registry
3050                    .get_module_by_connection(connection_id)
3051                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3052                let Some(registration) = registered else {
3053                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3054                };
3055                let response = self
3056                    .forwarding
3057                    .live_roots(&registration.manifest.module_id)
3058                    .map_err(RouterError::Forwarding)?;
3059                Ok(vec![control_response_body_frame(
3060                    &frame,
3061                    &response,
3062                    "ModuleControlResponseToModule::LiveRoots",
3063                )?])
3064            }
3065            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3066                self.handle_scope_sync(connection_id, frame, generation, scopes)
3067            }
3068            ModuleControlRequestFromModule::ScopeApply {
3069                generation,
3070                upsert,
3071                end,
3072            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3073            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3074                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3075            }
3076        }
3077    }
3078
3079    fn handle_scope_served_by(
3080        &self,
3081        connection_id: ConnectionId,
3082        frame: Frame,
3083    ) -> Result<Vec<Frame>, RouterError> {
3084        let registered = self
3085            .registry
3086            .get_module_by_connection(connection_id)
3087            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3088        let Some(registered) = registered else {
3089            return Ok(vec![control_error_frame(
3090                &frame,
3091                "not_registered",
3092                "scope.served_by requires an active module registration owned by this connection",
3093            )?]);
3094        };
3095        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3096        {
3097            Ok(request) => request,
3098            Err(err) => {
3099                return Ok(vec![control_error_frame(
3100                    &frame,
3101                    "invalid_control_body",
3102                    format!("malformed scope.served_by body: {err}"),
3103                )?])
3104            }
3105        };
3106        let answer = self
3107            .scopes
3108            .read()
3109            .unwrap_or_else(|poisoned| poisoned.into_inner())
3110            .served_by(
3111                &registered.manifest.module_id,
3112                &request.scope_ref,
3113                request.scope_epoch,
3114                &request.module_id,
3115            );
3116        let reply = subc_control::ScopeServedByReply::new(
3117            answer,
3118            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3119        );
3120        Ok(vec![control_response_body_frame(
3121            &frame,
3122            &reply,
3123            "ScopeServedByReply",
3124        )?])
3125    }
3126
3127    fn handle_operator_confirm(
3128        &self,
3129        ctx: &RouteCtx,
3130        frame: Frame,
3131    ) -> Result<Vec<Frame>, RouterError> {
3132        use crate::operator_confirm::{audit, Outcome};
3133        let registration = self
3134            .registry
3135            .get_module_by_connection(ctx.connection_id)
3136            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3137        let Some(registration) = registration else {
3138            let outcome = Outcome::refusal("not_registered");
3139            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3140            return Ok(vec![outcome.frame(&frame)]);
3141        };
3142        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3143            Ok(request) => request,
3144            Err(_) => {
3145                let outcome = Outcome::refusal("invalid_control_body");
3146                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3147                return Ok(vec![outcome.frame(&frame)]);
3148            }
3149        };
3150        let module_id = registration.manifest.module_id;
3151        // Read the launch nonce (under its own lock) before taking the forwarding
3152        // table's lock below: holding forwarding while waiting on another daemon
3153        // lock risks a lock-order deadlock with paths that take them the other way.
3154        let nonce = self
3155            .hello_launch_nonces
3156            .lock()
3157            .unwrap_or_else(|p| p.into_inner())
3158            .nonce(ctx.connection_id)
3159            .map(str::to_owned);
3160        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3161            self.supervisor
3162                .spawned_consumer_authorized(&module_id, nonce)
3163        });
3164        let confirms = self.forwarding.operator_confirms();
3165        self.forwarding
3166            .with_operator_route(
3167                ctx.connection_id,
3168                request.route_channel,
3169                request.route_epoch,
3170                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3171            )
3172            .map_err(RouterError::Forwarding)
3173    }
3174
3175    /// Resolve folders before taking the scope-table lock: a slow filesystem
3176    /// must not hold up admissions for unrelated scoped routes.
3177    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3178        let mut scopes = ScopeRecords::from(scopes);
3179        for (index, record) in scopes.records.iter_mut().enumerate() {
3180            if let Some(refusal) = Self::canonicalize_workdir(record) {
3181                scopes.refusals.insert(index, refusal);
3182            }
3183        }
3184        scopes
3185    }
3186
3187    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3188        let workdir = record.workdir.as_mut()?;
3189        let path = std::path::Path::new(workdir);
3190        if !path.is_absolute() {
3191            return Some((
3192                error_codes::INVALID_WORKDIR,
3193                "workdir must be absolute".to_string(),
3194            ));
3195        }
3196        match ProjectRootId::from_path_allowing_missing(path) {
3197            Ok(root) => match root.as_path().to_str() {
3198                Some(canonical) => {
3199                    *workdir = canonical.to_string();
3200                    None
3201                }
3202                None => Some((
3203                    error_codes::INVALID_WORKDIR,
3204                    "canonical workdir is not UTF-8".to_string(),
3205                )),
3206            },
3207            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3208        }
3209    }
3210
3211    /// `scope.sync`: the owner is the module registered on this connection.
3212    /// A connection with no registration (every client connection, `direct`
3213    /// included) is refused `not_registered` before the table is consulted.
3214    fn handle_scope_sync(
3215        &self,
3216        connection_id: ConnectionId,
3217        frame: Frame,
3218        generation: u64,
3219        scopes: Vec<ScopeRecord>,
3220    ) -> Result<Vec<Frame>, RouterError> {
3221        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3222    }
3223
3224    fn handle_scope_change(
3225        &self,
3226        connection_id: ConnectionId,
3227        frame: Frame,
3228        generation: u64,
3229        scopes: Vec<ScopeRecord>,
3230        end: Option<Vec<ScopeEnd>>,
3231    ) -> Result<Vec<Frame>, RouterError> {
3232        let is_apply = end.is_some();
3233        let Some(registration) = self
3234            .registry
3235            .get_module_by_connection(connection_id)
3236            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3237        else {
3238            return Ok(vec![control_error_frame(
3239                &frame,
3240                "not_registered",
3241                "scope.sync and scope.apply require an active module registration owned by this connection",
3242            )?]);
3243        };
3244        let owner = registration.manifest.module_id;
3245        let scopes = Self::prepare_scope_records(scopes);
3246        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3247        let is_current_launch = |connection: ConnectionId| {
3248            self.hello_launch_nonces
3249                .lock()
3250                .unwrap_or_else(|poisoned| poisoned.into_inner())
3251                .presented(connection, current_nonce.as_deref())
3252        };
3253        // Lock order is the scope table, then the forwarding table: the new
3254        // tags are published, and the routes the change closes are selected,
3255        // while the scope table is still write-locked, so no admission can read
3256        // a record whose tag is not yet published.
3257        let mut table = self
3258            .scopes
3259            .write()
3260            .unwrap_or_else(|poisoned| poisoned.into_inner());
3261        let outcome = match end {
3262            Some(end) => table.apply(
3263                &owner,
3264                connection_id,
3265                is_current_launch,
3266                generation,
3267                scopes,
3268                end,
3269            ),
3270            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3271        };
3272        let drained = match &outcome {
3273            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3274            Err(_) => Vec::new(),
3275        };
3276        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3277        drop(table);
3278        if has_deadlines {
3279            self.scope_deadline_added.notify_one();
3280        }
3281        match outcome {
3282            Ok(applied) => {
3283                let counts = ScopeOutcomeCounts::of(&applied.results);
3284                info!(
3285                    owner = %owner,
3286                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3287                    generation,
3288                    records = applied.results.len(),
3289                    created = counts.created,
3290                    replaced = counts.replaced,
3291                    updated = counts.updated,
3292                    unchanged = counts.unchanged,
3293                    refused = counts.refused,
3294                    ended = applied.ended.len(),
3295                    tag_changes = applied.tag_changes.len(),
3296                    routes_closed = drained.len(),
3297                    "scope change accepted"
3298                );
3299                // An accepted scope change can still refuse individual records, and the
3300                // owner is the only party that sees the reply. Name them here so
3301                // an operator can tell a refused session from a missing one
3302                // without the owner's logs. Capped so a call that refuses
3303                // thousands cannot flood the log; the count above is complete.
3304                for refused in applied
3305                    .results
3306                    .iter()
3307                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3308                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3309                {
3310                    warn!(
3311                        owner = %owner,
3312                        generation,
3313                        scope_ref = %refused.scope_ref,
3314                        scope_epoch = refused.scope_epoch,
3315                        code = refused.code.as_deref().unwrap_or(""),
3316                        "scope record refused"
3317                    );
3318                }
3319                self.close_scope_drained_routes(drained);
3320                let response = if is_apply {
3321                    ModuleControlResponseToModule::ScopeApply {
3322                        generation,
3323                        results: applied.results,
3324                        end_results: applied.end_results,
3325                        ended: applied.ended,
3326                    }
3327                } else {
3328                    ModuleControlResponseToModule::ScopeSync {
3329                        generation,
3330                        results: applied.results,
3331                        ended: applied.ended,
3332                    }
3333                };
3334                Ok(vec![control_response_body_frame(
3335                    &frame,
3336                    &response,
3337                    "scope change response",
3338                )?])
3339            }
3340            Err(refusal) => {
3341                info!(
3342                    owner = %owner,
3343                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3344                    generation,
3345                    code = refusal.code,
3346                    "scope change refused"
3347                );
3348                Ok(vec![control_error_frame(
3349                    &frame,
3350                    refusal.code,
3351                    refusal.message,
3352                )?])
3353            }
3354        }
3355    }
3356
3357    /// Tell both ends of each route a scope change closed. The module gets a
3358    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3359    /// the client's route handle. The client also gets `route.closed` with the
3360    /// scope reason, one push per module and reason, so it can tell a revoked
3361    /// route from an ordinary close and not reopen it.
3362    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3363        if drained.is_empty() {
3364            return;
3365        }
3366        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3367            BTreeMap::new();
3368        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3369        for route in drained {
3370            warn!(
3371                module_id = %route.module_id,
3372                reason = ?route.reason,
3373                client_connection_id = route.client.connection_id.get(),
3374                route_channel = route.client.channel,
3375                "closing route because its scope changed"
3376            );
3377            pushes
3378                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3379                .or_insert_with(|| (route.reason, Vec::new()))
3380                .1
3381                .push(EndpointRoute {
3382                    goodbye_target: route.client.clone(),
3383                    principal: Principal::Unverified,
3384                    bound_at: Instant::now(),
3385                    draining: false,
3386                    drain_reason: None,
3387                });
3388            goodbyes.push(route.module);
3389            goodbyes.push(route.client);
3390        }
3391        for ((module_id, _), (reason, routes)) in pushes {
3392            send_route_control_pushes(
3393                &self.forwarding,
3394                routes,
3395                ClientControlPush::RouteClosed {
3396                    module_id,
3397                    channels: Vec::new(),
3398                    reason,
3399                    drained: false,
3400                    abandoned: 0,
3401                    excluded_subscriptions: 0,
3402                    terminal: Some(false),
3403                },
3404            );
3405        }
3406        self.emit_route_goodbyes(goodbyes);
3407    }
3408
3409    /// `scope.describe`: any registered module may read any scope, because a
3410    /// provider must read the scope a route it serves is stamped with.
3411    fn handle_scope_describe(
3412        &self,
3413        connection_id: ConnectionId,
3414        frame: Frame,
3415        owner: Principal,
3416        scope_ref: String,
3417    ) -> Result<Vec<Frame>, RouterError> {
3418        let registered = self
3419            .registry
3420            .get_module_by_connection(connection_id)
3421            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3422        let Some(registered) = registered else {
3423            return Ok(vec![control_error_frame(
3424                &frame,
3425                "not_registered",
3426                "scope.describe requires an active module registration owned by this connection",
3427            )?]);
3428        };
3429        let mut description = self
3430            .scopes
3431            .read()
3432            .unwrap_or_else(|poisoned| poisoned.into_inner())
3433            .describe(&owner, &scope_ref);
3434        // Describe readers also decode strict ScopeAttributes. Attribution is
3435        // safe to omit unless the requesting module declares it understands it.
3436        if !registered
3437            .manifest
3438            .capabilities
3439            .as_ref()
3440            .is_some_and(|caps| {
3441                caps.provides
3442                    .iter()
3443                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3444            })
3445        {
3446            if let Some(stamp) = description.stamp.as_mut() {
3447                stamp.attributes.on_behalf_of_agent = None;
3448            }
3449        }
3450        if !registered
3451            .manifest
3452            .capabilities
3453            .as_ref()
3454            .is_some_and(|caps| {
3455                caps.provides
3456                    .iter()
3457                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3458            })
3459        {
3460            if let Some(stamp) = description.stamp.as_mut() {
3461                stamp.attributes.flow_version = None;
3462                stamp.attributes.flow_code_hash = None;
3463                stamp.attributes.flow_install_id = None;
3464            }
3465        }
3466        let owner_configured = match &owner {
3467            // Ask whether the owner is configured (`is_configured`), not
3468            // whether it is on the roster (`get(..).is_some()`): a supervised
3469            // module's process can register and describe a scope before the
3470            // supervisor has put it on the roster.
3471            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3472            _ => false,
3473        };
3474        let response = ModuleControlResponseToModule::ScopeDescribe {
3475            status: description.status,
3476            scope_epoch: description.scope_epoch,
3477            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3478            owner_synced: description.owner_synced,
3479            owner_configured,
3480            scope: description.stamp,
3481        };
3482        Ok(vec![control_response_body_frame(
3483            &frame,
3484            &response,
3485            "ModuleControlResponseToModule::ScopeDescribe",
3486        )?])
3487    }
3488
3489    fn handle_catalog_update(
3490        &self,
3491        connection_id: ConnectionId,
3492        frame: Frame,
3493        provides: Vec<ProviderRole>,
3494        capabilities: Option<CapabilityDeclarations>,
3495        ready: Option<bool>,
3496    ) -> Result<Vec<Frame>, RouterError> {
3497        self.refresh_capability_requirements();
3498        let Some(registration) = self
3499            .registry
3500            .get_module_by_connection(connection_id)
3501            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3502        else {
3503            return Ok(vec![control_error_frame(
3504                &frame,
3505                "not_registered",
3506                "catalog.update requires an active module registration owned by this connection",
3507            )?]);
3508        };
3509
3510        if let Some(message) =
3511            catalog_update_frozen_field_message(&registration.manifest, &provides)
3512        {
3513            return Ok(vec![control_error_frame(
3514                &frame,
3515                "catalog_update_frozen_field",
3516                message,
3517            )?]);
3518        }
3519
3520        let mut candidate = registration.manifest.clone();
3521        candidate.provides = provides.clone();
3522        candidate.capabilities = capabilities
3523            .clone()
3524            .or_else(|| registration.manifest.capabilities.clone());
3525        if let Some(rich_ui) = &candidate.rich_ui {
3526            if let Err(err) = rich_ui.validate() {
3527                return Ok(vec![control_error_frame(
3528                    &frame,
3529                    "invalid_manifest",
3530                    err.to_string(),
3531                )?]);
3532            }
3533        }
3534        if let Err(err) = candidate.validate_capability_grammar() {
3535            return Ok(vec![control_error_frame(
3536                &frame,
3537                "invalid_capability_grammar",
3538                err.to_string(),
3539            )?]);
3540        }
3541
3542        // Updates must honor the same reserved owner as initial registration;
3543        // otherwise an empty HELLO could acquire the claim after admission.
3544        let mut conflicts = self
3545            .capability_evaluator
3546            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3547        if let Some(conflict) = conflicts.first() {
3548            let message = format!(
3549                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3550                conflict.capability, conflict.claimants[0], candidate.module_id
3551            );
3552            for conflict in &mut conflicts {
3553                conflict.source = DuplicateClaimSource::CatalogUpdate;
3554            }
3555            log_duplicate_claim_events(conflicts);
3556            return Ok(vec![control_error_frame(
3557                &frame,
3558                "reserved_capability",
3559                message,
3560            )?]);
3561        }
3562
3563        let updated = self
3564            .registry
3565            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3566            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3567        if updated.is_none() {
3568            return Ok(vec![control_error_frame(
3569                &frame,
3570                "not_registered",
3571                "catalog.update requires an active module registration owned by this connection",
3572            )?]);
3573        }
3574        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3575            log_duplicate_claim_events(
3576                self.capability_evaluator
3577                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3578            );
3579        }
3580        if capability_census_trigger(
3581            registration.manifest.capabilities.as_ref(),
3582            updated
3583                .as_ref()
3584                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3585        ) {
3586            self.enforce_capability_denies();
3587        }
3588        self.refresh_capability_requirements();
3589
3590        let response = ModuleControlResponseToModule::CatalogUpdate {};
3591        control_response_body_frame(
3592            &frame,
3593            &response,
3594            "ModuleControlResponseToModule::CatalogUpdate",
3595        )
3596        .map(|frame| vec![frame])
3597    }
3598
3599    pub(crate) fn with_activity_ledger_config(
3600        mut self,
3601        run_dir: &Path,
3602        incarnation: String,
3603        config: crate::activity::ActivityConfig,
3604    ) -> Self {
3605        self.activity = Some(crate::activity::ActivityLedger::open_config(
3606            run_dir.join("activity.jsonl"),
3607            incarnation,
3608            config,
3609        ));
3610        self.supervisor.set_activity(self.activity.clone());
3611        self
3612    }
3613
3614    pub(crate) async fn configure_activity(
3615        &self,
3616        config: crate::activity::ActivityConfig,
3617    ) -> Result<(), &'static str> {
3618        match &self.activity {
3619            Some(ledger) => ledger.configure(config).await,
3620            None => Ok(()),
3621        }
3622    }
3623
3624    fn cancel_activity_subscription(
3625        &self,
3626        ctx: &RouteCtx,
3627        frame: &Frame,
3628    ) -> Result<bool, RouterError> {
3629        let task = self
3630            .activity_streams
3631            .lock()
3632            .unwrap_or_else(|p| p.into_inner())
3633            .remove(&(ctx.connection_id, frame.header.corr));
3634        let Some(task) = task else {
3635            return Ok(false);
3636        };
3637        task.abort();
3638        let end = Frame::build_with_version(
3639            response_version(frame),
3640            FrameType::StreamEnd,
3641            control_flags(),
3642            0,
3643            0,
3644            frame.header.corr,
3645            Vec::new(),
3646        )
3647        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3648        let sink = ctx.egress.clone();
3649        tokio::spawn(async move {
3650            let _ = sink.send(end).await;
3651        });
3652        Ok(true)
3653    }
3654
3655    async fn handle_activity_read(
3656        &self,
3657        ctx: &RouteCtx,
3658        frame: Frame,
3659    ) -> Result<Vec<Frame>, RouterError> {
3660        use subc_control::{
3661            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3662            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3663        };
3664        #[derive(Deserialize)]
3665        struct Identity {
3666            #[serde(default)]
3667            consumer_identity: Option<ConsumerIdentity>,
3668        }
3669        // A registered module connection already identifies its caller. Ignore
3670        // consumer_identity there so a module cannot borrow a mirror's identity.
3671        // Only separate consumer connections use the Direct/nonce trust boundary.
3672        let principal = match self
3673            .forwarding
3674            .module_id_for_connection(ctx.connection_id)
3675            .map_err(RouterError::Forwarding)?
3676        {
3677            Some(module_id) => Ok(Principal::Reserved { module_id }),
3678            None => {
3679                let identity: Identity = match serde_json::from_slice(&frame.body) {
3680                    Ok(identity) => identity,
3681                    Err(_) => {
3682                        return Ok(vec![control_error_frame(
3683                            &frame,
3684                            ACTIVITY_MALFORMED,
3685                            "malformed activity read",
3686                        )?])
3687                    }
3688                };
3689                self.route_open_principal(&frame, identity.consumer_identity)?
3690            }
3691        };
3692        let admitted = matches!(principal, Ok(Principal::Direct))
3693            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3694        if !admitted {
3695            return Ok(vec![control_error_frame(
3696                &frame,
3697                ACTIVITY_READ_FORBIDDEN,
3698                "activity reads require an operator or trusted mirror principal",
3699            )?]);
3700        }
3701        let Some(ledger) = &self.activity else {
3702            return Ok(vec![control_error_frame(
3703                &frame,
3704                ACTIVITY_UNAVAILABLE,
3705                "activity ledger unavailable",
3706            )?]);
3707        };
3708        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3709            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3710        {
3711            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3712                Ok(request) => ledger.query(request).await,
3713                Err(_) => Err(ACTIVITY_MALFORMED),
3714            };
3715            return match result {
3716                Ok(reply) => Ok(vec![control_response_body_frame(
3717                    &frame,
3718                    &reply,
3719                    ops::ACTIVITY_QUERY,
3720                )?]),
3721                Err(code) => Ok(vec![control_error_frame(
3722                    &frame,
3723                    code,
3724                    "activity query refused",
3725                )?]),
3726            };
3727        }
3728        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3729            Ok(request) => request,
3730            Err(_) => {
3731                return Ok(vec![control_error_frame(
3732                    &frame,
3733                    ACTIVITY_MALFORMED,
3734                    "malformed activity subscription",
3735                )?])
3736            }
3737        };
3738        let mut subscription = match ledger.subscribe(request.since).await {
3739            Ok(subscription) => subscription,
3740            Err(code) => {
3741                return Ok(vec![control_error_frame(
3742                    &frame,
3743                    code,
3744                    "activity subscription refused",
3745                )?])
3746            }
3747        };
3748        let key = (ctx.connection_id, frame.header.corr);
3749        let sink = ctx.egress.clone();
3750        let mut streams = self
3751            .activity_streams
3752            .lock()
3753            .unwrap_or_else(|p| p.into_inner());
3754        streams.retain(|_, task| !task.is_finished());
3755        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3756            return Ok(vec![control_error_frame(
3757                &frame,
3758                "duplicate_subscription",
3759                "activity subscription correlation id already in use",
3760            )?]);
3761        }
3762        let task = tokio::spawn(async move {
3763            loop {
3764                let entry = if let Some(entry) = subscription.replay.pop_front() {
3765                    Some(entry)
3766                } else {
3767                    subscription.receiver.recv().await
3768                };
3769                let Some(entry) = entry else {
3770                    break;
3771                };
3772                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3773                    return;
3774                };
3775                let Ok(next) = Frame::build_with_version(
3776                    response_version(&frame),
3777                    FrameType::StreamData,
3778                    control_flags(),
3779                    0,
3780                    0,
3781                    frame.header.corr,
3782                    body,
3783                ) else {
3784                    return;
3785                };
3786                if sink.send(next).await.is_err() {
3787                    return;
3788                }
3789            }
3790            // Dropping a full subscriber's sender drains its accepted entries
3791            // before the terminal error, preserving their order on the wire.
3792            if let Some(code) = subscription.terminal_code() {
3793                if let Ok(end) =
3794                    control_error_frame(&frame, code, "activity subscriber fell behind")
3795                {
3796                    let _ = sink.send(end).await;
3797                }
3798            }
3799        });
3800        streams.insert(key, task.abort_handle());
3801        Ok(Vec::new())
3802    }
3803
3804    async fn handle_activity_record(
3805        &self,
3806        ctx: &RouteCtx,
3807        frame: Frame,
3808    ) -> Result<Vec<Frame>, RouterError> {
3809        use subc_control::{
3810            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3811            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3812        };
3813        let result = async {
3814            let module = self
3815                .forwarding
3816                .module_id_for_connection(ctx.connection_id)
3817                .ok()
3818                .flatten()
3819                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3820            let request: ActivityRecordRequest =
3821                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3822            crate::activity::validate(&module, &request)?;
3823            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3824            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3825            let at_ms = std::time::SystemTime::now()
3826                .duration_since(std::time::UNIX_EPOCH)
3827                .unwrap_or_default()
3828                .as_millis() as u64;
3829            ledger.record(module, request, actor, at_ms).await
3830        }
3831        .await;
3832        match result {
3833            Ok(entry) => Ok(vec![control_response_body_frame(
3834                &frame,
3835                &ActivityRecordResponse { seq: entry.seq },
3836                "activity.record",
3837            )?]),
3838            Err(code) => Ok(vec![control_error_frame(
3839                &frame,
3840                code,
3841                "activity recording refused",
3842            )?]),
3843        }
3844    }
3845
3846    fn activity_actor(
3847        &self,
3848        connection: ConnectionId,
3849        module: &str,
3850        on: Option<&subc_control::ActivityOn>,
3851    ) -> Result<subc_control::ActivityActor, &'static str> {
3852        use crate::forwarding::{DataRoute, DataRouteState};
3853        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3854        let principal = |id: &str| Principal::Reserved {
3855            module_id: id.to_owned(),
3856        };
3857        let mut actor = ActivityActor {
3858            principal: principal(module),
3859            scope_owner: None,
3860            agent_id: None,
3861            run_id: None,
3862            flow_id: None,
3863            project_root: None,
3864            workdir: None,
3865            project_id: None,
3866        };
3867        let scope = match on {
3868            None => return Ok(actor),
3869            Some(ActivityOn::Scope(scope)) => {
3870                let selector = subc_protocol::scope::ScopeSelector::new(
3871                    principal(module),
3872                    scope.scope_ref.clone(),
3873                    Some(scope.epoch),
3874                );
3875                self.scopes
3876                    .read()
3877                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3878                    .attribute(&principal(module), module, &selector, true)
3879                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3880            }
3881            Some(ActivityOn::Route(route)) => {
3882                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3883                    .forwarding
3884                    .lookup_data_route(connection, route.channel, route.epoch)
3885                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3886                else {
3887                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3888                };
3889                if binding.module_id != module {
3890                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3891                }
3892                actor.principal = binding.principal.clone();
3893                actor.project_root = binding
3894                    .project_root
3895                    .as_ref()
3896                    .map(|root| root.as_path().to_path_buf());
3897                let Some(bound) = &binding.scope else {
3898                    return Ok(actor);
3899                };
3900                let owner = principal(&bound.owner);
3901                let selector = subc_protocol::scope::ScopeSelector::new(
3902                    owner.clone(),
3903                    bound.scope_ref.clone(),
3904                    Some(bound.tag.scope_epoch),
3905                );
3906                let admitted = self
3907                    .scopes
3908                    .read()
3909                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3910                    .attribute(&owner, module, &selector, true)
3911                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3912                if admitted.tag != bound.tag {
3913                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3914                }
3915                admitted
3916            }
3917        };
3918        actor.scope_owner = Some(scope.stamp.owner);
3919        actor.agent_id = scope.stamp.attributes.agent_id;
3920        actor.run_id = scope.stamp.attributes.run_id;
3921        actor.flow_id = scope.stamp.attributes.flow_id;
3922        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3923        actor.project_id = scope.stamp.project_id;
3924        Ok(actor)
3925    }
3926
3927    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3928        self.refresh_capability_requirements();
3929        // A bare connection count is ambiguous between many clients holding a
3930        // route each and one client accumulating hundreds, so publish the
3931        // concentration alongside it. Route state is best-effort here: a
3932        // diagnostic endpoint must still answer if the forwarding lock is
3933        // contended.
3934        let mut counters = self.counters.snapshot();
3935        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3936            obj.insert(
3937                "activity_unparseable_lines".into(),
3938                ledger.read_errors().into(),
3939            );
3940            obj.insert(
3941                "activity_lifecycle_append_failures".into(),
3942                ledger.lifecycle_failures().into(),
3943            );
3944            obj.insert(
3945                "activity_started_digest_skips".into(),
3946                ledger.lifecycle_digest_skips().into(),
3947            );
3948        }
3949        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3950            self.forwarding.client_route_concentration(),
3951            counters.as_object_mut(),
3952        ) {
3953            obj.insert(
3954                "client_connections_with_routes".into(),
3955                connections_with_routes.into(),
3956            );
3957            obj.insert("max_routes_on_one_connection".into(), max.into());
3958        }
3959        // A module that is being fast-refused and a module that is fine look
3960        // identical from a client that retries and succeeds, so name the open
3961        // breakers here. This rides the existing free-form counters object
3962        // rather than a new wire field, so no sibling that deserializes
3963        // `ServerDescribe` has to be rebuilt to keep reading it.
3964        if let (Some(open_breakers), Some(obj)) = (
3965            self.route_bind_breakers.open_snapshot(),
3966            counters.as_object_mut(),
3967        ) {
3968            obj.insert("route_bind_breakers_open".into(), open_breakers);
3969        }
3970        let response = ClientControlResponse::ServerDescribe {
3971            protocol_ver: PROTOCOL_VERSION,
3972            subc_ops: subc_ops(),
3973            capabilities: self.describe_capabilities(),
3974            connected_clients: self.connected_clients.count(),
3975            counters: Some(counters),
3976            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3977            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3978            capability_requirements: self.capability_requirement_statuses(),
3979            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3980            config_path: self
3981                .rescan
3982                .as_ref()
3983                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3984        };
3985        Ok(vec![control_response_body_frame(
3986            &frame,
3987            &response,
3988            "ClientControlResponse::ServerDescribe",
3989        )?])
3990    }
3991
3992    fn handle_catalog_list(
3993        &self,
3994        frame: Frame,
3995        module_id: Option<String>,
3996    ) -> Result<Vec<Frame>, RouterError> {
3997        let (generation, modules) = self
3998            .registry
3999            .list_modules_with_nonce_verification()
4000            .map_err(|err| {
4001                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4002            })?;
4003        let entries = modules
4004            .into_iter()
4005            .filter(|(registration, _)| {
4006                module_id
4007                    .as_deref()
4008                    .map(|wanted| registration.manifest.module_id == wanted)
4009                    .unwrap_or(true)
4010            })
4011            .map(|(registration, nonce_verified)| {
4012                let not_ready = self.not_ready_reason(&registration);
4013                let roles = registration.manifest.provides;
4014                CatalogEntry::new(
4015                    registration.manifest.module_id,
4016                    roles,
4017                    registration.control_ops,
4018                )
4019                .with_ready(not_ready.is_none())
4020                .with_not_ready(not_ready)
4021                .with_module_version(Some(registration.manifest.module_version))
4022                .with_capabilities(registration.manifest.capabilities)
4023                .with_self_signals(registration.manifest.self_signals)
4024                .with_events(registration.manifest.events)
4025                .with_operation_delivery(registration.manifest.operation_delivery)
4026                .with_operation_payload(registration.manifest.operation_payload)
4027                .with_rich_ui(registration.manifest.rich_ui)
4028                .with_role_declarations(registration.manifest.role_declarations)
4029                .with_nonce_verified(nonce_verified)
4030            })
4031            .collect();
4032        let response = ClientControlResponse::CatalogList {
4033            generation,
4034            modules: entries,
4035            subc_ops: subc_ops(),
4036        };
4037        Ok(vec![control_response_body_frame(
4038            &frame,
4039            &response,
4040            "ClientControlResponse::CatalogList",
4041        )?])
4042    }
4043
4044    /// The principal a route.open's `consumer_identity` attests: `Direct`
4045    /// when it presents none, `Reserved` when its launch nonce verifies, and
4046    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4047    /// can run before any refusal check without changing which check refuses.
4048    fn attest_route_open_consumer(
4049        &self,
4050        consumer_identity: Option<&ConsumerIdentity>,
4051    ) -> Option<Principal> {
4052        let Some(consumer_identity) = consumer_identity else {
4053            return Some(Principal::Direct);
4054        };
4055        self.supervisor
4056            .spawned_consumer_authorized(
4057                &consumer_identity.module_id,
4058                &consumer_identity.launch_nonce,
4059            )
4060            .then(|| Principal::Reserved {
4061                module_id: consumer_identity.module_id.clone(),
4062            })
4063    }
4064
4065    /// The caller fields for a refusal that only has the raw request frame,
4066    /// such as the connection-level admission refusal sent before the request
4067    /// reaches [`Self::handle_route_open`].
4068    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4069        let Ok(ClientControlRequest::RouteOpen {
4070            identity,
4071            consumer_identity,
4072            scope,
4073            ..
4074        }) = parse_client_control_request(&frame.body)
4075        else {
4076            return RouteOpenCaller::unknown();
4077        };
4078        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4079        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4080    }
4081
4082    fn route_open_principal(
4083        &self,
4084        frame: &Frame,
4085        consumer_identity: Option<ConsumerIdentity>,
4086    ) -> Result<Result<Principal, Frame>, RouterError> {
4087        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4088        self.route_open_principal_attested(frame, consumer_identity, attested)
4089    }
4090
4091    /// [`Self::route_open_principal`] with `attested` already taken by
4092    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4093    /// open checks the launch nonce once.
4094    fn route_open_principal_attested(
4095        &self,
4096        frame: &Frame,
4097        consumer_identity: Option<ConsumerIdentity>,
4098        attested: Option<Principal>,
4099    ) -> Result<Result<Principal, Frame>, RouterError> {
4100        if let Some(principal) = attested {
4101            return Ok(Ok(principal));
4102        }
4103        let Some(consumer_identity) = consumer_identity else {
4104            return Ok(Ok(Principal::Direct));
4105        };
4106
4107        Ok(Err(control_error_frame(
4108            frame,
4109            "bad_consumer_identity",
4110            format!(
4111                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4112                consumer_identity.module_id
4113            ),
4114        )?))
4115    }
4116
4117    /// Ordinary `route.open` refusals go through here; admission and breaker
4118    /// refusals log separately with their capacity or breaker state. The daemon can
4119    /// attest which code it sent: without the event, a client's "the daemon
4120    /// refused me" and the daemon's own view could only be reconciled by
4121    /// argument. Malformed input (`invalid_project_root`) does not come here;
4122    /// rejecting a request that was never a valid open is not a refusal of one.
4123    #[allow(clippy::too_many_arguments)]
4124    fn route_open_refusal_frame(
4125        &self,
4126        ctx: &RouteCtx,
4127        caller: &RouteOpenCaller,
4128        frame: &Frame,
4129        module_id: &str,
4130        reason: &'static str,
4131        code: &'static str,
4132        message: impl Into<String>,
4133    ) -> Result<Frame, RouterError> {
4134        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4135        control_error_frame(frame, code, message.into())
4136    }
4137
4138    /// Refuse a `route.open` because the target module's bind-relay breaker is
4139    /// open, without attempting the relay.
4140    ///
4141    /// The wire code is `module_timeout`, which is the truth (the module has
4142    /// not been answering binds) and which both SDKs already classify as
4143    /// retryable with capped backoff. Reusing it is what keeps this change out
4144    /// of both SDKs; the daemon-side distinction lives in the counter key
4145    /// instead.
4146    ///
4147    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4148    /// While a breaker is open this fires on every open to that module, and the
4149    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4150    /// produced 261 lines about a single module inside 3000 lines of daemon
4151    /// log. The rare transitions are logged at warn/info instead and the volume
4152    /// is carried by the counter, so the evidence survives without the flood.
4153    /// The debug line keeps a per-refusal record reachable for whoever turns
4154    /// the level up.
4155    #[allow(clippy::too_many_arguments)]
4156    fn route_open_breaker_refusal_frame(
4157        &self,
4158        ctx: &RouteCtx,
4159        caller: &RouteOpenCaller,
4160        frame: &Frame,
4161        module_id: &str,
4162        consecutive_timeouts: u32,
4163        retry_in: Duration,
4164        probe_in_flight: bool,
4165    ) -> Result<Frame, RouterError> {
4166        self.counters
4167            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4168        debug!(
4169            target: "control",
4170            code = "module_timeout",
4171            module_id = ?module_id,
4172            connection_id = ctx.connection_id.get(),
4173            principal = %caller.principal,
4174            scope = %caller.scope,
4175            harness = %caller.harness,
4176            consecutive_timeouts,
4177            retry_in_ms = retry_in.as_millis() as u64,
4178            probe_in_flight,
4179            "route.open refused by open bind-relay breaker"
4180        );
4181        // Say what a caller can act on. An open bind-relay breaker means the
4182        // module timed out accepting several new routes in a row. The module
4183        // is still running and its established routes keep working; only new
4184        // route.open requests are refused until the cooldown ends and one
4185        // test route (the probe) gets through. A message that only counts
4186        // failed relays reads as "the module is down" to a worker that sees it.
4187        let detail = if probe_in_flight {
4188            "one test route is already being tried; retry once it settles".to_string()
4189        } else {
4190            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4191        };
4192        control_error_frame(
4193            frame,
4194            "module_timeout",
4195            format!(
4196                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4197                 timed out in a row); {detail}; its established routes are unaffected"
4198            ),
4199        )
4200    }
4201
4202    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4203    /// requester's bytes (an unknown target is whatever the client sent) and
4204    /// is Debug-formatted so control characters land in the log escaped
4205    /// rather than as terminal sequences for whoever tails it.
4206    ///
4207    /// `reason` names the check that refused, because one wire code has
4208    /// several senders: after a module registers, `target_unavailable` can
4209    /// come from a missing role, an inactive registration, a supervisor that
4210    /// has not marked the process live, a missing forwarding connection, or a
4211    /// failed relay, and a log that records only the code cannot say which of
4212    /// them fired. It is a static, daemon-chosen label per branch, so it is
4213    /// safe to print plainly and stays a closed set.
4214    fn observe_route_open_refusal(
4215        &self,
4216        ctx: &RouteCtx,
4217        caller: &RouteOpenCaller,
4218        module_id: &str,
4219        reason: &'static str,
4220        code: &'static str,
4221    ) {
4222        self.counters.increment_route_open_refused(code);
4223        info!(
4224            target: "control",
4225            code,
4226            reason,
4227            module_id = ?module_id,
4228            connection_id = ctx.connection_id.get(),
4229            principal = %caller.principal,
4230            scope = %caller.scope,
4231            harness = %caller.harness,
4232            "route.open refused"
4233        );
4234        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4235            self.route_outages.record_not_serving(module_id, reason);
4236        }
4237    }
4238
4239    /// Record an ACCEPTED route.open.
4240    ///
4241    /// Refusals have been logged and counted since the attestation work; accepts
4242    /// were invisible, so the daemon knew every principal it stamped and wrote
4243    /// none of them down. The party that attests the identity was the only party
4244    /// not recording it, which left a credential vault unable to name the sender
4245    /// of a call that reached it (claustrum #43) and left the launch-nonce
4246    /// concurrency question unanswerable from the outside.
4247    ///
4248    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4249    /// `code`/`module_id`/`connection_id` returns both directions of the same
4250    /// decision rather than two shapes a reader has to join by hand.
4251    ///
4252    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4253    /// and the difference carries information rather than being an
4254    /// inconsistency. This line is only reachable after a successful bind to a
4255    /// REGISTERED module, so the value has already passed HELLO validation
4256    /// including the path-hazard refusal and cannot contain control bytes. A
4257    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4258    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4259    ///
4260    /// Bare is also what every other daemon line already emits (`module
4261    /// registered`, `configured module supervised`). Shipping `?module_id` here
4262    /// made this instrument the only one in the file whose ids did not answer
4263    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4264    /// line whose whole purpose is being grepped beside its sibling.
4265    ///
4266    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4267    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4268    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4269    /// one are recorded identically. A test written against that harness passes
4270    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4271    /// green assertion that cannot fail. The same limit applies to the escaping
4272    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4273    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4274    /// Fencing either needs the real formatter, not the capture layer.
4275    ///
4276    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4277    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4278    /// to record. The ephemeral port would decay within minutes and answer only a
4279    /// live question. The identity question is instead answered by counting
4280    /// distinct live connections presenting one module's `consumer_identity` --
4281    /// "is anyone else holding this secret" rather than "is this the right
4282    /// process".
4283    fn observe_route_open_accept(
4284        &self,
4285        ctx: &RouteCtx,
4286        module_id: &str,
4287        principal: &str,
4288        scope: &str,
4289    ) {
4290        self.route_outages.record_accepted(module_id);
4291        self.counters.increment_route_open_accepted(principal);
4292        info!(
4293            target: "control",
4294            principal,
4295            module_id,
4296            connection_id = ctx.connection_id.get(),
4297            scope = %scope,
4298            "route.open accepted"
4299        );
4300    }
4301
4302    fn supervised_absent_route_open_refusal_frame(
4303        &self,
4304        ctx: &RouteCtx,
4305        caller: &RouteOpenCaller,
4306        frame: &Frame,
4307        module_id: &str,
4308        code: &'static str,
4309        status: &crate::supervise::ModuleStatus,
4310    ) -> Result<Frame, RouterError> {
4311        self.counters.increment_route_open_refused(code);
4312        info!(
4313            target: "control",
4314            code,
4315            reason = "supervised_not_registered",
4316            module_id = ?module_id,
4317            connection_id = ctx.connection_id.get(),
4318            principal = %caller.principal,
4319            scope = %caller.scope,
4320            harness = %caller.harness,
4321            state = %status.state,
4322            enabled = status.enabled,
4323            live = status.live,
4324            "route.open refused"
4325        );
4326        // A supervised module whose process has not registered is not
4327        // serving, whatever the reason; the supervisor knows this id, so it is
4328        // safe to track.
4329        self.route_outages
4330            .record_not_serving(module_id, "supervised_not_registered");
4331        let message = if let Some(pid) = status.waiting_for_previous_pid {
4332            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4333                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4334            })
4335        } else {
4336            format!(
4337                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4338                status.state, status.enabled, status.live
4339            )
4340        };
4341        control_error_frame(frame, code, message)
4342    }
4343
4344    async fn handle_route_open(
4345        &self,
4346        ctx: &RouteCtx,
4347        frame: Frame,
4348        request: RouteOpenRequest,
4349        timings: &mut RouteOpenTimings,
4350    ) -> Result<Vec<Frame>, RouterError> {
4351        let RouteOpenRequest {
4352            target,
4353            mut identity,
4354            consumer_identity,
4355            consumer_capabilities,
4356            role_versions,
4357            admission_facts,
4358            scope,
4359        } = request;
4360        let target_module_id = target_module_id(&target).to_string();
4361        timings.module_id.clone_from(&target_module_id);
4362        if self
4363            .registry
4364            .get_module_by_connection(ctx.connection_id)
4365            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4366            .is_some()
4367        {
4368            return Ok(vec![control_error_frame(
4369                &frame,
4370                "invalid_request",
4371                "module connections cannot open client routes",
4372            )?]);
4373        }
4374        debug!(
4375            connection_id = ctx.connection_id.get(),
4376            corr = frame.header.corr,
4377            module_id = %target_module_id,
4378            "handling route.open"
4379        );
4380
4381        // The caller's attested principal, requested scope and harness, for
4382        // every `route.open refused` line below. The launch nonce is checked
4383        // here, once, and the answer is reused where the principal is
4384        // enforced further down, so the order of refusals is unchanged.
4385        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4386        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4387
4388        // A malformed declaration is refused first, before anything about the
4389        // target is looked up: the same body would be refused against any
4390        // module, so the caller learns nothing by retrying or waiting. An empty
4391        // map declares nothing and travels as no field at all, so a provider
4392        // only ever sees a missing field or a non-empty one.
4393        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4394        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4395            self.observe_route_open_refusal(
4396                ctx,
4397                &caller,
4398                &target_module_id,
4399                "invalid_role_versions",
4400                error_codes::INVALID_REQUEST,
4401            );
4402            return Ok(vec![control_error_body_frame(
4403                &frame,
4404                ErrorBody {
4405                    code: error_codes::INVALID_REQUEST.to_string(),
4406                    message: error.to_string(),
4407                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4408                },
4409            )?]);
4410        }
4411
4412        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4413            if status.waiting_for_previous_pid.is_some() {
4414                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4415                    ctx,
4416                    &caller,
4417                    &frame,
4418                    &target_module_id,
4419                    "module_warming",
4420                    &status,
4421                )?]);
4422            }
4423        }
4424
4425        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4426        // opposite. Below, a caller learns whether a module is unregistered,
4427        // supervised-but-down (with state/enabled/live), or registered without the
4428        // requested role. Elsewhere that is an enumeration leak: a probe learning
4429        // the shape of a fleet it cannot otherwise see.
4430        //
4431        // It is not one here, and the reason is the ACCESS MODEL rather than
4432        // anything about these errors. Reaching route.open requires the
4433        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4434        // connection file, so any caller who completes it already runs as this
4435        // user -- and can read subc.jsonc for the module list and `ck module
4436        // status` for live state. The reply discloses nothing the caller cannot
4437        // read more easily from disk, while the precision is load-bearing:
4438        // `unknown_module` is retryable and a missing role is not.
4439        //
4440        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4441        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4442        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4443        let Some(registration) = self
4444            .registry
4445            .get_module(&target_module_id)
4446            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4447        else {
4448            if let Some((status, warming)) =
4449                self.supervisor_status(&target_module_id, frame.header.corr)?
4450            {
4451                // BEFORE the two availability codes below, because for a module
4452                // that speaks no subc wire both of them are false comfort: they
4453                // say "not right now" and are retried, and this module will
4454                // never register no matter how long the caller waits. The
4455                // absence here is the declaration being honoured, not a module
4456                // that is late.
4457                if status.protocol == ModuleProtocol::None {
4458                    return Ok(vec![self.route_open_refusal_frame(
4459                        ctx,
4460                        &caller,
4461                        &frame,
4462                        &target_module_id,
4463                        "protocol_none",
4464                        error_codes::MODULE_NO_PROTOCOL,
4465                        format!(
4466                            "module_id '{target_module_id}' is declared protocol: none; \
4467                             it speaks no subc wire and serves no routes"
4468                        ),
4469                    )?]);
4470                }
4471                let code = if warming {
4472                    "module_warming"
4473                } else {
4474                    "target_unavailable"
4475                };
4476                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4477                    ctx,
4478                    &caller,
4479                    &frame,
4480                    &target_module_id,
4481                    code,
4482                    &status,
4483                )?]);
4484            }
4485            if let Some(removed_ago_ms) =
4486                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4487            {
4488                return Ok(vec![self.route_open_refusal_frame(
4489                    ctx,
4490                    &caller,
4491                    &frame,
4492                    &target_module_id,
4493                    "removed",
4494                    error_codes::MODULE_REMOVED,
4495                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4496                )?]);
4497            }
4498            return Ok(vec![self.route_open_refusal_frame(
4499                ctx,
4500                &caller,
4501                &frame,
4502                &target_module_id,
4503                "not_registered",
4504                error_codes::UNKNOWN_MODULE,
4505                format!("module_id '{target_module_id}' is not registered"),
4506            )?]);
4507        };
4508
4509        // Best-effort only: registry readiness and forwarding reservation use
4510        // different locks, so a module can flip readiness between this read and
4511        // the relay. Modules must still tolerate an `on_bind` while not ready.
4512        if !registration.ready {
4513            self.counters
4514                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4515            info!(
4516                target: "control",
4517                code = error_codes::MODULE_WARMING,
4518                module_id = ?target_module_id,
4519                connection_id = ctx.connection_id.get(),
4520                principal = %caller.principal,
4521                scope = %caller.scope,
4522                harness = %caller.harness,
4523                reason = "declared_not_ready",
4524                "route.open refused"
4525            );
4526            // The module is registered but says it cannot take work, which is
4527            // an outage from the caller's side even though its process is up.
4528            self.route_outages
4529                .record_not_serving(&target_module_id, "declared_not_ready");
4530            return Ok(vec![control_error_body_frame(
4531                &frame,
4532                ErrorBody {
4533                    code: error_codes::MODULE_WARMING.to_string(),
4534                    message: format!(
4535                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4536                    ),
4537                    detail: Some(serde_json::json!({
4538                        "reason": "declared_not_ready"
4539                    })),
4540                },
4541            )?]);
4542        }
4543
4544        // Effective readiness, second half: a module that declares a capability
4545        // `need: required` is not routable while that capability has no
4546        // registered provider. It is enforced HERE, as a retryable routing
4547        // refusal, and deliberately not as spawn ordering or a boot block. The
4548        // module is still started and registered and can make its own calls;
4549        // spawn ordering is a promise that cannot be kept once a provider
4550        // crashes at runtime, and refusing to boot would stop the whole
4551        // machine, including the tools needed to fix its configuration.
4552        //
4553        // "Provided" is the evaluator's verdict, which counts a provider as
4554        // soon as it has REGISTERED, not once it is ready. Two modules that
4555        // require each other's capabilities are therefore both routable once
4556        // both register; counting readiness instead would deadlock them.
4557        //
4558        // Only new opens are refused. Routes already bound when a provider
4559        // goes away stay bound: nothing here tears them down, and the module
4560        // answers them as it can. Like the readiness read above this is
4561        // best-effort against a provider registering or leaving concurrently.
4562        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4563            self.counters
4564                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4565            info!(
4566                target: "control",
4567                code = error_codes::MODULE_WARMING,
4568                module_id = ?target_module_id,
4569                connection_id = ctx.connection_id.get(),
4570                principal = %caller.principal,
4571                scope = %caller.scope,
4572                harness = %caller.harness,
4573                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4574                capability = %capability,
4575                "route.open refused"
4576            );
4577            return Ok(vec![control_error_body_frame(
4578                &frame,
4579                ErrorBody {
4580                    code: error_codes::MODULE_WARMING.to_string(),
4581                    message: format!(
4582                        "module_id '{target_module_id}' requires capability '{capability}', \
4583                         which no registered module provides; retry"
4584                    ),
4585                    detail: Some(serde_json::json!({
4586                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4587                        "capability": capability,
4588                    })),
4589                },
4590            )?]);
4591        }
4592
4593        if !target_has_required_role(&target, &registration.manifest.provides) {
4594            return Ok(vec![self.route_open_refusal_frame(
4595                ctx,
4596                &caller,
4597                &frame,
4598                &target_module_id,
4599                "role_not_provided",
4600                "target_unavailable",
4601                format!("module_id '{target_module_id}' does not provide the requested target"),
4602            )?]);
4603        }
4604
4605        if registration.state != ChannelState::Active {
4606            return Ok(vec![self.route_open_refusal_frame(
4607                ctx,
4608                &caller,
4609                &frame,
4610                &target_module_id,
4611                "registration_not_active",
4612                "target_unavailable",
4613                format!("module_id '{target_module_id}' is not active"),
4614            )?]);
4615        }
4616
4617        if self
4618            .forwarding
4619            .module_is_draining(&target_module_id)
4620            .map_err(RouterError::Forwarding)?
4621        {
4622            return Ok(vec![self.route_open_refusal_frame(
4623                ctx,
4624                &caller,
4625                &frame,
4626                &target_module_id,
4627                "reloading",
4628                "module_reloading",
4629                format!("module_id '{target_module_id}' is reloading"),
4630            )?]);
4631        }
4632
4633        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4634            process_liveness.process_live(&target_module_id) == Some(false)
4635        }) {
4636            // A module the supervisor is restarting or reloading can still hold
4637            // a registration: the old process before its connection closes, or
4638            // a new one that registered while the supervisor was draining. The
4639            // forwarding table does not see that as draining, but the consumer
4640            // should still be told to retry soon, exactly as for the drain
4641            // above, rather than that the target is unavailable.
4642            if process_liveness.process_replacing(&target_module_id) {
4643                return Ok(vec![self.route_open_refusal_frame(
4644                    ctx,
4645                    &caller,
4646                    &frame,
4647                    &target_module_id,
4648                    "reloading",
4649                    "module_reloading",
4650                    format!("module_id '{target_module_id}' is reloading"),
4651                )?]);
4652            }
4653            return Ok(vec![self.route_open_refusal_frame(
4654                ctx,
4655                &caller,
4656                &frame,
4657                &target_module_id,
4658                "supervisor_not_live",
4659                "target_unavailable",
4660                format!("module_id '{target_module_id}' is not live"),
4661            )?]);
4662        }
4663
4664        if !self
4665            .forwarding
4666            .has_live_module_connection(&target_module_id)
4667            .map_err(RouterError::Forwarding)?
4668        {
4669            return Ok(vec![self.route_open_refusal_frame(
4670                ctx,
4671                &caller,
4672                &frame,
4673                &target_module_id,
4674                "no_forwarding_connection",
4675                "target_unavailable",
4676                format!("module_id '{target_module_id}' has no live forwarding connection"),
4677            )?]);
4678        }
4679
4680        if let Some(error) =
4681            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4682        {
4683            self.observe_route_open_refusal(
4684                ctx,
4685                &caller,
4686                &target_module_id,
4687                "op_not_allowed",
4688                "op_not_allowed",
4689            );
4690            return Ok(vec![error]);
4691        }
4692
4693        let principal =
4694            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4695                Ok(principal) => principal,
4696                Err(error) => {
4697                    self.observe_route_open_refusal(
4698                        ctx,
4699                        &caller,
4700                        &target_module_id,
4701                        "bad_consumer_identity",
4702                        "bad_consumer_identity",
4703                    );
4704                    return Ok(vec![error]);
4705                }
4706            };
4707        timings.principal = principal_label(&principal);
4708
4709        // This is attested, control-plane policy for supervised module origins.
4710        // Keep it before route reservation and out of the opaque forwarding hot
4711        // path: data frames must never acquire a per-frame capability check.
4712        if let Principal::Reserved {
4713            module_id: opening_module_id,
4714        } = &principal
4715        {
4716            if let Some(opening_registration) = self
4717                .registry
4718                .get_module(opening_module_id)
4719                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4720            {
4721                if let Some(capability) =
4722                    denied_capability(&opening_registration.manifest, &registration.manifest)
4723                {
4724                    warn!(
4725                        opening_module_id,
4726                        target_module_id,
4727                        capability,
4728                        "refusing route.open because an attested capability deny edge matches"
4729                    );
4730                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4731                        &frame,
4732                        &target_module_id,
4733                        "capability_deny_edge",
4734                        "capability_forbidden",
4735                        format!(
4736                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4737                        ),
4738                    )?]);
4739                }
4740            }
4741        }
4742
4743        if admission_facts.is_some() {
4744            let carrier_matches = matches!(
4745                &principal,
4746                Principal::Reserved { module_id }
4747                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4748            );
4749            if !carrier_matches {
4750                return Ok(vec![self.route_open_refusal_frame(
4751                    ctx,
4752                    &caller,
4753                    &frame,
4754                    &target_module_id,
4755                    "admission_facts_carrier_not_permitted",
4756                    "admission_facts_not_permitted",
4757                    "admission facts may only be carried by the configured reserved module",
4758                )?]);
4759            }
4760
4761            let target_allowed = self
4762                .admission_facts_targets
4763                .as_ref()
4764                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4765            if !target_allowed {
4766                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4767                    &frame,
4768                    &target_module_id,
4769                    "admission_facts_target_not_listed",
4770                    "admission_facts_target_not_allowed",
4771                    format!(
4772                        "admission facts are not permitted for target module_id '{target_module_id}'"
4773                    ),
4774                )?]);
4775            }
4776
4777            // Keep the value opaque to subc. The downstream admission validator owns
4778            // schema and semantic checks; this daemon only enforces carrier authority
4779            // and the configured destination allowlist.
4780        }
4781
4782        // Scope admission, on the attested principal above and never on the
4783        // request body. The tag read here travels with the pending bind and is
4784        // compared with the published one at commit, so a sync between here
4785        // and the module's ack refuses the open instead of binding a stamp
4786        // that is no longer true.
4787        let (bound_scope, mut scope_stamp) = match &scope {
4788            None => (None, None),
4789            Some(selector) => {
4790                let owner_configured = match &selector.owner {
4791                    // A reserved owner counts as configured from before its
4792                    // process is spawned (see `SupervisorHandle::is_configured`).
4793                    // So an owner that has not synced its scopes yet is refused
4794                    // as retryable (`scope_not_synced`), not as one that will
4795                    // never sync.
4796                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4797                    _ => false,
4798                };
4799                let mut scopes = self
4800                    .scopes
4801                    .write()
4802                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4803                let admitted = scopes
4804                    .admit(&principal, &target_module_id, selector, owner_configured)
4805                    .and_then(|admission| {
4806                        crate::scopes::check_target_flow_support(
4807                            &admission.stamp,
4808                            &target_module_id,
4809                            registration.manifest.capabilities.as_ref(),
4810                        )?;
4811                        crate::scopes::check_target_agent_run_support(
4812                            &admission.stamp,
4813                            &target_module_id,
4814                            registration.manifest.capabilities.as_ref(),
4815                        )?;
4816                        let bound = scopes.reserve_bind(
4817                            &admission,
4818                            &principal,
4819                            ctx.connection_id,
4820                            selector.hold_presence,
4821                        );
4822                        Ok((admission, bound))
4823                    });
4824                match admitted {
4825                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4826                    Err(refusal) => {
4827                        return Ok(vec![self.route_open_refusal_frame(
4828                            ctx,
4829                            &caller,
4830                            &frame,
4831                            &target_module_id,
4832                            refusal.code,
4833                            refusal.code,
4834                            refusal.message,
4835                        )?]);
4836                    }
4837                }
4838            }
4839        };
4840        // From here on a refusal names the scope by the stamp admission
4841        // issued, exactly as the `route.open accepted` line does.
4842        if let Some(stamp) = &scope_stamp {
4843            caller.scope = route_open_scope_label(stamp);
4844        }
4845
4846        // Bind admits a root that no longer exists on disk, because refusing here
4847        // closes the only exit from a paused run: cancel needs a bound route, and a
4848        // renamed or reclaimed directory makes that route unopenable forever. The
4849        // run itself is intact and still addressable by its recorded identity.
4850        //
4851        // This does NOT relax the rule the strict constructor protects. That rule is
4852        // that no root is ever aliased into NEW durable state -- a missing component
4853        // can reappear as a symlink elsewhere, which would move the identity and
4854        // split a session's history across two of them. The engine now refuses the
4855        // two operations that create such state (send and import) at admission,
4856        // which is a narrower way to hold the same invariant: reads and terminations
4857        // are admitted, writes are not. That refusal had to ship before this line
4858        // changed, or there is an interval where a send commits under a provisional
4859        // identity -- the exact failure the original policy existed to prevent.
4860        //
4861        // Resolution follows realpath rather than lexical cleanup: the longest
4862        // existing ancestor is canonicalized and the missing tail re-appended, so a
4863        // live root is unchanged and a vanished leaf keeps the identity it was
4864        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4865        // same caller the moment the directory vanished, which strands the run more
4866        // quietly than refusing it.
4867        timings.enter(RouteOpenPhase::ProjectRoot);
4868        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4869            Ok(project_root) => project_root,
4870            Err(err) => {
4871                return Ok(vec![control_error_frame(
4872                    &frame,
4873                    "invalid_project_root",
4874                    err.to_string(),
4875                )?])
4876            }
4877        };
4878        timings.enter(RouteOpenPhase::Admission);
4879        identity.project_root = project_root.as_path().to_path_buf();
4880
4881        // Last gate before any relay work, and deliberately after the cheap
4882        // registry and availability checks above: those name a more precise
4883        // condition (unknown, removed, reloading) and a caller is better served
4884        // by the precise code than by this one.
4885        //
4886        // Everything below this point costs an egress permit, a reserved handle
4887        // pair and, if the module does not answer, the whole relay budget. The
4888        // reader no longer waits for that budget, so cap each target explicitly;
4889        // serial dispatch used to provide the accidental cap of one relay per
4890        // connection. Admission is a mutex-protected count and never waits.
4891        let _concurrency_guard = match self
4892            .route_bind_concurrency
4893            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4894        {
4895            Ok(guard) => guard,
4896            Err(in_flight) => {
4897                return Ok(vec![self.route_open_target_capacity_refusal(
4898                    ctx,
4899                    &caller,
4900                    &frame,
4901                    &target_module_id,
4902                    in_flight,
4903                )?]);
4904            }
4905        };
4906
4907        // A module that has already burned the whole budget `threshold` times
4908        // in a row does not get to charge it again until a probe says it recovered.
4909        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4910            RouteBindAdmission::Admitted { guard, probe } => {
4911                if probe {
4912                    info!(
4913                        module_id = %target_module_id,
4914                        connection_id = ctx.connection_id.get(),
4915                        "route.bind breaker half-open: admitting one probe"
4916                    );
4917                }
4918                guard
4919            }
4920            RouteBindAdmission::Refused {
4921                consecutive_timeouts,
4922                retry_in,
4923                probe_in_flight,
4924            } => {
4925                return Ok(vec![self.route_open_breaker_refusal_frame(
4926                    ctx,
4927                    &caller,
4928                    &frame,
4929                    &target_module_id,
4930                    consecutive_timeouts,
4931                    retry_in,
4932                    probe_in_flight,
4933                )?]);
4934            }
4935        };
4936
4937        // Resolve the per-module budget here so the wait matches the operator's
4938        // intent for this specific target. A per-module override in
4939        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4940        // daemons) wins over the daemon-wide default.
4941        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4942        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4943        timings.enter(RouteOpenPhase::EgressReservation);
4944        let pending = match self
4945            .forwarding
4946            .begin_route_bind_relay_for(
4947                ctx.connection_id,
4948                ctx.egress.clone(),
4949                response_version(&frame),
4950                frame.header.corr,
4951                &target_module_id,
4952                principal.clone(),
4953                bound_scope,
4954                Some(project_root),
4955                relay_deadline,
4956            )
4957            .await
4958        {
4959            Ok(pending) => pending,
4960            Err(err) => {
4961                return Ok(vec![self.route_open_refusal_frame(
4962                    ctx,
4963                    &caller,
4964                    &frame,
4965                    &target_module_id,
4966                    "relay_reservation_failed",
4967                    forwarding_error_code(&err),
4968                    err.to_string(),
4969                )?])
4970            }
4971        };
4972        timings.enter(RouteOpenPhase::Admission);
4973        let crate::forwarding::PendingRouteBindRelay {
4974            endpoint,
4975            module_sink,
4976            negotiated_ver,
4977            client_channel,
4978            client_epoch,
4979            module_channel,
4980            module_epoch,
4981            corr: relay_corr,
4982            receiver,
4983        } = pending;
4984        let mut reservation =
4985            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4986
4987        // Reserving egress can wait while a module reconnects or a swap cuts
4988        // over. Check the relay's captured connection, not the earlier by-id
4989        // lookup: the original module's flow/run capabilities cannot authorize
4990        // its replacement. The captured sink stays bound to that connection.
4991        let relay_registration = self
4992            .registry
4993            .get_module_by_connection(endpoint.connection_id)
4994            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4995        let relay_capabilities = relay_registration
4996            .as_ref()
4997            .and_then(|registration| registration.manifest.capabilities.as_ref());
4998        if let Some(selector) = &scope {
4999            // Egress reservation may wait across a scope update. Recheck the
5000            // proof against the current record before sending the captured bind.
5001            if let Err(refusal) = self
5002                .scopes
5003                .read()
5004                .unwrap_or_else(|poisoned| poisoned.into_inner())
5005                .check_bind_proof(&principal, selector)
5006            {
5007                reservation.release_and_disarm();
5008                return Ok(vec![self.route_open_refusal_frame(
5009                    ctx,
5010                    &caller,
5011                    &frame,
5012                    &target_module_id,
5013                    refusal.code,
5014                    refusal.code,
5015                    refusal.message,
5016                )?]);
5017            }
5018        }
5019        if let Some(stamp) = scope_stamp
5020            .as_ref()
5021            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5022        {
5023            if let Err(refusal) = crate::scopes::check_target_flow_support(
5024                stamp,
5025                &target_module_id,
5026                relay_capabilities,
5027            )
5028            .and_then(|()| {
5029                crate::scopes::check_target_agent_run_support(
5030                    stamp,
5031                    &target_module_id,
5032                    relay_capabilities,
5033                )
5034            }) {
5035                reservation.release_and_disarm();
5036                return Ok(vec![self.route_open_refusal_frame(
5037                    ctx,
5038                    &caller,
5039                    &frame,
5040                    &target_module_id,
5041                    refusal.code,
5042                    refusal.code,
5043                    refusal.message,
5044                )?]);
5045            }
5046        }
5047
5048        // Older readers may reject unknown fields. Select optional fields from
5049        // the captured endpoint's declaration, including on unscoped binds.
5050        if !relay_capabilities.is_some_and(|caps| {
5051            caps.provides
5052                .iter()
5053                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5054        }) {
5055            if let Some(stamp) = scope_stamp.as_mut() {
5056                stamp.workdir = None;
5057                stamp.project_id = None;
5058            }
5059        }
5060        if !relay_capabilities.is_some_and(|caps| {
5061            caps.provides
5062                .iter()
5063                .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5064        }) {
5065            if let Some(stamp) = scope_stamp.as_mut() {
5066                stamp.attributes.on_behalf_of_agent = None;
5067            }
5068        }
5069        if !relay_capabilities.is_some_and(|caps| {
5070            caps.provides
5071                .iter()
5072                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5073        }) {
5074            if let Some(stamp) = scope_stamp.as_mut() {
5075                stamp.attributes.flow_version = None;
5076                stamp.attributes.flow_code_hash = None;
5077                stamp.attributes.flow_install_id = None;
5078            }
5079        }
5080        let call_mode = if relay_capabilities
5081            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5082        {
5083            Some(derive_call_mode(scope_stamp.as_ref()))
5084        } else {
5085            if let Some(stamp) = scope_stamp.as_mut() {
5086                stamp.expires_at_ms = None;
5087            }
5088            None
5089        };
5090
5091        debug!(
5092            connection_id = ctx.connection_id.get(),
5093            client_channel,
5094            client_epoch,
5095            module_channel,
5096            module_epoch,
5097            "reserved route handle pair"
5098        );
5099        // Rendered BEFORE the move into the relay, because the accept arm below
5100        // is where it is logged and the principal is gone by then.
5101        let route_principal_label = principal_label(&principal);
5102        let relay = ModuleControlRequest::RouteBind {
5103            route_channel: module_channel,
5104            epoch: module_epoch,
5105            target,
5106            identity,
5107            principal: Some(principal),
5108            consumer_capabilities,
5109            role_versions,
5110            admission_facts,
5111            scope: scope_stamp,
5112            call_mode,
5113        };
5114        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5115            RouterError::backend(
5116                0,
5117                frame.header.corr,
5118                format!("failed to encode route.bind request: {err}"),
5119            )
5120        })?;
5121        let relay_frame = Frame::build_with_version(
5122            negotiated_ver,
5123            FrameType::Request,
5124            control_flags(),
5125            0,
5126            0,
5127            relay_corr,
5128            relay_body,
5129        )
5130        .map_err(RouterError::FrameBuild)?;
5131
5132        timings.enter(RouteOpenPhase::BindSend);
5133        let sent = module_sink.send(relay_frame).await;
5134        timings.enter(RouteOpenPhase::Admission);
5135        if let Err(err) = sent {
5136            reservation.release_and_disarm();
5137            return Ok(vec![self.route_open_refusal_frame(
5138                ctx,
5139                &caller,
5140                &frame,
5141                &target_module_id,
5142                "relay_send_failed",
5143                "target_unavailable",
5144                err.to_string(),
5145            )?]);
5146        }
5147
5148        #[cfg(test)]
5149        if let Some(hold) = &self.route_bind_relay_sent_hold {
5150            // First wait: tell the test the bind is queued. Second wait: stay
5151            // here until the test has driven whatever it wants to happen
5152            // between queueing the bind and marking it enqueued.
5153            hold.wait().await;
5154            hold.wait().await;
5155        }
5156
5157        // A missing relay is not by itself an abandoned bind: the module may
5158        // already have acked it and the route may be committed. Only a relay
5159        // dropped without any module reply needs this GOODBYE; one a reply
5160        // settled has already had its GOODBYE decided by the reply path.
5161        if self
5162            .forwarding
5163            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5164            .map_err(RouterError::Forwarding)?
5165            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5166        {
5167            self.send_abandoned_route_bind_goodbye(
5168                &module_sink,
5169                negotiated_ver,
5170                module_channel,
5171                module_epoch,
5172            );
5173        }
5174
5175        timings.enter(RouteOpenPhase::BindAckWait);
5176        let outcome = timeout_at(relay_deadline, receiver).await;
5177        timings.enter(RouteOpenPhase::Admission);
5178        match outcome {
5179            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5180                reservation.disarm();
5181                if breaker.record_accepted() {
5182                    info!(
5183                        module_id = %target_module_id,
5184                        "route.bind breaker closed: the probe was accepted"
5185                    );
5186                }
5187                self.observe_route_open_accept(
5188                    ctx,
5189                    &target_module_id,
5190                    &route_principal_label,
5191                    &caller.scope,
5192                );
5193                Ok(Vec::new())
5194            }
5195            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5196                reservation.release_and_disarm();
5197                // A module that says no in microseconds is healthy. Rejection
5198                // is a different condition with its own refusal and must not
5199                // move the breaker.
5200                breaker.record_inconclusive();
5201                // The daemon's own commit re-check refused the bind because the
5202                // scope ended or changed after admission. The module accepted;
5203                // counting it as a module rejection would blame the module.
5204                let scope_code = match body.code.as_str() {
5205                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5206                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5207                    _ => None,
5208                };
5209                if let Some(code) = scope_code {
5210                    self.observe_route_open_refusal(
5211                        ctx,
5212                        &caller,
5213                        &target_module_id,
5214                        "scope_changed_before_commit",
5215                        code,
5216                    );
5217                    return Ok(vec![control_error_body_frame(&frame, body)?]);
5218                }
5219                self.counters
5220                    .increment_route_open_refused("module_rejected");
5221                info!(
5222                    target: "control",
5223                    code = "module_rejected",
5224                    module_code = ?body.code,
5225                    module_id = ?target_module_id,
5226                    connection_id = ctx.connection_id.get(),
5227                    principal = %caller.principal,
5228                    scope = %caller.scope,
5229                    harness = %caller.harness,
5230                    "route.open refused"
5231                );
5232                Ok(vec![control_error_body_frame(&frame, body)?])
5233            }
5234            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5235                reservation.release_and_disarm();
5236                breaker.record_inconclusive();
5237                // Fires when the module's connection closes while a relayed
5238                // bind is pending -- typically a caller racing a module restart
5239                // whose bind was relayed BEFORE the drain mark went up. Logged
5240                // because the caller sees only its own error and the fleet has
5241                // already spent one diagnosis round unable to tell this arm
5242                // from a relay timeout without daemon-side evidence.
5243                tracing::warn!(
5244                    module_id = %target_module_id,
5245                    "route.bind relay abandoned: {message}"
5246                );
5247                Ok(vec![self.route_open_refusal_frame(
5248                    ctx,
5249                    &caller,
5250                    &frame,
5251                    &target_module_id,
5252                    "relay_abandoned",
5253                    "target_unavailable",
5254                    message,
5255                )?])
5256            }
5257            Ok(Err(_)) => {
5258                reservation.release_and_disarm();
5259                breaker.record_inconclusive();
5260                Ok(vec![self.route_open_refusal_frame(
5261                    ctx,
5262                    &caller,
5263                    &frame,
5264                    &target_module_id,
5265                    "relay_waiter_canceled",
5266                    "target_unavailable",
5267                    "route.bind relay waiter was canceled before the module responded",
5268                )?])
5269            }
5270            Err(_) => {
5271                reservation.release_and_disarm();
5272                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5273                // answer at all is the one condition a fast refusal can
5274                // usefully stand in for; every other arm already answered.
5275                if let Some(opened) = breaker.record_timeout(
5276                    self.route_bind_breaker_threshold,
5277                    self.route_bind_breaker_cooldown,
5278                ) {
5279                    warn!(
5280                        module_id = %target_module_id,
5281                        consecutive_timeouts = opened.consecutive_timeouts,
5282                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5283                        reopened_after_probe = opened.reopened_after_probe,
5284                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5285                    );
5286                }
5287                // The generous budget just burned to no answer: the module is
5288                // registered and its connection is up, but its bind handler sat
5289                // on the ack for the full budget (warm-on-bind, cold configure,
5290                // or a wedged handler). Every earlier unavailability shape
5291                // fast-refuses BEFORE the relay, so this arm firing means the
5292                // slowness is module-side -- log it so the per-module timeline
5293                // is reconstructable without client audit rows.
5294                tracing::warn!(
5295                    module_id = %target_module_id,
5296                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5297                    "route.bind relay timed out: module did not ack within budget"
5298                );
5299                Ok(vec![self.route_open_refusal_frame(
5300                    ctx,
5301                    &caller,
5302                    &frame,
5303                    &target_module_id,
5304                    "relay_timed_out",
5305                    "module_timeout",
5306                    format!(
5307                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5308                        route_bind_relay_timeout
5309                    ),
5310                )?])
5311            }
5312        }
5313    }
5314
5315    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5316        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5317            snapshot: self.supervisor.spawn_snapshot(),
5318        };
5319        Ok(vec![control_response_body_frame(
5320            &frame,
5321            &response,
5322            "ClientControlResponse::SupervisorSpawnSnapshot",
5323        )?])
5324    }
5325
5326    fn handle_supervisor_spawn_subscribe(
5327        &self,
5328        ctx: &RouteCtx,
5329        frame: Frame,
5330        since: Option<SpawnCursor>,
5331    ) -> Result<Vec<Frame>, RouterError> {
5332        match self.supervisor.subscribe_spawns(
5333            ctx.connection_id,
5334            frame.header.corr,
5335            response_version(&frame),
5336            since,
5337            ctx.egress.clone(),
5338        ) {
5339            Ok(()) => Ok(Vec::new()),
5340            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5341                Ok(vec![control_error_body_frame(
5342                    &frame,
5343                    ErrorBody {
5344                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5345                        message: "spawn cursor belongs to a different daemon incarnation"
5346                            .to_string(),
5347                        detail: Some(serde_json::json!({
5348                            "current_daemon_incarnation": current
5349                        })),
5350                    },
5351                )?])
5352            }
5353            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5354                &frame,
5355                ErrorBody {
5356                    code: "spawn_cursor_too_old".to_string(),
5357                    message: "spawn cursor predates the retained event ring".to_string(),
5358                    detail: Some(serde_json::json!({
5359                        "oldest_retained_cursor": oldest
5360                    })),
5361                },
5362            )?]),
5363            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5364                0,
5365                frame.header.corr,
5366                format!("failed to open supervisor spawn subscription: {error}"),
5367            )),
5368        }
5369    }
5370
5371    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5372        let generation = self
5373            .registry
5374            .generation()
5375            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5376        let mut modules = Vec::new();
5377        for module in self.supervisor.list() {
5378            let status = module.status_for_control("list").map_err(|err| {
5379                RouterError::backend(
5380                    0,
5381                    frame.header.corr,
5382                    format!("failed to read supervisor status: {err}"),
5383                )
5384            })?;
5385            let (configured, _) = module.configuration().map_err(|err| {
5386                RouterError::backend(
5387                    0,
5388                    frame.header.corr,
5389                    format!("failed to read module configuration: {err}"),
5390                )
5391            })?;
5392            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5393                RouterError::backend(
5394                    0,
5395                    frame.header.corr,
5396                    format!("failed to read retry time: {err}"),
5397                )
5398            })?;
5399            // Status and configuration snapshots release their locks before the image probe awaits.
5400            let image = module.running_image_agreement().await;
5401            // Read per request so the figure is current when the operator asks;
5402            // the daemon samples nothing in between.
5403            let resources = Some(module.child_resource_usage());
5404            let mut verdict =
5405                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5406            verdict.spec = Some(launch_spec_agreement(
5407                &configured,
5408                status.process_alive,
5409                status.launch_spec_digest.as_ref(),
5410            ));
5411            let pending_reload = Some(verdict);
5412            let configured_enabled = self
5413                .capability_evaluator
5414                .configured_enabled(&status.module_id);
5415            // Keep the retired policy field on the wire for one release so
5416            // existing status consumers still receive the platform policy.
5417            modules.push(
5418                SupervisorEntry::new(
5419                    status.module_id,
5420                    status.state.to_string(),
5421                    status.enabled,
5422                    status.live,
5423                    status.health.status,
5424                )
5425                .with_configured_enabled(configured_enabled)
5426                .with_next_retry_at_ms(next_retry_at_ms)
5427                .with_launch_nonce_env(Some(!cfg!(unix)))
5428                .with_launch_nonce_source(status.launch_nonce_source)
5429                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5430                .with_protocol(status.protocol)
5431                .with_pending_reload(pending_reload)
5432                .with_last_probe_ms(status.health.last_probe_ms)
5433                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5434                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5435                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5436                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5437                .with_restart_count(Some(status.restart_count))
5438                .with_max_restarts(Some(status.max_restarts))
5439                .with_lifetime_restarts(Some(status.lifetime_restarts))
5440                .with_spawn_generation(Some(status.spawn_generation))
5441                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5442                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5443                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5444                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5445                .with_resources(resources),
5446            );
5447        }
5448        let response = ClientControlResponse::SupervisorList {
5449            generation,
5450            modules,
5451        };
5452        Ok(vec![control_response_body_frame(
5453            &frame,
5454            &response,
5455            "ClientControlResponse::SupervisorList",
5456        )?])
5457    }
5458
5459    fn handle_supervisor_stderr_tail(
5460        &self,
5461        frame: Frame,
5462        module_id: String,
5463        max_lines: Option<u32>,
5464        max_bytes: Option<u32>,
5465    ) -> Result<Vec<Frame>, RouterError> {
5466        let Some(module) = self.supervisor.get(&module_id) else {
5467            return Ok(vec![control_error_frame(
5468                &frame,
5469                "unknown_module",
5470                format!("module_id '{module_id}' is not supervised"),
5471            )?]);
5472        };
5473
5474        let snapshot = module.stderr_tail(
5475            max_lines.map(|value| value as usize),
5476            max_bytes.map(|value| value as usize),
5477        );
5478
5479        let response = ClientControlResponse::SupervisorStderrTail {
5480            module_id,
5481            tail: StderrTail {
5482                capture: match snapshot.capture {
5483                    CaptureState::Captured => StderrCaptureState::Captured,
5484                    CaptureState::Incomplete { reason } => {
5485                        StderrCaptureState::Incomplete { reason }
5486                    }
5487                    CaptureState::NotCaptured { reason } => {
5488                        StderrCaptureState::NotCaptured { reason }
5489                    }
5490                },
5491                entries: snapshot
5492                    .entries
5493                    .into_iter()
5494                    .map(|entry| match entry {
5495                        TailEntry::Line {
5496                            text,
5497                            truncated,
5498                            at_ms,
5499                        } => StderrTailEntry::Line {
5500                            text,
5501                            truncated,
5502                            at_ms,
5503                        },
5504                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5505                    })
5506                    .collect(),
5507                dropped_lines: snapshot.dropped_lines,
5508            },
5509        };
5510        Ok(vec![control_response_body_frame(
5511            &frame,
5512            &response,
5513            "ClientControlResponse::SupervisorStderrTail",
5514        )?])
5515    }
5516
5517    async fn handle_supervisor_terminals(
5518        &self,
5519        frame: Frame,
5520        module_id: String,
5521    ) -> Result<Vec<Frame>, RouterError> {
5522        let Some(module) = self.supervisor.get(&module_id) else {
5523            return Ok(vec![control_error_frame(
5524                &frame,
5525                "unknown_module",
5526                format!("module_id '{module_id}' is not supervised"),
5527            )?]);
5528        };
5529
5530        // The journal read runs on a blocking thread: it can be megabytes of
5531        // file I/O and must not occupy a runtime worker.
5532        let terminals = module
5533            .read_durable_terminal_history()
5534            .await
5535            .map_err(|error| {
5536                RouterError::backend(
5537                    0,
5538                    frame.header.corr,
5539                    format!("failed to read terminal history: {error}"),
5540                )
5541            })?;
5542        let response = ClientControlResponse::SupervisorTerminals {
5543            module_id,
5544            terminals,
5545        };
5546        Ok(vec![control_response_body_frame(
5547            &frame,
5548            &response,
5549            "ClientControlResponse::SupervisorTerminals",
5550        )?])
5551    }
5552
5553    fn handle_supervisor_routes(
5554        &self,
5555        frame: Frame,
5556        module_id: Option<String>,
5557    ) -> Result<Vec<Frame>, RouterError> {
5558        let modules = self
5559            .forwarding
5560            .route_census(module_id.as_deref())
5561            .map_err(RouterError::Forwarding)?
5562            .into_iter()
5563            .map(|(module_id, routes)| SupervisorRouteModule {
5564                module_id,
5565                routes: routes
5566                    .into_iter()
5567                    .map(|route| SupervisorRoute {
5568                        consumer: match route.principal {
5569                            Principal::Reserved { module_id } => {
5570                                SupervisorRouteConsumer::Reserved { module_id }
5571                            }
5572                            Principal::Direct | Principal::Unverified => {
5573                                SupervisorRouteConsumer::Direct {
5574                                    connection_id: route.goodbye_target.connection_id.get(),
5575                                }
5576                            }
5577                        },
5578                        age_ms: Instant::now()
5579                            .saturating_duration_since(route.bound_at)
5580                            .as_millis()
5581                            .try_into()
5582                            .unwrap_or(u64::MAX),
5583                        draining: route.draining,
5584                        drain_reason: route.drain_reason,
5585                    })
5586                    .collect(),
5587            })
5588            .collect();
5589        let response = ClientControlResponse::SupervisorRoutes { modules };
5590        Ok(vec![control_response_body_frame(
5591            &frame,
5592            &response,
5593            "ClientControlResponse::SupervisorRoutes",
5594        )?])
5595    }
5596
5597    async fn handle_supervisor_provenance(
5598        &self,
5599        frame: Frame,
5600        module_id: Option<String>,
5601    ) -> Result<Vec<Frame>, RouterError> {
5602        let mut selected = if let Some(module_id) = module_id {
5603            let Some(module) = self.supervisor.get(&module_id) else {
5604                return Ok(vec![control_error_frame(
5605                    &frame,
5606                    "unknown_module",
5607                    format!("module_id '{module_id}' is not supervised"),
5608                )?]);
5609            };
5610            vec![module]
5611        } else {
5612            self.supervisor.list()
5613        };
5614
5615        let mut modules = Vec::with_capacity(selected.len());
5616        for module in selected.drain(..) {
5617            let (status, observed_image) = module
5618                .status_and_running_image_agreement()
5619                .await
5620                .map_err(|err| {
5621                    RouterError::backend(
5622                        0,
5623                        frame.header.corr,
5624                        format!("failed to read supervisor status: {err}"),
5625                    )
5626                })?;
5627            let module_declared = self
5628                .registry
5629                .get_module(&status.module_id)
5630                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5631                .and_then(|registration| registration.manifest.provenance)
5632                .map(|build| ModuleDeclaredProvenance::Reported { build })
5633                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5634            #[cfg(test)]
5635            let running_image = match &self.provenance_probe_override {
5636                Some(result) => result.clone(),
5637                None => observed_image,
5638            };
5639            #[cfg(not(test))]
5640            let running_image = observed_image;
5641            modules.push(SupervisorModuleProvenance {
5642                module_id: status.module_id,
5643                module_declared,
5644                daemon_observed: SupervisorObservedProcess {
5645                    launch_nonce_source: status.launch_nonce_source,
5646                    pid: status.pid,
5647                    spawned_at_ms: status.spawned_at_ms,
5648                    spawned_from: status.spawned_from,
5649                    running_image,
5650                },
5651            });
5652        }
5653        let daemon = SupervisorDaemonProvenance {
5654            daemon_build: self.daemon_provenance.build.clone(),
5655            daemon_observed: DaemonObservedProcess {
5656                pid: self.daemon_provenance.pid,
5657                started_at_ms: self
5658                    .daemon_provenance
5659                    .start_clock
5660                    .map(|clock| clock.started_at_ms())
5661                    .or(self.daemon_provenance.started_at_ms),
5662                running_image: self
5663                    .daemon_provenance
5664                    .probe
5665                    .observe(
5666                        self.daemon_provenance.pid,
5667                        self.daemon_provenance.executable_path.as_deref(),
5668                        self.daemon_provenance.executable_identity.clone(),
5669                        self.daemon_provenance.process_start_time,
5670                    )
5671                    .await,
5672            },
5673        };
5674        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5675        Ok(vec![control_response_body_frame(
5676            &frame,
5677            &response,
5678            "ClientControlResponse::SupervisorProvenance",
5679        )?])
5680    }
5681
5682    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5683        self.refresh_capability_requirements();
5684        let generation = self
5685            .registry
5686            .generation()
5687            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5688        let modules = self
5689            .supervisor
5690            .list()
5691            .into_iter()
5692            .map(|module| {
5693                let status = module.status_for_control("health").map_err(|err| {
5694                    RouterError::backend(
5695                        0,
5696                        frame.header.corr,
5697                        format!("failed to read supervisor health: {err}"),
5698                    )
5699                })?;
5700                let module_id = status.module_id;
5701                let capability_detail = self
5702                    .capability_evaluator
5703                    .required_problem_detail(&module_id);
5704                Ok(SupervisorHealthEntry {
5705                    module_id,
5706                    status: status.health.status,
5707                    detail: append_capability_problem_detail(
5708                        status.health.detail,
5709                        capability_detail,
5710                    ),
5711                    metrics: status.health.metrics,
5712                    consecutive_failures: status.health.consecutive_failures,
5713                    late_answer_count: status.health.late_answer_count,
5714                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5715                    last_action: status.health.last_action,
5716                    last_action_ms: status.health.last_action_ms,
5717                    last_probe_ms: status.health.last_probe_ms,
5718                })
5719            })
5720            .collect::<Result<Vec<_>, RouterError>>()?;
5721        let response = ClientControlResponse::SupervisorHealth {
5722            generation,
5723            modules,
5724        };
5725        Ok(vec![control_response_body_frame(
5726            &frame,
5727            &response,
5728            "ClientControlResponse::SupervisorHealth",
5729        )?])
5730    }
5731
5732    async fn handle_supervisor_restart(
5733        &self,
5734        frame: Frame,
5735        module_id: String,
5736        drain_timeout_ms: Option<u64>,
5737        requester: Principal,
5738    ) -> Result<Vec<Frame>, RouterError> {
5739        let operation_lock = self.supervisor.operation_lock();
5740        let _operation_guard = operation_lock.lock().await;
5741        let Some(module) = self.supervisor.get(&module_id) else {
5742            return Ok(vec![control_error_frame(
5743                &frame,
5744                "unknown_module",
5745                format!("module_id '{module_id}' is not supervised"),
5746            )?]);
5747        };
5748
5749        self.route_outages.mark_operator_action(&module_id);
5750        if let Err(err) = module
5751            .restart_requested(drain_timeout_ms, Some(requester))
5752            .await
5753        {
5754            self.route_outages
5755                .operator_action_ended_unrefused(&module_id);
5756            let (code, message) = match err {
5757                crate::supervise::SuperviseError::Disabled { .. } => {
5758                    ("module_disabled", err.to_string())
5759                }
5760                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5761                    ("swap_in_progress", err.to_string())
5762                }
5763                _ => (
5764                    "target_unavailable",
5765                    format!("failed to restart module_id '{module_id}': {err}"),
5766                ),
5767            };
5768            return Ok(vec![control_error_frame(&frame, code, message)?]);
5769        }
5770
5771        let response = ClientControlResponse::SupervisorAck {
5772            module_id,
5773            applied: true,
5774        };
5775        Ok(vec![control_response_body_frame(
5776            &frame,
5777            &response,
5778            "ClientControlResponse::SupervisorAck",
5779        )?])
5780    }
5781
5782    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5783    /// when the old process has finished draining: a caller whose own lane
5784    /// rides the old process must get its reply before that drain waits on it.
5785    async fn handle_supervisor_swap(
5786        &self,
5787        frame: Frame,
5788        module_id: String,
5789        ready_timeout_ms: Option<u64>,
5790        requester: Principal,
5791    ) -> Result<Vec<Frame>, RouterError> {
5792        // The daemon-wide operation lock is held only to resolve the handle,
5793        // not across the swap. The swap can take its whole readiness budget,
5794        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5795        // holding it here would park an operator's stop behind the swap it is
5796        // meant to abort. A rescan or stop that reaches the module during the
5797        // swap is served by the swap itself (see `supervise_swap`).
5798        let module = {
5799            let operation_lock = self.supervisor.operation_lock();
5800            let _operation_guard = operation_lock.lock().await;
5801            self.supervisor.get(&module_id)
5802        };
5803        let Some(module) = module else {
5804            return Ok(vec![control_error_frame(
5805                &frame,
5806                "unknown_module",
5807                format!("module_id '{module_id}' is not supervised"),
5808            )?]);
5809        };
5810
5811        self.route_outages.mark_operator_action(&module_id);
5812        if let Err(err) = module
5813            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5814            .await
5815        {
5816            self.route_outages
5817                .operator_action_ended_unrefused(&module_id);
5818            use crate::supervise::SuperviseError;
5819            let message = err.to_string();
5820            let error = match err {
5821                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5822                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5823                    code: "swap_refused".to_string(),
5824                    message,
5825                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5826                },
5827                SuperviseError::SwapFailed {
5828                    arm,
5829                    candidate_exit,
5830                    ..
5831                } => ErrorBody {
5832                    code: "swap_failed".to_string(),
5833                    message,
5834                    detail: Some(serde_json::json!({
5835                        "arm": arm.as_str(),
5836                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5837                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5838                    })),
5839                },
5840                _ => ErrorBody::new(
5841                    "target_unavailable",
5842                    format!("failed to swap module_id '{module_id}': {message}"),
5843                ),
5844            };
5845            return Ok(vec![control_error_body_frame(&frame, error)?]);
5846        }
5847        // A completed swap kept the incumbent serving until cutover, so it
5848        // usually opened no outage; a mark left behind would make the next,
5849        // unrelated outage read as requested.
5850        self.route_outages
5851            .operator_action_ended_unrefused(&module_id);
5852
5853        let response = ClientControlResponse::SupervisorAck {
5854            module_id,
5855            applied: true,
5856        };
5857        Ok(vec![control_response_body_frame(
5858            &frame,
5859            &response,
5860            "ClientControlResponse::SupervisorAck",
5861        )?])
5862    }
5863
5864    async fn handle_supervisor_reload(
5865        &self,
5866        frame: Frame,
5867        module_id: String,
5868        requester: Principal,
5869    ) -> Result<Vec<Frame>, RouterError> {
5870        let operation_lock = self.supervisor.operation_lock();
5871        let _operation_guard = operation_lock.lock().await;
5872        let Some(module) = self.supervisor.get(&module_id) else {
5873            return Ok(vec![control_error_frame(
5874                &frame,
5875                "unknown_module",
5876                format!("module_id '{module_id}' is not supervised"),
5877            )?]);
5878        };
5879
5880        self.route_outages.mark_operator_action(&module_id);
5881        if let Err(err) = module.reload_requested(Some(requester)).await {
5882            self.route_outages
5883                .operator_action_ended_unrefused(&module_id);
5884            let (code, message) = match err {
5885                crate::supervise::SuperviseError::Disabled { .. } => {
5886                    ("module_disabled", err.to_string())
5887                }
5888                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5889                    ("swap_in_progress", err.to_string())
5890                }
5891                _ => (
5892                    "reload_failed",
5893                    format!("failed to reload module_id '{module_id}': {err}"),
5894                ),
5895            };
5896            return Ok(vec![control_error_frame(&frame, code, message)?]);
5897        }
5898
5899        let response = ClientControlResponse::SupervisorAck {
5900            module_id,
5901            applied: true,
5902        };
5903        Ok(vec![control_response_body_frame(
5904            &frame,
5905            &response,
5906            "ClientControlResponse::SupervisorAck",
5907        )?])
5908    }
5909
5910    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5911    /// the configuration comes from and whether to apply it; every mode then
5912    /// runs the same comparison, so a preview describes the apply it previews.
5913    async fn handle_supervisor_rescan(
5914        &self,
5915        frame: Frame,
5916        mode: RescanMode,
5917        requester: Principal,
5918    ) -> Result<Vec<Frame>, RouterError> {
5919        let preview = mode.previews();
5920        let Some(context) = self.rescan.clone() else {
5921            return Ok(vec![control_error_frame(
5922                &frame,
5923                "rescan_unavailable",
5924                "the daemon was not started with a reloadable config path".to_string(),
5925            )?]);
5926        };
5927
5928        let operation_lock = self.supervisor.operation_lock();
5929        let _operation_guard = operation_lock.lock().await;
5930        // Loaded under the operation lock, so no other rescan can change the
5931        // staged configuration between a checked apply's comparison and the
5932        // apply itself.
5933        let loaded = match self.load_rescan_config(&mode, &context) {
5934            Ok(config) => config,
5935            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5936        };
5937        // `load` reports a missing file as Ok(None), which is correct at boot
5938        // (no config, nothing to supervise) and catastrophic here: rescan treats
5939        // "not in the config" as "remove it", so an absent file would read as an
5940        // empty module list and retire the entire running fleet. An editor
5941        // writing via write-new-then-rename, or a half-finished edit, is enough
5942        // to open that window. Refuse instead: a config that cannot be read
5943        // carries no instruction to remove anything.
5944        let Some((config, activity_config)) = loaded else {
5945            return Ok(vec![control_error_frame(
5946                &frame,
5947                "invalid_daemon_config",
5948                format!(
5949                    "daemon config not found at {}; refusing to rescan (an absent config would \
5950                     retire every supervised module)",
5951                    context.config_path.display()
5952                ),
5953            )?]);
5954        };
5955        let (
5956            configured_port,
5957            storage_config,
5958            admission_facts_carrier_module_id,
5959            admission_facts_targets,
5960            scope_authority_owners,
5961            privileged_operator_prompt,
5962            modules,
5963            reserved_capabilities,
5964        ) = (
5965            config.port,
5966            config.storage,
5967            config.admission_facts_carrier_module_id,
5968            config.admission_facts_targets,
5969            config.scope_authority_owners,
5970            config.privileged_operator_prompt,
5971            config.modules,
5972            config.reserved_capabilities,
5973        );
5974
5975        // Collect the sections rescan cannot apply, so the REPLY carries them.
5976        //
5977        // The warning below has always been correct and has always gone only to
5978        // the journal -- addressed to whoever reads logs, while the person who
5979        // just edited the config is looking at the CLI. Naming each section
5980        // individually rather than setting a flag: "something outside modules
5981        // changed" sends the operator back to diffing their own file, which is
5982        // the work this is meant to save.
5983        let mut restart_required = Vec::new();
5984        for section in RestartRequiredSection::ALL {
5985            let changed = match section {
5986                RestartRequiredSection::Port => configured_port != context.configured_port,
5987                RestartRequiredSection::Storage => storage_config != context.storage_config,
5988                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5989                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5990                }
5991                RestartRequiredSection::AdmissionFactsTargets => {
5992                    admission_facts_targets != context.admission_facts_targets
5993                }
5994                RestartRequiredSection::ScopeAuthorityOwners => {
5995                    scope_authority_owners != context.scope_authority_owners
5996                }
5997                RestartRequiredSection::PrivilegedOperatorPrompt => {
5998                    privileged_operator_prompt != context.privileged_operator_prompt
5999                }
6000            };
6001            if changed {
6002                restart_required.push(section.label().to_string());
6003            }
6004        }
6005        if !restart_required.is_empty() {
6006            warn!(
6007                config_path = %context.config_path.display(),
6008                sections = %restart_required.join(", "),
6009                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6010            );
6011        }
6012
6013        for configured in &modules {
6014            if let Err(err) = validate_spec(&configured.module_spec()) {
6015                return Ok(vec![control_error_frame(
6016                    &frame,
6017                    "invalid_daemon_config",
6018                    format!("supervisor rescan rejected daemon config: {err}"),
6019                )?]);
6020            }
6021        }
6022
6023        let configured_capabilities = modules
6024            .iter()
6025            .map(|module| (module.module_id.clone(), module.enabled))
6026            .collect::<Vec<_>>();
6027        let preview_capability_warnings = if preview {
6028            let (_, registrations) = self.runtime_capability_snapshot()?;
6029            let current_modules = self
6030                .supervisor
6031                .list()
6032                .into_iter()
6033                .map(|module| module.module_id().to_string())
6034                .collect::<BTreeSet<_>>();
6035            let resulting_modules = configured_capabilities.clone();
6036            let removed = current_modules
6037                .into_iter()
6038                .filter(|module_id| {
6039                    !resulting_modules
6040                        .iter()
6041                        .any(|(configured_id, _)| configured_id == module_id)
6042                })
6043                .collect::<Vec<_>>();
6044            // A candidate preview also names providers the candidate disables.
6045            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6046            let disabled = match &mode {
6047                RescanMode::Candidate(_) => resulting_modules
6048                    .iter()
6049                    .filter(|(_, enabled)| !enabled)
6050                    .map(|(module_id, _)| module_id.clone())
6051                    .collect::<Vec<_>>(),
6052                _ => Vec::new(),
6053            };
6054            self.capability_evaluator.preview_warnings(
6055                resulting_modules,
6056                &removed,
6057                &disabled,
6058                &registrations,
6059            )
6060        } else {
6061            Vec::new()
6062        };
6063        let reconciled = match &mode {
6064            RescanMode::Candidate(candidate) => self
6065                .preview_candidate(&context, modules, candidate)
6066                .map(|result| (result, Vec::new())),
6067            _ => {
6068                self.reconcile_supervised_modules(
6069                    &context.supervisor,
6070                    modules,
6071                    preview,
6072                    requester.clone(),
6073                )
6074                .await
6075            }
6076        };
6077        let (result, changed) = match reconciled {
6078            Ok(result) => result,
6079            Err(message) => {
6080                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6081            }
6082        };
6083        if !preview {
6084            if let Some(ledger) = &self.activity {
6085                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6086                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6087            }
6088            if let Err(code) = self.configure_activity(activity_config).await {
6089                warn!(
6090                    code,
6091                    "activity retention could not be applied during rescan"
6092                );
6093            }
6094            self.capability_evaluator
6095                .configure(configured_capabilities, reserved_capabilities);
6096            self.capability_evaluator.wake_deadline_loop();
6097            self.refresh_capability_requirements();
6098        }
6099        let mut result = result;
6100        result.restart_required = restart_required;
6101        result.capability_warnings = preview_capability_warnings;
6102        let response = ClientControlResponse::SupervisorRescan { result };
6103        Ok(vec![control_response_body_frame(
6104            &frame,
6105            &response,
6106            "ClientControlResponse::SupervisorRescan",
6107        )?])
6108    }
6109
6110    async fn handle_supervisor_release_reserved(
6111        &self,
6112        frame: Frame,
6113        module_id: String,
6114    ) -> Result<Vec<Frame>, RouterError> {
6115        let Some(context) = self.rescan.clone() else {
6116            return Ok(vec![control_error_frame(
6117                &frame,
6118                "release_unavailable",
6119                "reserved-id release requires a daemon started with a reloadable config path",
6120            )?]);
6121        };
6122        let operation_lock = self.supervisor.operation_lock();
6123        let _operation_guard = operation_lock.lock().await;
6124        let loaded = match crate::daemon_config::load(&context.config_path) {
6125            Ok(Some(config)) => config,
6126            Ok(None) => {
6127                return Ok(vec![control_error_frame(
6128                    &frame,
6129                    "invalid_daemon_config",
6130                    format!(
6131                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6132                        context.config_path.display()
6133                    ),
6134                )?])
6135            }
6136            Err(err) => {
6137                return Ok(vec![control_error_frame(
6138                    &frame,
6139                    "invalid_daemon_config",
6140                    format!("unable to verify reserved-id release against daemon config: {err}"),
6141                )?])
6142            }
6143        };
6144        if loaded
6145            .modules
6146            .iter()
6147            .any(|configured| configured.module_id == module_id)
6148        {
6149            return Ok(vec![control_error_frame(
6150                &frame,
6151                "reserved_module_configured",
6152                format!(
6153                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6154                ),
6155            )?]);
6156        }
6157        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6158            return Ok(vec![control_error_frame(
6159                &frame,
6160                "reserved_gate_not_retained",
6161                format!(
6162                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6163                ),
6164            )?]);
6165        }
6166
6167        let response = ClientControlResponse::SupervisorAck {
6168            module_id,
6169            applied: true,
6170        };
6171        Ok(vec![control_response_body_frame(
6172            &frame,
6173            &response,
6174            "ClientControlResponse::SupervisorAck",
6175        )?])
6176    }
6177
6178    /// Compare the configured module set with the supervised one, changing
6179    /// nothing. The staged spec each supervised module holds is what the
6180    /// configured entry is compared with.
6181    fn plan_rescan(
6182        &self,
6183        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6184    ) -> Result<RescanPlan, String> {
6185        let mut current = BTreeMap::new();
6186        for module in self.supervisor.list() {
6187            let (spec, health) = module.configuration().map_err(|err| {
6188                format!(
6189                    "failed to read configuration for module_id '{}': {err}",
6190                    module.module_id()
6191                )
6192            })?;
6193            let enabled = module
6194                .status()
6195                .map_err(|err| {
6196                    format!(
6197                        "failed to read status for module_id '{}': {err}",
6198                        module.module_id()
6199                    )
6200                })?
6201                .enabled;
6202            current.insert(
6203                module.module_id().to_string(),
6204                (module, spec, health, enabled),
6205            );
6206        }
6207        let configured = configured_modules
6208            .into_iter()
6209            .map(|module| (module.module_id.clone(), module))
6210            .collect::<BTreeMap<_, _>>();
6211
6212        let added = configured
6213            .keys()
6214            .filter(|module_id| !current.contains_key(*module_id))
6215            .cloned()
6216            .collect::<Vec<_>>();
6217        let removed = current
6218            .keys()
6219            .filter(|module_id| !configured.contains_key(*module_id))
6220            .cloned()
6221            .collect::<Vec<_>>();
6222        let mut changed_pending_reload = Vec::new();
6223        let mut configuration_changes = BTreeSet::new();
6224        let mut enabled_changes = BTreeSet::new();
6225        let mut unchanged = 0_u32;
6226
6227        for (module_id, configured_module) in &configured {
6228            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6229            else {
6230                continue;
6231            };
6232            // Compare the whole launch spec so a future launch field cannot
6233            // accidentally become a live-only policy change. Health is stored
6234            // separately and applies live without replacing the process.
6235            let launch_changed = *current_spec != configured_module.module_spec();
6236            let configuration_changed =
6237                launch_changed || *current_health != configured_module.health;
6238            let enabled_changed = *current_enabled != configured_module.enabled;
6239            if configuration_changed {
6240                configuration_changes.insert(module_id.clone());
6241            }
6242            if launch_changed {
6243                changed_pending_reload.push(module_id.clone());
6244            }
6245            if enabled_changed {
6246                enabled_changes.insert(module_id.clone());
6247            }
6248            if !configuration_changed && !enabled_changed {
6249                unchanged = unchanged.saturating_add(1);
6250            }
6251        }
6252        Ok(RescanPlan {
6253            current,
6254            configured,
6255            added,
6256            removed,
6257            changed_pending_reload,
6258            configuration_changes,
6259            enabled_changes,
6260            unchanged,
6261        })
6262    }
6263
6264    /// Reconcile the running module set against the configured one.
6265    ///
6266    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6267    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6268    /// deliberately shares this function with the executing path rather than
6269    /// computing the same diff somewhere else -- two implementations of one
6270    /// decision agree until they do not, and the whole value of a preview is that
6271    /// it describes the operation that will actually run.
6272    async fn reconcile_supervised_modules(
6273        &self,
6274        supervisor: &Supervisor,
6275        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6276        preview: bool,
6277        requester: Principal,
6278    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6279        let plan = self.plan_rescan(configured_modules)?;
6280
6281        // Everything above this point is pure computation over two snapshots.
6282        // Everything below MUTATES. The preview returns here so the boundary is a
6283        // single early return rather than a condition repeated at each mutation
6284        // site, where one missed guard would apply part of a change the caller was
6285        // told would not happen.
6286        if preview {
6287            return Ok((plan.preview_result(), Vec::new()));
6288        }
6289        let changed = plan
6290            .configuration_changes
6291            .union(&plan.enabled_changes)
6292            .cloned()
6293            .collect::<Vec<_>>();
6294        let RescanPlan {
6295            current,
6296            configured,
6297            added,
6298            removed,
6299            changed_pending_reload,
6300            configuration_changes,
6301            enabled_changes,
6302            unchanged,
6303        } = plan;
6304        // The staged generation moves before the first change rather than
6305        // after the last, so a rescan that fails partway still invalidates
6306        // every earlier `supervisor.rescan_checked` preview: part of its diff
6307        // may have run.
6308        if !(added.is_empty()
6309            && removed.is_empty()
6310            && configuration_changes.is_empty()
6311            && enabled_changes.is_empty())
6312        {
6313            if let Some(context) = &self.rescan {
6314                context
6315                    .generation
6316                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6317            }
6318        }
6319
6320        for module_id in &removed {
6321            let module = &current
6322                .get(module_id)
6323                .expect("removed module came from current supervisor state")
6324                .0;
6325            module
6326                .retire_requested(Some(requester.clone()))
6327                .await
6328                .map_err(|err| {
6329                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6330                })?;
6331            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6332            //
6333            // `handle_route_open` resolves an absent module in three steps:
6334            // registry, then supervisor status, then tombstone. Retiring first
6335            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6336            // went with the teardown above, the supervisor entry went with
6337            // `retire`, and the tombstone does not exist yet -- so a route.open
6338            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6339            // it") for a module that was deliberately removed and whose caller
6340            // should get `module_removed` (TERMINAL, carrying a removal age).
6341            //
6342            // Writing the tombstone first closes it: during the window the
6343            // supervisor entry still answers, so the caller gets
6344            // `target_unavailable` -- retryable, and TRUE, because the module
6345            // is mid-teardown. After both statements it is `module_removed`.
6346            // No instant remains where a removed module reads as one that
6347            // never existed.
6348            //
6349            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6350            // the absence of a test beside a fix invites deletion: these are
6351            // two sync statements with no await between them, so reaching the
6352            // window needs a second worker thread to land exactly between them
6353            // and there is no hook to force it. MEASURED: the 25 daemon_config
6354            // tests pass identically with the old order and the new one, so
6355            // the existing suite cannot see this and a green run is not
6356            // evidence either way. What the suite does hold is the
6357            // post-condition -- a removed module answers `module_removed` --
6358            // which this preserves.
6359            //
6360            // Found by an Athena panel reading the shipped tree against a
6361            // design note (2026-09-19), as the one concrete instance of that
6362            // note's class that survived contact with source. Direction is
6363            // benign: retryable where terminal was intended, never the reverse.
6364            self.supervisor.record_rescan_removal(module_id);
6365            self.supervisor.retire(module_id);
6366            self.route_outages.forget(module_id);
6367        }
6368
6369        for module_id in configured.keys() {
6370            let Some((module, _, _, _)) = current.get(module_id) else {
6371                continue;
6372            };
6373            let configured_module = configured
6374                .get(module_id)
6375                .expect("configured module id came from configured map");
6376            if configuration_changes.contains(module_id) {
6377                module
6378                    .update_configuration(
6379                        configured_module.module_spec(),
6380                        configured_module.health.clone(),
6381                        configured_module.drain_timeout_ms,
6382                    )
6383                    .await
6384                    .map_err(|err| {
6385                        format!(
6386                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6387                        )
6388                    })?;
6389            }
6390            if enabled_changes.contains(module_id) {
6391                // A rescan that starts or stops a module applies an operator's
6392                // edit to the config, so the resulting outage was asked for.
6393                self.route_outages.mark_operator_action(module_id);
6394                module
6395                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6396                    .await
6397                    .map_err(|err| {
6398                        self.route_outages.operator_action_ended_unrefused(module_id);
6399                        format!(
6400                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6401                            configured_module.enabled
6402                        )
6403                    })?;
6404            }
6405        }
6406
6407        for module_id in &added {
6408            let configured_module = configured
6409                .get(module_id)
6410                .expect("added module id came from configured map");
6411            supervisor
6412                .supervise_configured_requested(
6413                    configured_module.module_spec(),
6414                    configured_module.enabled,
6415                    configured_module.health.clone(),
6416                    configured_module.drain_timeout_ms,
6417                    configured_module.restart,
6418                    configured_module.register_timeout,
6419                    Some(requester.clone()),
6420                )
6421                .map_err(|err| {
6422                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6423                })?;
6424        }
6425
6426        Ok((
6427            SupervisorRescanResult {
6428                added,
6429                removed,
6430                changed_pending_reload,
6431                enabled_changes: enabled_changes.iter().cloned().collect(),
6432                unchanged,
6433                preview: false,
6434                // Filled by the caller, which is the only layer that can see the
6435                // previous config to diff against.
6436                restart_required: Vec::new(),
6437                capability_warnings: Vec::new(),
6438                module_actions: Vec::new(),
6439                earlier_pending: Vec::new(),
6440                expect: None,
6441            },
6442            changed,
6443        ))
6444    }
6445
6446    async fn handle_supervisor_set_enabled(
6447        &self,
6448        frame: Frame,
6449        module_id: String,
6450        enabled: bool,
6451        requester: Principal,
6452    ) -> Result<Vec<Frame>, RouterError> {
6453        let operation_lock = self.supervisor.operation_lock();
6454        let _operation_guard = operation_lock.lock().await;
6455        let Some(module) = self.supervisor.get(&module_id) else {
6456            return Ok(vec![control_error_frame(
6457                &frame,
6458                "unknown_module",
6459                format!("module_id '{module_id}' is not supervised"),
6460            )?]);
6461        };
6462
6463        // Enabling counts as well as disabling: a module an operator starts
6464        // is refused until it registers, and that wait was asked for.
6465        self.route_outages.mark_operator_action(&module_id);
6466        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6467            Ok(applied) => applied,
6468            Err(err) => {
6469                self.route_outages
6470                    .operator_action_ended_unrefused(&module_id);
6471                return Ok(vec![control_error_frame(
6472                    &frame,
6473                    "target_unavailable",
6474                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6475                )?]);
6476            }
6477        };
6478        if !applied {
6479            // Already in the requested state: nothing was made unavailable,
6480            // so the mark must not outlive this request.
6481            self.route_outages
6482                .operator_action_ended_unrefused(&module_id);
6483        }
6484
6485        self.capability_evaluator.wake_deadline_loop();
6486        self.refresh_capability_requirements();
6487        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6488        Ok(vec![control_response_body_frame(
6489            &frame,
6490            &response,
6491            "ClientControlResponse::SupervisorAck",
6492        )?])
6493    }
6494
6495    async fn handle_supervisor_health_probe(
6496        &self,
6497        frame: Frame,
6498        module_id: String,
6499    ) -> Result<Vec<Frame>, RouterError> {
6500        self.refresh_capability_requirements();
6501        let Some(registration) = self
6502            .registry
6503            .get_module(&module_id)
6504            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6505        else {
6506            return Ok(vec![control_error_frame(
6507                &frame,
6508                "unknown_module",
6509                format!("module_id '{module_id}' is not registered"),
6510            )?]);
6511        };
6512
6513        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6514        // named for it. Making `module_registration_grants_op` return false
6515        // unconditionally reddens five tests, and every one is named for something
6516        // else -- capability relay, probe/bind demultiplexing, supervision-only
6517        // probing. They exercise a successful advertisement check on the way to their
6518        // own subject.
6519        //
6520        // Real protection, fragile in a specific way: narrowing any of those tests to
6521        // focus on its stated subject would silently remove coverage nobody knows
6522        // they are carrying. Recorded here rather than as a sixth test, because the
6523        // useful fact is WHICH tests hold the guard up -- a new test would add
6524        // coverage without telling the next person what the existing ones quietly do.
6525        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6526        {
6527            return Ok(vec![control_error_frame(
6528                &frame,
6529                "health_not_advertised",
6530                format!("module_id '{module_id}' did not advertise health.check"),
6531            )?]);
6532        }
6533
6534        let deadline = Instant::now() + self.health_probe_timeout;
6535        let pending = match self.forwarding.begin_module_control_rpc_for(
6536            &module_id,
6537            MODULE_CONTROL_OP_HEALTH_CHECK,
6538            deadline,
6539        ) {
6540            Ok(pending) => pending,
6541            Err(err) => {
6542                return Ok(vec![control_error_frame(
6543                    &frame,
6544                    forwarding_error_code(&err),
6545                    err.to_string(),
6546                )?])
6547            }
6548        };
6549
6550        let PendingModuleControlRpc {
6551            endpoint,
6552            module_sink,
6553            negotiated_ver,
6554            corr: probe_corr,
6555            receiver,
6556        } = pending;
6557        let mut guard =
6558            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6559        let probe_body =
6560            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6561                RouterError::backend(
6562                    0,
6563                    frame.header.corr,
6564                    format!("failed to encode health.check request: {err}"),
6565                )
6566            })?;
6567        let probe_frame = Frame::build_with_version(
6568            negotiated_ver,
6569            FrameType::Request,
6570            control_flags(),
6571            0,
6572            0,
6573            probe_corr,
6574            probe_body,
6575        )
6576        .map_err(RouterError::FrameBuild)?;
6577
6578        // Queue admission and the reply share one deadline, so a module that
6579        // stops reading cannot park the requesting connection's reader forever.
6580        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6581            Ok(Ok(())) => {}
6582            Ok(Err(err)) => {
6583                return Ok(vec![control_error_frame(
6584                    &frame,
6585                    "target_unavailable",
6586                    err.to_string(),
6587                )?]);
6588            }
6589            Err(_) => {
6590                return Ok(vec![control_error_frame(
6591                    &frame,
6592                    "module_timeout",
6593                    format!(
6594                        "module_id '{module_id}' did not answer health.check within {:?}",
6595                        self.health_probe_timeout
6596                    ),
6597                )?]);
6598            }
6599        }
6600
6601        match timeout_at(deadline, receiver).await {
6602            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6603                guard.disarm();
6604                let Some(report) = response.health_report() else {
6605                    return Ok(vec![control_error_frame(
6606                        &frame,
6607                        "invalid_control_body",
6608                        "health.check RPC returned a non-health response",
6609                    )?]);
6610                };
6611                // Metrics go out whole here. The supervisor's cached snapshot
6612                // caps this blob (see truncate_health_metrics), and this path
6613                // exists precisely to answer without that cap -- so applying it
6614                // here would leave no way to see what the cached view drops.
6615                let HealthReport {
6616                    status,
6617                    detail,
6618                    metrics,
6619                } = report;
6620                let capability_detail = self
6621                    .capability_evaluator
6622                    .required_problem_detail(&module_id);
6623                let response = ClientControlResponse::SupervisorHealthProbe {
6624                    module_id,
6625                    status,
6626                    detail: append_capability_problem_detail(detail, capability_detail),
6627                    metrics,
6628                };
6629                Ok(vec![control_response_body_frame(
6630                    &frame,
6631                    &response,
6632                    "ClientControlResponse::SupervisorHealthProbe",
6633                )?])
6634            }
6635            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6636                guard.disarm();
6637                Ok(vec![control_error_body_frame(&frame, body)?])
6638            }
6639            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6640                guard.disarm();
6641                Ok(vec![control_error_frame(
6642                    &frame,
6643                    "target_unavailable",
6644                    message,
6645                )?])
6646            }
6647            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6648                guard.disarm();
6649                Ok(vec![control_error_frame(
6650                    &frame,
6651                    "invalid_control_body",
6652                    message,
6653                )?])
6654            }
6655            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6656                guard.disarm();
6657                Ok(vec![control_error_frame(
6658                    &frame,
6659                    "invalid_control_body",
6660                    format!("expected module-control op '{expected}', got '{actual}'"),
6661                )?])
6662            }
6663            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6664                guard.disarm();
6665                Ok(vec![control_error_frame(
6666                    &frame,
6667                    "module_timeout",
6668                    format!(
6669                        "module_id '{module_id}' answered health.check after {:?}",
6670                        self.health_probe_timeout
6671                    ),
6672                )?])
6673            }
6674            Ok(Err(_)) => Ok(vec![control_error_frame(
6675                &frame,
6676                "target_unavailable",
6677                "health.check waiter was canceled before the module responded",
6678            )?]),
6679            Err(_) => Ok(vec![control_error_frame(
6680                &frame,
6681                "module_timeout",
6682                format!(
6683                    "module_id '{module_id}' did not answer health.check within {:?}",
6684                    self.health_probe_timeout
6685                ),
6686            )?]),
6687        }
6688    }
6689
6690    fn supervisor_status(
6691        &self,
6692        module_id: &str,
6693        corr: u64,
6694    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6695        self.supervisor
6696            .get(module_id)
6697            .map(|module| {
6698                let warming = module.is_warming_for_control("status").map_err(|err| {
6699                    RouterError::backend(
6700                        0,
6701                        corr,
6702                        format!(
6703                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6704                        ),
6705                    )
6706                })?;
6707                module.status_for_control("status").map_err(|err| {
6708                    RouterError::backend(
6709                        0,
6710                        corr,
6711                        format!(
6712                            "failed to read supervisor status for module_id '{module_id}': {err}"
6713                        ),
6714                    )
6715                }).map(|status| (status, warming))
6716            })
6717            .transpose()
6718    }
6719
6720    fn guard_module_control_op(
6721        &self,
6722        frame: &Frame,
6723        module_id: &str,
6724        op: &str,
6725    ) -> Result<Option<Frame>, RouterError> {
6726        if self.module_grants_op(module_id, op, frame.header.corr)? {
6727            return Ok(None);
6728        }
6729
6730        Ok(Some(control_error_frame(
6731            frame,
6732            "op_not_allowed",
6733            format!("module_id '{module_id}' did not grant control op '{op}'"),
6734        )?))
6735    }
6736
6737    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6738        let Some(registration) = self
6739            .registry
6740            .get_module(module_id)
6741            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6742        else {
6743            return Ok(false);
6744        };
6745        Ok(module_registration_grants_op(&registration.control_ops, op))
6746    }
6747
6748    fn handle_status_update(
6749        &self,
6750        endpoint: ModuleEndpointId,
6751        frame: Frame,
6752    ) -> Result<Vec<Frame>, RouterError> {
6753        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6754            Ok(update) => update,
6755            Err(err) => {
6756                // Forward-compat: a newer module may push a channel-0 op this subc
6757                // version doesn't know. The control contract says unknown push ops
6758                // are IGNORED, never answered with an error. Only a malformed body
6759                // for an op we DO know is a real error worth surfacing.
6760                if is_known_module_push_op(&frame.body) {
6761                    return Ok(vec![control_error_frame(
6762                        &frame,
6763                        "invalid_control_body",
6764                        format!("malformed module control push body: {err}"),
6765                    )?]);
6766                }
6767                return Ok(Vec::new());
6768            }
6769        };
6770
6771        match update {
6772            ModuleControlPush::RouteStatus {
6773                route_channel,
6774                route_epoch,
6775                status,
6776            } => {
6777                self.forwarding
6778                    .cache_status(endpoint, route_channel, route_epoch, status)
6779                    .map_err(RouterError::Forwarding)?;
6780            }
6781        }
6782        Ok(Vec::new())
6783    }
6784
6785    fn handle_route_poll(
6786        &self,
6787        ctx: &RouteCtx,
6788        frame: Frame,
6789        route_channel: u16,
6790        route_epoch: u32,
6791        kind: PollKind,
6792    ) -> Result<Vec<Frame>, RouterError> {
6793        let snapshot = self
6794            .forwarding
6795            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6796            .map_err(RouterError::Forwarding)?;
6797        let response = match (kind, snapshot) {
6798            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6799                ClientControlResponse::RoutePoll {
6800                    route_channel,
6801                    route_epoch,
6802                    status,
6803                    live: None,
6804                }
6805            }
6806            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6807                route_channel,
6808                route_epoch,
6809                status: None,
6810                live: None,
6811            },
6812            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6813                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6814                // is what makes reporting `true` correct rather than a
6815                // confident guess. `process_live` returns None only when the
6816                // module id has no supervisor snapshot at all -- an
6817                // externally-started module the daemon did not spawn -- and
6818                // for those the supervisor has no opinion to offer, ever. It
6819                // is never None for a supervised module in an unknown state:
6820                // a supervised module always has a snapshot, and the answer
6821                // comes from `state == Running && process_alive`.
6822                //
6823                // The route is Bound, so the module completed a HELLO on a
6824                // live connection; "the process this route points at is
6825                // running" is therefore attested by the binding rather than
6826                // assumed. Reporting `false` for an unsupervised module would
6827                // be the actual lie -- it would tell a client its healthy
6828                // route is dead because the daemon does not manage the
6829                // process.
6830                //
6831                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6832                // module whose liveness is genuinely unknown, e.g. a snapshot
6833                // that has not been populated yet -- THIS DEFAULT BECOMES
6834                // WRONG and must split: unsupervised stays true, unknown
6835                // becomes null so the client can tell the two apart. The
6836                // response field is already `Option<bool>`, so the wire can
6837                // carry that distinction today.
6838                let live = self
6839                    .process_liveness
6840                    .as_ref()
6841                    .and_then(|source| source.process_live(&module_id))
6842                    .unwrap_or(true);
6843                ClientControlResponse::RoutePoll {
6844                    route_channel,
6845                    route_epoch,
6846                    status: None,
6847                    live: Some(live),
6848                }
6849            }
6850            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6851                route_channel,
6852                route_epoch,
6853                status: None,
6854                live: Some(false),
6855            },
6856        };
6857
6858        Ok(vec![control_response_body_frame(
6859            &frame,
6860            &response,
6861            "ClientControlResponse::RoutePoll",
6862        )?])
6863    }
6864
6865    pub(crate) fn observe_module_control_completion(
6866        &self,
6867        completion: ModuleControlRpcCompletion,
6868    ) -> bool {
6869        match completion {
6870            ModuleControlRpcCompletion::Unknown => false,
6871            ModuleControlRpcCompletion::Settled => true,
6872            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6873                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6874                info!(
6875                    module_id = %module_id,
6876                    latency_ms,
6877                    "late health.check answer proves the module is alive"
6878                );
6879                match self
6880                    .supervisor
6881                    .record_late_health_answer(&module_id, latency_ms)
6882                {
6883                    Ok(true) => {}
6884                    Ok(false) => debug!(
6885                        module_id = %module_id,
6886                        latency_ms,
6887                        "late health.check answer has no active supervisor snapshot"
6888                    ),
6889                    Err(err) => warn!(
6890                        module_id = %module_id,
6891                        latency_ms,
6892                        error = %err,
6893                        "failed to record late health.check answer"
6894                    ),
6895                }
6896                true
6897            }
6898        }
6899    }
6900
6901    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6902    /// the module connection whose frame is being handled, or to the client that
6903    /// relay was opened for.
6904    ///
6905    /// This runs on the MODULE connection's frame handler, where returning `Err`
6906    /// ends that connection -- and a module connection carries every client's
6907    /// routes to that module, so ending it costs the whole fleet its tools.
6908    /// `ConnectionClosing` carries the id of the connection that is closing, and
6909    /// when that id is a CLIENT's, the condition is entirely about that one
6910    /// client's route.open. A client-scoped condition has no authority over a
6911    /// shared module connection, so it is logged and the single relay is dropped:
6912    /// the client is going away, and `complete_pending_relay` already removed the
6913    /// relay before failing, so there is nothing left to settle. Anything that
6914    /// relay still reserved is released by that client's own connection teardown,
6915    /// which is already under way -- that is what "closing" means.
6916    ///
6917    /// Every other failure is a statement about THIS connection and stays fatal:
6918    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6919    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6920    /// frames correctly.
6921    fn refuse_to_end_module_connection_for_a_client(
6922        &self,
6923        module_connection_id: ConnectionId,
6924        corr: u64,
6925        err: ForwardingError,
6926    ) -> Result<(), RouterError> {
6927        if let ForwardingError::ConnectionClosing { connection_id } = err {
6928            if connection_id != module_connection_id {
6929                warn!(
6930                    module_connection_id = module_connection_id.get(),
6931                    client_connection_id = connection_id.get(),
6932                    corr,
6933                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6934                );
6935                return Ok(());
6936            }
6937        }
6938        Err(RouterError::Forwarding(err))
6939    }
6940
6941    fn handle_module_relay_response(
6942        &self,
6943        connection_id: ConnectionId,
6944        frame: Frame,
6945    ) -> Result<Vec<Frame>, RouterError> {
6946        let mut secondary_error = None;
6947        let outcome = match frame.header.ty {
6948            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6949                Ok(probe) if probe.op == "route.bind" => {
6950                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6951                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6952                            RouteBindRelayOutcome::Accepted
6953                        }
6954                        Ok(other) => {
6955                            let message =
6956                                format!("route.bind response carried unexpected body: {other:?}");
6957                            secondary_error = Some(control_error_frame(
6958                                &frame,
6959                                "invalid_control_body",
6960                                message.clone(),
6961                            )?);
6962                            RouteBindRelayOutcome::ModuleGone(message)
6963                        }
6964                        Err(err) => {
6965                            let message = format!("malformed route.bind response body: {err}");
6966                            secondary_error = Some(control_error_frame(
6967                                &frame,
6968                                "invalid_control_body",
6969                                message.clone(),
6970                            )?);
6971                            RouteBindRelayOutcome::ModuleGone(message)
6972                        }
6973                    }
6974                }
6975                Ok(probe) => {
6976                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6977                    {
6978                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6979                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6980                            "malformed {} response body: {err}",
6981                            probe.op
6982                        )),
6983                    };
6984                    let completion = self
6985                        .forwarding
6986                        .complete_module_control_rpc(
6987                            connection_id,
6988                            frame.header.corr,
6989                            Some(&probe.op),
6990                            outcome,
6991                        )
6992                        .map_err(RouterError::Forwarding)?;
6993                    if !self.observe_module_control_completion(completion) {
6994                        debug!(
6995                            connection_id = connection_id.get(),
6996                            corr = frame.header.corr,
6997                            op = %probe.op,
6998                            "dropping late or unknown module-control RPC response"
6999                        );
7000                    }
7001                    return Ok(Vec::new());
7002                }
7003                Err(err) => {
7004                    if let Some(expected_op) = self
7005                        .forwarding
7006                        .pending_module_control_op(connection_id, frame.header.corr)
7007                        .map_err(RouterError::Forwarding)?
7008                    {
7009                        let completion = self
7010                            .forwarding
7011                            .complete_module_control_rpc(
7012                                connection_id,
7013                                frame.header.corr,
7014                                None,
7015                                ModuleControlRpcOutcome::MalformedResponse(format!(
7016                                    "malformed {expected_op} response body: {err}"
7017                                )),
7018                            )
7019                            .map_err(RouterError::Forwarding)?;
7020                        if !self.observe_module_control_completion(completion) {
7021                            debug!(
7022                                connection_id = connection_id.get(),
7023                                corr = frame.header.corr,
7024                                "dropping late malformed module-control RPC response"
7025                            );
7026                        }
7027                        return Ok(Vec::new());
7028                    }
7029                    let message = format!("malformed route.bind response body: {err}");
7030                    secondary_error = Some(control_error_frame(
7031                        &frame,
7032                        "invalid_control_body",
7033                        message.clone(),
7034                    )?);
7035                    RouteBindRelayOutcome::ModuleGone(message)
7036                }
7037            },
7038            FrameType::Error => {
7039                if self
7040                    .forwarding
7041                    .pending_module_control_op(connection_id, frame.header.corr)
7042                    .map_err(RouterError::Forwarding)?
7043                    .is_some()
7044                {
7045                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7046                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7047                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7048                            "malformed module-control ERROR body: {err}"
7049                        )),
7050                    };
7051                    let completion = self
7052                        .forwarding
7053                        .complete_module_control_rpc(
7054                            connection_id,
7055                            frame.header.corr,
7056                            None,
7057                            outcome,
7058                        )
7059                        .map_err(RouterError::Forwarding)?;
7060                    if !self.observe_module_control_completion(completion) {
7061                        debug!(
7062                            connection_id = connection_id.get(),
7063                            corr = frame.header.corr,
7064                            "dropping late or unknown module-control RPC error"
7065                        );
7066                    }
7067                    return Ok(Vec::new());
7068                }
7069                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7070                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7071                    Err(err) => {
7072                        let message = format!("malformed route.bind ERROR body: {err}");
7073                        secondary_error = Some(control_error_frame(
7074                            &frame,
7075                            "invalid_control_body",
7076                            message.clone(),
7077                        )?);
7078                        RouteBindRelayOutcome::ModuleGone(message)
7079                    }
7080                }
7081            }
7082            ty => {
7083                return Ok(vec![control_error_frame(
7084                    &frame,
7085                    "unsupported_control_frame",
7086                    format!("unsupported module channel-0 frame {ty:?}"),
7087                )?])
7088            }
7089        };
7090
7091        let settled =
7092            self.forwarding
7093                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7094        let completion = match settled {
7095            Ok(completion) => completion,
7096            Err(err) => {
7097                self.refuse_to_end_module_connection_for_a_client(
7098                    connection_id,
7099                    frame.header.corr,
7100                    err,
7101                )?;
7102                return Ok(secondary_error.into_iter().collect());
7103            }
7104        };
7105        if let Some(target) = completion.abandoned.as_ref() {
7106            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7107        }
7108        if !completion.settled {
7109            debug!(
7110                connection_id = connection_id.get(),
7111                corr = frame.header.corr,
7112                frame_type = ?frame.header.ty,
7113                "dropping late or unknown route.bind relay response"
7114            );
7115        }
7116        Ok(secondary_error.into_iter().collect())
7117    }
7118
7119    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7120        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7121        // GOODBYE ends the connection's logical session even when its socket
7122        // stays open. Use disconnect teardown so verdicts, client notices and
7123        // scope authority are released at the same lifecycle boundary.
7124        self.cleanup_connection_with_end_reason(
7125            connection_id,
7126            RegistrationEndReason::ExplicitGoodbye,
7127        )
7128        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7129        Ok(Vec::new())
7130    }
7131}
7132
7133impl Default for ControlHandler {
7134    fn default() -> Self {
7135        Self::new(Arc::new(Registry::default()))
7136    }
7137}
7138
7139impl crate::supervise::SwapPromotionObserver for ControlHandler {
7140    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7141        self.apply_registration_capabilities(registration);
7142    }
7143}
7144
7145fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7146    CapabilityRequirementStatus {
7147        consumer: status.consumer,
7148        capability: status.capability,
7149        need: match status.need {
7150            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7151            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7152        },
7153        verdict: status.verdict.as_str().to_string(),
7154        episode_seq: status.episode_seq,
7155        config_satisfiable: status.config_satisfiable,
7156        runtime_available: status.runtime_available,
7157        detail: status.detail,
7158    }
7159}
7160
7161fn append_capability_problem_detail(
7162    detail: Option<String>,
7163    capability_detail: Option<String>,
7164) -> Option<String> {
7165    match (detail, capability_detail) {
7166        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7167        (Some(detail), None) => Some(detail),
7168        (None, Some(capability_detail)) => Some(capability_detail),
7169        (None, None) => None,
7170    }
7171}
7172
7173fn subc_ops() -> Vec<String> {
7174    SUBC_CONTROL_OPS
7175        .iter()
7176        .map(|op| (*op).to_string())
7177        .collect()
7178}
7179
7180fn module_subc_ops() -> Vec<String> {
7181    SUBC_CONTROL_OPS
7182        .iter()
7183        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7184        .map(|op| (*op).to_string())
7185        .collect()
7186}
7187
7188#[cfg(test)]
7189fn module_baseline_control_ops() -> Vec<String> {
7190    MODULE_BASELINE_CONTROL_OPS
7191        .iter()
7192        .map(|op| (*op).to_string())
7193        .collect()
7194}
7195
7196fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7197    let mut seen = HashSet::new();
7198    let mut effective = Vec::new();
7199    for op in MODULE_BASELINE_CONTROL_OPS {
7200        if seen.insert((*op).to_string()) {
7201            effective.push((*op).to_string());
7202        }
7203    }
7204    for op in declared.unwrap_or_default() {
7205        if seen.insert(op.clone()) {
7206            effective.push(op);
7207        }
7208    }
7209    effective
7210}
7211
7212fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7213    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7214}
7215
7216fn target_module_id(target: &RouteTarget) -> &str {
7217    match target {
7218        RouteTarget::ToolProvider { module_id }
7219        | RouteTarget::ManagementSurface { module_id }
7220        | RouteTarget::InternalService { module_id, .. } => module_id,
7221    }
7222}
7223
7224fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7225    roles.iter().any(|role| match (target, role) {
7226        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7227        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7228        (
7229            RouteTarget::InternalService { service_id, .. },
7230            ProviderRole::InternalService {
7231                service_id: provided,
7232                ..
7233            },
7234        ) => service_id == provided,
7235        _ => false,
7236    })
7237}
7238
7239fn is_routable_role(role: &ProviderRole) -> bool {
7240    matches!(
7241        role,
7242        ProviderRole::ToolProvider { .. }
7243            | ProviderRole::ManagementSurface { .. }
7244            | ProviderRole::InternalService { .. }
7245    )
7246}
7247
7248#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7249enum ControlRequestBodyError {
7250    UnknownOp,
7251    InvalidBody,
7252}
7253
7254#[derive(Debug, Deserialize)]
7255struct ControlOpProbe {
7256    op: String,
7257}
7258
7259/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7260/// this set is treated as a forward-compat unknown and ignored rather than errored.
7261const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7262
7263fn is_known_module_push_op(body: &[u8]) -> bool {
7264    serde_json::from_slice::<ControlOpProbe>(body)
7265        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7266        .unwrap_or(false)
7267}
7268
7269fn is_known_module_request_op(body: &[u8]) -> bool {
7270    serde_json::from_slice::<ControlOpProbe>(body)
7271        .map(|probe| is_module_to_subc_op(&probe.op))
7272        .unwrap_or(false)
7273}
7274
7275fn is_module_to_subc_op(op: &str) -> bool {
7276    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7277}
7278
7279fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7280    debug!(
7281        op = %op,
7282        connection_id = connection_id.get(),
7283        corr,
7284        "control dispatch"
7285    );
7286}
7287
7288fn log_slow_control_dispatch(
7289    dispatch_started_at: Option<StdInstant>,
7290    op: &'static str,
7291    connection_id: ConnectionId,
7292    corr: u64,
7293    route_open_timings: Option<&RouteOpenTimings>,
7294) {
7295    let Some(dispatch_started_at) = dispatch_started_at else {
7296        return;
7297    };
7298    // Use the same monotonic clock as the phase timers and relay deadline.
7299    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7300    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7301        if let Some(timings) = route_open_timings {
7302            warn!(
7303                op = %op,
7304                connection_id = connection_id.get(),
7305                corr,
7306                elapsed_ms = elapsed.as_millis() as u64,
7307                module_id = ?timings.module_id,
7308                principal = %timings.principal,
7309                admission_ms = timings.admission.as_millis() as u64,
7310                project_root_ms = timings.project_root.as_millis() as u64,
7311                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7312                bind_send_ms = timings.bind_send.as_millis() as u64,
7313                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7314                "slow control dispatch"
7315            );
7316            return;
7317        }
7318        warn!(
7319            op = %op,
7320            connection_id = connection_id.get(),
7321            corr,
7322            elapsed_ms = elapsed.as_millis() as u64,
7323            "slow control dispatch"
7324        );
7325    }
7326}
7327
7328/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7329#[cfg(test)]
7330type TestCallerLookup =
7331    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7332
7333/// Every new client operation needs an explicit privilege decision here.
7334fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7335    match request {
7336        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7337            Some(PrivilegedOperation::Restart { module_id })
7338        }
7339        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7340            Some(PrivilegedOperation::Swap { module_id })
7341        }
7342        ClientControlRequest::SupervisorReload { module_id } => {
7343            Some(PrivilegedOperation::Reload { module_id })
7344        }
7345        ClientControlRequest::SupervisorRescan { preview } => {
7346            if *preview {
7347                None
7348            } else {
7349                Some(PrivilegedOperation::Rescan)
7350            }
7351        }
7352        // Only the well-formed apply asks for approval. A preview changes
7353        // nothing, and any other combination is refused before it touches
7354        // anything, so prompting the operator for it would ask about a request
7355        // that cannot run.
7356        ClientControlRequest::SupervisorRescanChecked {
7357            preview,
7358            candidate,
7359            expect,
7360        } => (!*preview && candidate.is_none() && expect.is_some())
7361            .then_some(PrivilegedOperation::Rescan),
7362        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7363            Some(PrivilegedOperation::SetEnabled {
7364                module_id,
7365                enabled: *enabled,
7366            })
7367        }
7368        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7369            Some(PrivilegedOperation::ReleaseReserved { module_id })
7370        }
7371        ClientControlRequest::ServerDescribe {}
7372        | ClientControlRequest::CatalogList { .. }
7373        | ClientControlRequest::RouteOpen { .. }
7374        | ClientControlRequest::RoutePoll { .. }
7375        | ClientControlRequest::SupervisorList {}
7376        | ClientControlRequest::SupervisorHealth {}
7377        | ClientControlRequest::SupervisorHealthProbe { .. }
7378        | ClientControlRequest::SupervisorRoutes { .. }
7379        | ClientControlRequest::SupervisorProvenance { .. }
7380        | ClientControlRequest::SupervisorStderrTail { .. }
7381        | ClientControlRequest::SupervisorTerminals { .. }
7382        | ClientControlRequest::SupervisorSpawnSnapshot {}
7383        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7384    }
7385}
7386
7387fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7388    match request {
7389        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7390        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7391        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7392        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7393        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7394        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7395        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7396        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7397        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7398        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7399        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7400        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7401        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7402        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7403        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7404        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7405        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7406        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7407        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7408        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7409    }
7410}
7411
7412/// The caller of one `route.open`, as its `route.open refused` line names it.
7413///
7414/// A connection id alone cannot name a caller after the fact: ids restart at
7415/// every daemon restart, so the same number belongs to different modules
7416/// before and after one. These three fields are what the refused line needs
7417/// to be attributable on its own, and they are rendered with the same helpers
7418/// as the `route.open accepted` line so both directions grep alike.
7419struct RouteOpenCaller {
7420    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
7421    /// when the request presented a consumer identity whose launch nonce did
7422    /// not verify. The claimed module id is never printed as a principal.
7423    principal: String,
7424    /// `none` for an unscoped open; the admitted stamp's label once scope
7425    /// admission issued one; before that, the requested selector's label.
7426    scope: String,
7427    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
7428    /// because it is a string the caller chose.
7429    harness: String,
7430}
7431
7432impl RouteOpenCaller {
7433    fn new(
7434        principal: Option<&Principal>,
7435        scope: Option<&ScopeSelector>,
7436        identity: &BindIdentity,
7437    ) -> Self {
7438        Self {
7439            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7440            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7441            harness: bounded_log_text(&identity.harness),
7442        }
7443    }
7444
7445    /// For a refusal that has no parsed request at hand: names whatever the
7446    /// frame still says, and `unknown` where it says nothing parseable.
7447    fn unknown() -> Self {
7448        Self {
7449            principal: "unknown".to_string(),
7450            scope: "unknown".to_string(),
7451            harness: "unknown".to_string(),
7452        }
7453    }
7454}
7455
7456/// Longest client-chosen text (a scope ref, a harness) a route.open log line
7457/// prints before cutting it.
7458const MAX_LOGGED_TEXT_CHARS: usize = 64;
7459
7460/// `value` Debug-quoted, so control characters land escaped and cannot forge
7461/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
7462/// `…` inside the quotes so one request cannot flood the line.
7463fn bounded_log_text(value: &str) -> String {
7464    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7465    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7466        bounded.push('…');
7467    }
7468    format!("{bounded:?}")
7469}
7470
7471/// The `scope=` field of the `route.open accepted` line, from the stamp that
7472/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7473/// so a ref holding control characters cannot forge a log line, and cut to 64
7474/// characters with a trailing `…` to bound the line. The bind proof and the
7475/// scope's attributes never appear.
7476fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7477    use subc_protocol::scope::ScopeKind;
7478    let kind = match stamp.kind {
7479        ScopeKind::Head => "head",
7480        ScopeKind::Worker => "worker",
7481        ScopeKind::Ephemeral => "ephemeral",
7482    };
7483    format!(
7484        "{kind}/{}/{}@{}",
7485        principal_label(&stamp.owner),
7486        bounded_log_text(&stamp.scope_ref),
7487        stamp.scope_epoch
7488    )
7489}
7490
7491/// The `scope=` field of a `route.open refused` line for an open refused
7492/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
7493/// The kind is not known yet, so the literal `requested` stands in its place.
7494/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
7495/// owner it has not been checked against the configured modules: a quoted
7496/// owner means the value was never validated. The bind proof never appears.
7497fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7498    let epoch = selector
7499        .scope_epoch
7500        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7501    format!(
7502        "requested/{}/{}@{epoch}",
7503        bounded_log_text(&principal_label(&selector.owner)),
7504        bounded_log_text(&selector.scope_ref),
7505    )
7506}
7507
7508fn principal_label(principal: &Principal) -> String {
7509    match principal {
7510        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7511        Principal::Direct => "direct".to_string(),
7512        other => format!("{other:?}"),
7513    }
7514}
7515
7516fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7517    match request {
7518        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7519        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7520        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7521        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7522        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7523    }
7524}
7525
7526fn parse_client_control_request(
7527    body: &[u8],
7528) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7529    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7530        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7531            Ok(probe)
7532                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7533                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7534            {
7535                ControlRequestBodyError::InvalidBody
7536            }
7537            Ok(_) => ControlRequestBodyError::UnknownOp,
7538            Err(_) => ControlRequestBodyError::InvalidBody,
7539        };
7540        (err, classification)
7541    })
7542}
7543
7544fn parse_module_control_request_from_module(
7545    body: &[u8],
7546) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7547    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7548        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7549            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7550            Ok(_) => ControlRequestBodyError::UnknownOp,
7551            Err(_) => ControlRequestBodyError::InvalidBody,
7552        };
7553        (err, classification)
7554    })
7555}
7556
7557#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7558enum ProviderRoleKind {
7559    ToolProvider,
7560    PipelineStage,
7561    ManagementSurface,
7562    InternalService,
7563}
7564
7565fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7566    match role {
7567        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7568        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7569        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7570        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7571    }
7572}
7573
7574fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7575    roles.iter().map(provider_role_kind).collect()
7576}
7577
7578/// Most refused scope records named individually in the log per sync; the
7579/// `refused` count on the accepted line is always complete.
7580const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7581
7582/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7583#[derive(Debug, Default, PartialEq, Eq)]
7584struct ScopeOutcomeCounts {
7585    created: usize,
7586    replaced: usize,
7587    updated: usize,
7588    unchanged: usize,
7589    refused: usize,
7590}
7591
7592impl ScopeOutcomeCounts {
7593    fn of(results: &[ScopeRecordResult]) -> Self {
7594        let mut counts = Self::default();
7595        for result in results {
7596            let slot = match result.outcome {
7597                ScopeRecordOutcome::Created => &mut counts.created,
7598                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7599                ScopeRecordOutcome::Updated => &mut counts.updated,
7600                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7601                ScopeRecordOutcome::Refused => &mut counts.refused,
7602            };
7603            *slot += 1;
7604        }
7605        counts
7606    }
7607}
7608
7609#[cfg(test)]
7610mod scope_outcome_count_tests {
7611    use super::*;
7612
7613    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7614        ScopeRecordResult {
7615            scope_ref: "r".to_string(),
7616            scope_epoch: 1,
7617            outcome,
7618            code: None,
7619            message: None,
7620            version: None,
7621            parent_state: None,
7622        }
7623    }
7624
7625    /// Each outcome lands in its own count, so a refused record can never be
7626    /// hidden inside the total the log already printed.
7627    #[test]
7628    fn every_outcome_is_counted_in_its_own_field() {
7629        let results = [
7630            result(ScopeRecordOutcome::Created),
7631            result(ScopeRecordOutcome::Created),
7632            result(ScopeRecordOutcome::Replaced),
7633            result(ScopeRecordOutcome::Updated),
7634            result(ScopeRecordOutcome::Unchanged),
7635            result(ScopeRecordOutcome::Refused),
7636            result(ScopeRecordOutcome::Refused),
7637            result(ScopeRecordOutcome::Refused),
7638        ];
7639        assert_eq!(
7640            ScopeOutcomeCounts::of(&results),
7641            ScopeOutcomeCounts {
7642                created: 2,
7643                replaced: 1,
7644                updated: 1,
7645                unchanged: 1,
7646                refused: 3,
7647            }
7648        );
7649    }
7650}
7651
7652/// Return whether a catalog change can create a newly violating live route.
7653/// Removing an attested claim is intentionally excluded: it makes fewer routes
7654/// forbidden and therefore must leave the existing route census untouched.
7655fn capability_census_trigger(
7656    old: Option<&CapabilityDeclarations>,
7657    new: Option<&CapabilityDeclarations>,
7658) -> bool {
7659    let old_provides = old
7660        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7661        .unwrap_or_default();
7662    let old_denies = old
7663        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7664        .unwrap_or_default();
7665    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7666        provides: Vec::new(),
7667        requires: Vec::new(),
7668        must_never_reach: Vec::new(),
7669    });
7670
7671    new.provides
7672        .iter()
7673        .any(|capability| !old_provides.contains(capability))
7674        || new
7675            .must_never_reach
7676            .iter()
7677            .any(|capability| !old_denies.contains(capability))
7678}
7679
7680/// Find the first capability an attested opener denies that an attested target
7681/// claims. Both manifests are live registry records, never cached or client data.
7682fn denied_capability<'a>(
7683    opening_manifest: &'a ModuleManifest,
7684    target_manifest: &ModuleManifest,
7685) -> Option<&'a str> {
7686    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7687    let target_capabilities = target_manifest.capabilities.as_ref()?;
7688    opening_capabilities
7689        .must_never_reach
7690        .iter()
7691        .find(|denied| {
7692            target_capabilities
7693                .provides
7694                .iter()
7695                .any(|provided| provided == *denied)
7696        })
7697        .map(String::as_str)
7698}
7699
7700fn catalog_update_frozen_field_message(
7701    registered: &ModuleManifest,
7702    provides: &[ProviderRole],
7703) -> Option<String> {
7704    let old_has_provides = !registered.provides.is_empty();
7705    let new_has_provides = !provides.is_empty();
7706    if old_has_provides != new_has_provides {
7707        return Some(format!(
7708            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7709            registered.module_id
7710        ));
7711    }
7712
7713    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7714        return Some(format!(
7715            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7716            registered.module_id
7717        ));
7718    }
7719
7720    let registered_concurrency = manifest_concurrency(registered);
7721    let mut candidate = registered.clone();
7722    candidate.provides = provides.to_vec();
7723    let candidate_concurrency = manifest_concurrency(&candidate);
7724    if candidate_concurrency != registered_concurrency {
7725        return Some(format!(
7726            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7727            registered.module_id, registered_concurrency, candidate_concurrency
7728        ));
7729    }
7730
7731    // control_ops live beside the manifest in the HELLO body, not inside
7732    // ModuleManifest, so a provides-only catalog.update cannot change them.
7733    None
7734}
7735
7736fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7737    manifest.provides.iter().any(is_routable_role)
7738}
7739
7740/// Returns the routable-provider concurrency subc should enforce for this manifest.
7741///
7742/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7743/// InternalService has no role-specific concurrency field, so it retains the
7744/// existing ModuleManaged default for backward compatibility.
7745fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7746    manifest
7747        .provides
7748        .iter()
7749        .find_map(|provider| match provider {
7750            ProviderRole::ToolProvider { concurrency, .. }
7751            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7752            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7753        })
7754        .unwrap_or(Concurrency::ModuleManaged)
7755}
7756
7757/// True when the manifest carries a ManagementSurface role whose concurrency
7758/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7759/// bytes because the typed manifest deliberately erases that distinction: the
7760/// default exists for wire compatibility, and this probe exists so the default
7761/// stays observable. Any parse irregularity returns false -- the caller only
7762/// logs, and a malformed body already failed registration upstream.
7763fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7764    let has_management_surface = manifest
7765        .provides
7766        .iter()
7767        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7768    if !has_management_surface {
7769        return false;
7770    }
7771    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7772        return false;
7773    };
7774    let Some(provides) = raw
7775        .get("manifest")
7776        .and_then(|manifest| manifest.get("provides"))
7777        .and_then(serde_json::Value::as_array)
7778    else {
7779        return false;
7780    };
7781    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7782    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7783    // against the management_surface_manifest_without_concurrency golden, not
7784    // recalled (the externally-tagged guess was this function's first bug).
7785    provides.iter().any(|role| {
7786        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7787            && role.get("concurrency").is_none()
7788    })
7789}
7790
7791fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7792    if peer_version != PROTOCOL_VERSION {
7793        return Err(format!(
7794            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7795        ));
7796    }
7797    Ok(PROTOCOL_VERSION)
7798}
7799
7800fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7801    Frame::build_with_version(
7802        response_version(frame),
7803        FrameType::Pong,
7804        frame.header.flags,
7805        0,
7806        0,
7807        frame.header.corr,
7808        Vec::new(),
7809    )
7810    .map_err(RouterError::FrameBuild)
7811}
7812
7813pub(crate) fn control_error_frame(
7814    frame: &Frame,
7815    code: &'static str,
7816    message: impl Into<String>,
7817) -> Result<Frame, RouterError> {
7818    control_error_body_frame(
7819        frame,
7820        ErrorBody {
7821            code: code.to_string(),
7822            message: message.into(),
7823            detail: None,
7824        },
7825    )
7826}
7827
7828fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7829    let body = serde_json::to_vec(&error).map_err(|err| {
7830        RouterError::backend(
7831            0,
7832            frame.header.corr,
7833            format!("failed to encode control ERROR: {err}"),
7834        )
7835    })?;
7836
7837    Frame::build_with_version(
7838        response_version(frame),
7839        FrameType::Error,
7840        control_flags(),
7841        0,
7842        0,
7843        frame.header.corr,
7844        body,
7845    )
7846    .map_err(RouterError::FrameBuild)
7847}
7848
7849fn control_response_body_frame<T: Serialize>(
7850    frame: &Frame,
7851    reply: &T,
7852    label: &'static str,
7853) -> Result<Frame, RouterError> {
7854    let body = serde_json::to_vec(reply).map_err(|err| {
7855        RouterError::backend(
7856            0,
7857            frame.header.corr,
7858            format!("failed to encode {label}: {err}"),
7859        )
7860    })?;
7861
7862    Frame::build_with_version(
7863        response_version(frame),
7864        FrameType::Response,
7865        control_flags(),
7866        0,
7867        0,
7868        frame.header.corr,
7869        body,
7870    )
7871    .map_err(RouterError::FrameBuild)
7872}
7873
7874/// Map a forwarding failure to the wire code a client sees.
7875///
7876/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7877/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7878/// chosen here decides whether a caller retries or gives up.
7879///
7880/// That makes attribution the load-bearing property, not merely having a code. A
7881/// permanent fault published as a retryable one produces a fleet-wide retry storm
7882/// against something that can never recover; a transient fault published as
7883/// permanent gives up on work that would have succeeded. Both look correct in a
7884/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7885/// the mapping per variant rather than merely asserting that some code exists.
7886///
7887/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7888/// two codes on the same side of the boundary passes it. Measured rather than
7889/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7890/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7891/// a test named for something else that happens to assert the string.
7892///
7893/// That accidental coverage is deliberately left alone rather than promoted to a
7894/// named test, because it guards a property this function does not promise.
7895/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7896/// specific code within a class, so identity is free to change and only the
7897/// partition is a contract. Splitting it out would assert a guarantee nothing
7898/// depends on — and a suite that promises more than the code does is the harder
7899/// thing to correct later, because the next reader cannot tell which assertions
7900/// are load-bearing.
7901///
7902/// Pin identity here the moment a consumer branches on a specific code.
7903fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7904    match err {
7905        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7906        ForwardingError::NoModuleConnection => "target_unavailable",
7907        ForwardingError::ModuleReloading { .. } => "module_reloading",
7908        ForwardingError::ClientRouteChannelExhausted { .. }
7909        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7910        ForwardingError::StaleModuleEndpoint
7911        | ForwardingError::UnknownReservation { .. }
7912        | ForwardingError::ConnectionClosing { .. }
7913        | ForwardingError::ClientEgressClosed { .. }
7914        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7915        // Only a swap candidate's registration can produce this, and it means
7916        // exactly what a second active HELLO for a live id means.
7917        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7918        ForwardingError::RelayCorrelationExhausted
7919        | ForwardingError::RouteOpenBuild(_)
7920        | ForwardingError::Poisoned => "forwarding_error",
7921    }
7922}
7923
7924fn response_version(frame: &Frame) -> u8 {
7925    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7926        frame.header.ver
7927    } else {
7928        PROTOCOL_VERSION
7929    }
7930}
7931
7932fn control_flags() -> Flags {
7933    Flags::new(false, Priority::Passive, false)
7934}
7935
7936/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7937/// channel. The target is the module (a client never saw the route), so this
7938/// takes the module path: delivered late rather than dropped when the module's
7939/// queue is momentarily full, and never closing its connection.
7940fn send_goodbye_target_best_effort(
7941    counters: &DaemonCounters,
7942    target: &GoodbyeTarget,
7943    context: &'static str,
7944) {
7945    let Ok(frame) = Frame::build_with_version(
7946        target.negotiated_ver,
7947        FrameType::Goodbye,
7948        control_flags(),
7949        target.channel,
7950        target.epoch,
7951        0,
7952        Vec::new(),
7953    ) else {
7954        return;
7955    };
7956    crate::forwarding::send_module_route_goodbye(
7957        counters,
7958        &target.sink,
7959        frame,
7960        target.module_id.as_deref(),
7961        context,
7962    );
7963}
7964
7965pub(crate) fn send_route_control_pushes(
7966    forwarding: &ForwardingTable,
7967    routes: Vec<EndpointRoute>,
7968    push: ClientControlPush,
7969) {
7970    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7971    for route in routes {
7972        let target = route.goodbye_target;
7973        if let Some((existing, channels)) = targets
7974            .iter_mut()
7975            .find(|(existing, _)| existing.connection_id == target.connection_id)
7976        {
7977            debug_assert_eq!(
7978                existing.negotiated_ver, target.negotiated_ver,
7979                "one connection cannot negotiate multiple frame versions"
7980            );
7981            if !channels.contains(&target.channel) {
7982                channels.push(target.channel);
7983            }
7984            continue;
7985        }
7986        let channel = target.channel;
7987        targets.push((target, vec![channel]));
7988    }
7989    for (target, mut channels) in targets {
7990        channels.sort_unstable();
7991        let mut push = push.clone();
7992        match &mut push {
7993            ClientControlPush::RouteClosing {
7994                channels: covered, ..
7995            }
7996            | ClientControlPush::RouteClosed {
7997                channels: covered, ..
7998            } => *covered = channels,
7999        }
8000        let body = match serde_json::to_vec(&push) {
8001            Ok(body) => body,
8002            Err(err) => {
8003                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8004                continue;
8005            }
8006        };
8007        let frame = match Frame::build_with_version(
8008            target.negotiated_ver,
8009            FrameType::Push,
8010            control_flags(),
8011            0,
8012            0,
8013            0,
8014            body.clone(),
8015        ) {
8016            Ok(frame) => frame,
8017            Err(err) => {
8018                warn!(
8019                    route_channel = target.channel,
8020                    error = %err,
8021                    "failed to build route lifecycle control PUSH frame"
8022                );
8023                continue;
8024            }
8025        };
8026        if let Err(err) = target.sink.try_send(frame) {
8027            if target.close_on_delivery_failure() {
8028                warn!(
8029                    target_connection_id = target.connection_id.get(),
8030                    route_channel = target.channel,
8031                    error = %err,
8032                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8033                );
8034                let _ = forwarding.escalate_client_delivery_failure(
8035                    target.connection_id,
8036                    target.channel,
8037                    target.epoch,
8038                    CloseReason::new(
8039                        "route_lifecycle_push_delivery_failed",
8040                        format!(
8041                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8042                            target.channel
8043                        ),
8044                    ),
8045                    crate::forwarding::UndeliveredFrame {
8046                        module_id: target.module_id.as_deref(),
8047                        sink: &target.sink,
8048                    },
8049                );
8050            }
8051        }
8052    }
8053}
8054
8055#[cfg(test)]
8056mod tests {
8057    mod activity_read {
8058        include!("activity_read_tests.rs");
8059    }
8060
8061    mod lifecycle_activity {
8062        include!("control_lifecycle_tests.rs");
8063    }
8064    #[cfg(unix)]
8065    #[tokio::test]
8066    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8067        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8068        let path = dir.join("subc.jsonc");
8069        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8070            "program":"/bin/sleep","args":["60"],"protocol":"none",
8071            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8072        }}}).to_string()).unwrap();
8073        let mut configured = crate::daemon_config::load(&path)
8074            .unwrap()
8075            .unwrap()
8076            .modules
8077            .pop()
8078            .unwrap();
8079        let registry = std::sync::Arc::new(crate::Registry::default());
8080        let handle = crate::SupervisorHandle::new();
8081        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8082            .with_handle(handle.clone());
8083        let module = supervisor
8084            .supervise_configured_with_health(
8085                configured.module_spec(),
8086                true,
8087                configured.health.clone(),
8088                None,
8089                configured.restart,
8090            )
8091            .unwrap();
8092        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8093        let before = module.status().unwrap().pid;
8094        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8095        configured.health.cadence = std::time::Duration::from_secs(3600);
8096        let health_only = handler
8097            .reconcile_supervised_modules(
8098                &supervisor,
8099                vec![configured.clone()],
8100                false,
8101                Principal::Direct,
8102            )
8103            .await
8104            .unwrap();
8105        assert!(
8106            health_only.0.changed_pending_reload.is_empty(),
8107            "health policy is already applied live"
8108        );
8109        assert_eq!(module.status().unwrap().pid, before);
8110        assert_eq!(
8111            module.configuration().unwrap().1.http,
8112            configured.health.http
8113        );
8114        configured.args = vec!["61".into()];
8115        let launch = handler
8116            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8117            .await
8118            .unwrap();
8119        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8120        assert_eq!(
8121            module.status().unwrap().pid,
8122            before,
8123            "a launch edit is stored until reload"
8124        );
8125        module.drain().await.unwrap();
8126    }
8127    use cortexkit_test_support::ScratchDir;
8128    use std::{
8129        collections::BTreeMap,
8130        fmt,
8131        path::PathBuf,
8132        sync::{Arc, Mutex},
8133        time::Duration,
8134    };
8135
8136    use serde_json::{json, Value};
8137    use subc_protocol::{
8138        manifest::{
8139            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8140            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8141        },
8142        session::HealthStatus,
8143        FrameType,
8144    };
8145
8146    use super::*;
8147    use crate::{
8148        forwarding::{DataRoute, DataRouteState},
8149        registry::ChannelState,
8150        router::FrameSink,
8151        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8152        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8153        RouteCtx, Router,
8154    };
8155    use tokio::{
8156        sync::mpsc,
8157        time::{sleep, Instant},
8158    };
8159    use tracing::{
8160        field::{Field, Visit},
8161        Event, Subscriber,
8162    };
8163    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8164
8165    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8166    ///
8167    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8168    /// is only populated for `tests/*.rs` integration test binaries -- this file
8169    /// compiles as part of the library target, which gets neither. This test's
8170    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8171    /// and the sibling binary lives two directories up at
8172    /// `<target-dir>/<profile>/fake-aft-stub`.
8173    ///
8174    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8175    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8176    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8177    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8178    /// `NotFound`, which reads as a broken test rather than an unbuilt
8179    /// dependency -- so state the cause and the remedy instead. Deliberately a
8180    /// panic and not a silent skip: a test that quietly passes when it could not
8181    /// run is worse than one that fails, because it reports health it never
8182    /// verified.
8183    fn fake_aft_stub_path() -> PathBuf {
8184        let mut path = std::env::current_exe().expect("current_exe available in tests");
8185        path.pop(); // .../deps/
8186        path.pop(); // .../<profile>/
8187        path.push(if cfg!(windows) {
8188            "fake-aft-stub.exe"
8189        } else {
8190            "fake-aft-stub"
8191        });
8192        assert!(
8193            path.exists(),
8194            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8195             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8196            path.display()
8197        );
8198        path
8199    }
8200
8201    /// Whether clients retry `code` in place: the predicate itself, never a copy
8202    /// of its set. A copied list breaks silently when a code is added to or
8203    /// removed from the real one, and a stale copy here would let exactly the
8204    /// failure this test exists to catch pass.
8205    fn client_retries(code: &str) -> bool {
8206        subc_protocol::error_codes::is_retryable_route_open(code)
8207    }
8208
8209    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8210    /// of failure is worse than publishing none. A permanent fault dressed as
8211    /// retryable makes every client in the fleet retry forever against something
8212    /// that cannot recover; a transient fault dressed as permanent abandons work
8213    /// that would have succeeded.
8214    ///
8215    /// Asserting "a code exists" cannot catch either, because the string is free
8216    /// to say anything. This enumerates every variant and pins which side of the
8217    /// retry boundary it lands on, so a new variant must be classified here
8218    /// deliberately rather than inheriting whichever arm it was appended to.
8219    #[test]
8220    fn retryability_of_forwarding_codes_matches_the_failure() {
8221        // Transient by nature: the target is booting, reloading, or its endpoint
8222        // was swapped mid-flight. Retrying is how these resolve.
8223        let transient = [
8224            ForwardingError::NoModuleConnection,
8225            ForwardingError::ModuleReloading {
8226                module_id: "m".into(),
8227            },
8228            ForwardingError::StaleModuleEndpoint,
8229            ForwardingError::UnknownReservation {
8230                client_channel: 1,
8231                module_channel: 1,
8232            },
8233            ForwardingError::ConnectionClosing {
8234                connection_id: ConnectionId::new(1),
8235            },
8236            ForwardingError::ClientEgressClosed {
8237                connection_id: ConnectionId::new(1),
8238            },
8239            ForwardingError::ModuleEgressUnavailable {
8240                connection_id: ConnectionId::new(1),
8241            },
8242        ];
8243        for err in transient {
8244            let code = forwarding_error_code(&err);
8245            assert!(
8246                client_retries(code),
8247                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8248            );
8249        }
8250
8251        // Not fixed by retrying. Channel and correlation exhaustion need the
8252        // caller to close routes, and a poisoned lock is a daemon that cannot
8253        // recover at all — the worst thing to advertise as retryable, since every
8254        // client would storm a daemon that will never answer.
8255        let permanent = [
8256            ForwardingError::ConnectionRoleConflict {
8257                connection_id: ConnectionId::new(1),
8258            },
8259            ForwardingError::ClientRouteChannelExhausted {
8260                connection_id: ConnectionId::new(1),
8261            },
8262            ForwardingError::ModuleRouteChannelExhausted {
8263                endpoint: ModuleEndpointId {
8264                    connection_id: ConnectionId::new(1),
8265                    generation: 1,
8266                },
8267            },
8268            ForwardingError::RelayCorrelationExhausted,
8269            ForwardingError::RouteOpenBuild("x".into()),
8270            ForwardingError::Poisoned,
8271        ];
8272        for err in permanent {
8273            let code = forwarding_error_code(&err);
8274            assert!(
8275                !client_retries(code),
8276                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8277            );
8278        }
8279    }
8280
8281    /// The principal is the daemon's answer to "who is calling", and modules
8282    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8283    /// plexus gates connector invocation. So a stamp is an authorization input in
8284    /// another process, not a label — and both possible answers SUCCEED, which is
8285    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8286    /// first-party capability to something that never proved it; a supervised one
8287    /// stamped `Direct` silently strips a module of capability it is entitled to.
8288    ///
8289    /// Neither shows up in a test that only checks the bind succeeded. Before this
8290    /// test the only coverage was accidental —
8291    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8292    /// stamped principal on its way past, so narrowing that wire-shape test to its
8293    /// stated subject would have deleted the last assertion on this value. It
8294    /// still asserts the stamp, which is now redundancy rather than the only
8295    /// guard: both fail under the same mutation, and this one names the reason.
8296    /// SCOPE: this handler's supervisor has spawned nothing, so
8297    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8298    /// is unreachable here. Both assertions below are refusals, and a mutant that
8299    /// refuses everything would satisfy them.
8300    ///
8301    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8302    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8303    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8304    /// at source rather than assumed, since a citation is a claim about another
8305    /// file and ages like one. Recorded because a harness that structurally
8306    /// cannot reach an arm reports "none" for that arm identically to one that
8307    /// covers it and found nothing.
8308    #[tokio::test]
8309    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8310        let handler = ControlHandler::default();
8311        let frame =
8312            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8313
8314        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8315        // any process holding the connection file. Nothing was proved, so nothing
8316        // may be granted beyond the unattested floor.
8317        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8318        assert_eq!(
8319            stamped,
8320            Principal::Direct,
8321            "a caller that proved nothing must not be stamped as a supervised module"
8322        );
8323
8324        // A claimed module_id with a nonce no supervised child was given is a
8325        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8326        // quietly demoted to Direct, or an impersonation attempt looks identical
8327        // to an ordinary unattested connection.
8328        let forged = handler
8329            .route_open_principal(
8330                &frame,
8331                Some(ConsumerIdentity {
8332                    module_id: "aft".to_string(),
8333                    launch_nonce: "not-a-real-nonce".to_string(),
8334                }),
8335            )
8336            .unwrap();
8337        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8338        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8339    }
8340
8341    /// The test above hands `route_open_principal` an identity it built itself,
8342    /// which proves the stamping rule and nothing about where the identity comes
8343    /// from. The real producer is a wire body, and the two are joined by a serde
8344    /// field name that nothing else asserts.
8345    ///
8346    /// That join fails quietly in one specific way: an unrecognised key is simply
8347    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8348    /// yields `None` and every supervised module silently drops to `Direct`.
8349    /// Capability-wise that is the safe direction, but it surfaces far from its
8350    /// cause — as a module mysteriously refused bash — and it would pass every
8351    /// test that builds its own input.
8352    ///
8353    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8354    /// would break every client the moment the daemon gains a field, trading a
8355    /// quiet demotion for a hard refusal on additive change. Asserting the join
8356    /// instead means a rename breaks a test here rather than the fleet.
8357    #[test]
8358    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8359        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8360        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8361        let ClientControlRequest::RouteOpen {
8362            consumer_identity, ..
8363        } = parsed
8364        else {
8365            panic!("route.open body must parse as RouteOpen");
8366        };
8367        assert_eq!(
8368            consumer_identity,
8369            Some(ConsumerIdentity {
8370                module_id: "aft".to_string(),
8371                launch_nonce: "n".to_string(),
8372            }),
8373            "the wire field name must reach the value route_open_principal reads"
8374        );
8375    }
8376
8377    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8378        ModuleManifest::builder(module_id, "0.1.0")
8379            .protocol_ver(protocol_ver)
8380            .provides(vec![ProviderRole::ToolProvider {
8381                tools: vec![Tool {
8382                    name: "read".to_string(),
8383                    description: None,
8384                    execution_mode: ExecutionMode::Pure,
8385                    schema: json!({"type": "object"}),
8386                }],
8387                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8388                concurrency: Concurrency::ModuleManaged,
8389                emits_push: true,
8390                sub_supervises: true,
8391            }])
8392            .build()
8393    }
8394
8395    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8396        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8397    }
8398
8399    fn hello_frame_with_control_ops(
8400        module_id: &str,
8401        protocol_ver: u8,
8402        corr: u64,
8403        control_ops: Option<Vec<String>>,
8404    ) -> Frame {
8405        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8406    }
8407
8408    fn hello_frame_with_nonce(
8409        module_id: &str,
8410        protocol_ver: u8,
8411        corr: u64,
8412        launch_nonce: Option<&str>,
8413    ) -> Frame {
8414        hello_frame_full(
8415            module_id,
8416            protocol_ver,
8417            corr,
8418            None,
8419            launch_nonce.map(ToOwned::to_owned),
8420        )
8421    }
8422
8423    fn hello_frame_full(
8424        module_id: &str,
8425        protocol_ver: u8,
8426        corr: u64,
8427        control_ops: Option<Vec<String>>,
8428        launch_nonce: Option<String>,
8429    ) -> Frame {
8430        let body = serde_json::to_vec(&ModuleHelloBody {
8431            manifest: manifest(module_id, protocol_ver),
8432            protocol_ver,
8433            control_ops,
8434            launch_nonce,
8435        })
8436        .unwrap();
8437        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8438    }
8439
8440    fn hello_frame_with_rich_ui(
8441        module_id: &str,
8442        protocol_ver: u8,
8443        corr: u64,
8444        launch_nonce: Option<&str>,
8445        rich_ui: Value,
8446    ) -> Frame {
8447        let mut body = serde_json::to_value(ModuleHelloBody {
8448            manifest: manifest(module_id, protocol_ver),
8449            protocol_ver,
8450            control_ops: None,
8451            launch_nonce: launch_nonce.map(str::to_string),
8452        })
8453        .unwrap();
8454        body["manifest"]["rich_ui"] = rich_ui;
8455        Frame::build(
8456            FrameType::Hello,
8457            control_flags(),
8458            0,
8459            0,
8460            corr,
8461            serde_json::to_vec(&body).unwrap(),
8462        )
8463        .unwrap()
8464    }
8465
8466    fn non_routable_hello_frame_with_control_ops(
8467        module_id: &str,
8468        corr: u64,
8469        control_ops: Option<Vec<String>>,
8470    ) -> Frame {
8471        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8472        manifest.provides.clear();
8473        let body = serde_json::to_vec(&ModuleHelloBody {
8474            manifest,
8475            protocol_ver: PROTOCOL_VERSION,
8476            control_ops,
8477            launch_nonce: None,
8478        })
8479        .unwrap();
8480        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8481    }
8482
8483    fn capability_grammar_hello_frame(
8484        capabilities: Value,
8485        runtime_computed: Option<Value>,
8486        corr: u64,
8487    ) -> Frame {
8488        let mut body = serde_json::to_value(ModuleHelloBody {
8489            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8490            protocol_ver: PROTOCOL_VERSION,
8491            control_ops: None,
8492            launch_nonce: None,
8493        })
8494        .expect("HELLO body serializes");
8495        body["manifest"]["capabilities"] = capabilities;
8496        if let Some(runtime_computed) = runtime_computed {
8497            body["runtime_computed"] = runtime_computed;
8498        }
8499        Frame::build(
8500            FrameType::Hello,
8501            control_flags(),
8502            0,
8503            0,
8504            corr,
8505            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8506        )
8507        .expect("HELLO frame builds")
8508    }
8509
8510    fn channel_request(channel: u16, corr: u64) -> Frame {
8511        Frame::build(
8512            FrameType::Request,
8513            Flags::new(true, Priority::Interactive, false),
8514            channel,
8515            0,
8516            corr,
8517            b"opaque".to_vec(),
8518        )
8519        .unwrap()
8520    }
8521
8522    fn route_ctx(
8523        connection_id: ConnectionId,
8524    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8525        let (tx, rx) = mpsc::channel(8);
8526        (
8527            RouteCtx {
8528                connection_id,
8529                egress: FrameSink::new(tx),
8530            },
8531            rx,
8532        )
8533    }
8534
8535    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8536        serde_json::from_slice(&frame.body).unwrap()
8537    }
8538
8539    /// Register a module over a connection that has a sink and return the
8540    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8541    /// module's own sink rather than returning it as a reply, so the ack is
8542    /// taken off `rx` here and whatever the test reads next is what followed it.
8543    async fn hello_via_sink(
8544        handler: &ControlHandler,
8545        ctx: &RouteCtx,
8546        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8547        hello: Frame,
8548    ) -> Frame {
8549        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8550        assert!(
8551            replies.is_empty(),
8552            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8553        );
8554        let ack = rx
8555            .try_recv()
8556            .expect("HELLO_ACK is queued on the module sink")
8557            .frame;
8558        assert_eq!(ack.header.ty, FrameType::HelloAck);
8559        ack
8560    }
8561
8562    fn parse_error(frame: &Frame) -> Value {
8563        serde_json::from_slice(&frame.body).unwrap()
8564    }
8565
8566    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8567        serde_json::from_slice(&frame.body).unwrap()
8568    }
8569
8570    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8571        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8572            route_channel,
8573            route_epoch: 0,
8574            kind,
8575        })
8576        .unwrap();
8577        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8578    }
8579
8580    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8581        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8582            module_id: module_id.to_string(),
8583        })
8584        .unwrap();
8585        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8586    }
8587
8588    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8589        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8590    }
8591
8592    fn route_open_frame_with_consumer_capabilities(
8593        corr: u64,
8594        module_id: &str,
8595        project_root: ScratchDir,
8596        consumer_capabilities: Option<Vec<String>>,
8597    ) -> Frame {
8598        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8599            target: RouteTarget::ToolProvider {
8600                module_id: module_id.to_string(),
8601            },
8602            identity: BindIdentity::new(
8603                project_root.path().to_path_buf(),
8604                "unit".to_string(),
8605                "session".to_string(),
8606            ),
8607            consumer_identity: None,
8608            consumer_capabilities,
8609            role_versions: None,
8610            admission_facts: None,
8611            scope: None,
8612        })
8613        .unwrap();
8614        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8615    }
8616
8617    fn route_open_frame_with_role_versions(
8618        corr: u64,
8619        module_id: &str,
8620        project_root: ScratchDir,
8621        role_versions: Option<BTreeMap<String, String>>,
8622    ) -> Frame {
8623        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8624            target: RouteTarget::ToolProvider {
8625                module_id: module_id.to_string(),
8626            },
8627            identity: BindIdentity::new(
8628                project_root.path().to_path_buf(),
8629                "unit".to_string(),
8630                format!("session-{corr}"),
8631            ),
8632            consumer_identity: None,
8633            consumer_capabilities: None,
8634            role_versions,
8635            admission_facts: None,
8636            scope: None,
8637        })
8638        .unwrap();
8639        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8640    }
8641
8642    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8643        entries
8644            .iter()
8645            .map(|(role, version)| (role.to_string(), version.to_string()))
8646            .collect()
8647    }
8648
8649    fn route_open_frame_with_admission_facts(
8650        corr: u64,
8651        module_id: &str,
8652        project_root: ScratchDir,
8653        consumer_identity: Option<subc_control::ConsumerIdentity>,
8654        facts: Option<Value>,
8655    ) -> Frame {
8656        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8657            target: RouteTarget::ToolProvider {
8658                module_id: module_id.to_string(),
8659            },
8660            identity: BindIdentity::new(
8661                project_root.path().to_path_buf(),
8662                "unit".to_string(),
8663                format!("session-{corr}"),
8664            ),
8665            consumer_identity,
8666            consumer_capabilities: None,
8667            role_versions: None,
8668            admission_facts: facts,
8669            scope: None,
8670        })
8671        .unwrap();
8672        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8673    }
8674
8675    #[derive(Clone, Default)]
8676    struct EventCapture {
8677        events: Arc<Mutex<Vec<CapturedEvent>>>,
8678    }
8679
8680    #[derive(Clone, Debug)]
8681    struct CapturedEvent {
8682        target: String,
8683        level: tracing::Level,
8684        fields: BTreeMap<String, String>,
8685    }
8686
8687    impl EventCapture {
8688        fn events(&self) -> Vec<CapturedEvent> {
8689            self.events.lock().unwrap().clone()
8690        }
8691    }
8692
8693    impl<S> Layer<S> for EventCapture
8694    where
8695        S: Subscriber,
8696    {
8697        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8698            let mut visitor = EventFieldVisitor::default();
8699            event.record(&mut visitor);
8700            self.events.lock().unwrap().push(CapturedEvent {
8701                target: event.metadata().target().to_string(),
8702                level: *event.metadata().level(),
8703                fields: visitor.fields,
8704            });
8705        }
8706    }
8707
8708    #[derive(Default)]
8709    struct EventFieldVisitor {
8710        fields: BTreeMap<String, String>,
8711    }
8712
8713    impl Visit for EventFieldVisitor {
8714        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8715            self.fields
8716                .insert(field.name().to_string(), format!("{value:?}"));
8717        }
8718    }
8719
8720    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8721        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8722            status,
8723            detail: Some("warming".to_string()),
8724            metrics: Some(json!({"queue_depth": 3})),
8725        })
8726        .unwrap();
8727        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8728    }
8729
8730    fn route_bind_ack(corr: u64) -> Frame {
8731        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8732        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8733    }
8734
8735    fn unique_project_root(label: &str) -> ScratchDir {
8736        ScratchDir::new(label)
8737    }
8738
8739    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8740        match parse_route_poll(frame) {
8741            ClientControlResponse::RoutePoll {
8742                status: None,
8743                live: Some(live),
8744                ..
8745            } => assert_eq!(live, expected_live),
8746            other => panic!("unexpected route.poll response: {other:?}"),
8747        }
8748    }
8749
8750    fn bind_liveness_route(
8751        registry: &Registry,
8752        forwarding: &ForwardingTable,
8753        module_id: &str,
8754    ) -> (RouteCtx, u16, u32) {
8755        let module_connection = ConnectionId::new(101);
8756        let client_connection = ConnectionId::new(202);
8757        let registration = registry
8758            .register_with_control_ops(
8759                manifest(module_id, PROTOCOL_VERSION),
8760                PROTOCOL_VERSION,
8761                module_connection,
8762                module_baseline_control_ops(),
8763            )
8764            .unwrap();
8765        let (module_tx, _module_rx) = mpsc::channel(8);
8766        let endpoint = forwarding
8767            .register_module_connection(
8768                module_connection,
8769                module_id.to_string(),
8770                PROTOCOL_VERSION,
8771                manifest_concurrency(&registration.manifest),
8772                FrameSink::new(module_tx),
8773            )
8774            .unwrap();
8775        let (client_ctx, _client_rx) = route_ctx(client_connection);
8776        let pending = forwarding
8777            .begin_route_bind_relay_for_test(
8778                client_connection,
8779                client_ctx.egress.clone(),
8780                1,
8781                module_id,
8782            )
8783            .unwrap();
8784        assert_eq!(pending.endpoint, endpoint);
8785        let route_channel = pending.client_channel;
8786        let route_epoch = pending.client_epoch;
8787        forwarding
8788            .complete_pending_relay(
8789                module_connection,
8790                pending.corr,
8791                RouteBindRelayOutcome::Accepted,
8792            )
8793            .unwrap();
8794        (client_ctx, route_channel, route_epoch)
8795    }
8796
8797    struct FakeProcessLiveness {
8798        live: Option<bool>,
8799    }
8800
8801    impl ModuleProcessLiveness for FakeProcessLiveness {
8802        fn process_live(&self, _module_id: &str) -> Option<bool> {
8803            self.live
8804        }
8805    }
8806
8807    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8808    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8809    {
8810        let registry = Arc::new(Registry::default());
8811        let supervisor_handle = SupervisorHandle::new();
8812        let supervisor = Supervisor::new_for_test(
8813            Arc::clone(&registry),
8814            RestartPolicy::new(1, Duration::from_millis(10)),
8815        )
8816        .with_handle(supervisor_handle.clone());
8817        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8818        let module = supervisor
8819            .spawn(ModuleSpec {
8820                module_id: "stderr-tail-wire".to_string(),
8821                program: fake_aft_stub_path(),
8822                args: Vec::new(),
8823                env: vec![
8824                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8825                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8826                ],
8827                reserved: false,
8828                reserved_prefixes: Vec::new(),
8829                protocol: ModuleProtocol::Subc,
8830                overlap: Default::default(),
8831            })
8832            .unwrap();
8833
8834        let deadline = Instant::now() + Duration::from_secs(5);
8835        loop {
8836            let tail = module.stderr_tail(None, None);
8837            if tail
8838                .entries
8839                .iter()
8840                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8841                && tail.entries.iter().any(|entry| {
8842                    matches!(
8843                        entry,
8844                        TailEntry::Line {
8845                            truncated: true,
8846                            ..
8847                        }
8848                    )
8849                })
8850            {
8851                break;
8852            }
8853            assert!(
8854                Instant::now() < deadline,
8855                "module did not produce a truncated line and restart boundary: {tail:?}"
8856            );
8857            sleep(Duration::from_millis(10)).await;
8858        }
8859
8860        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8861        let request = ClientControlRequest::SupervisorStderrTail {
8862            module_id: "stderr-tail-wire".to_string(),
8863            max_lines: None,
8864            max_bytes: None,
8865        };
8866        let frame = Frame::build(
8867            FrameType::Request,
8868            control_flags(),
8869            0,
8870            0,
8871            1,
8872            serde_json::to_vec(&request).unwrap(),
8873        )
8874        .unwrap();
8875        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8876        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8877        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8878            serde_json::from_slice(&responses[0].body).unwrap()
8879        else {
8880            panic!("expected supervisor.stderr_tail response");
8881        };
8882
8883        assert!(
8884            tail.entries
8885                .iter()
8886                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8887            "the control response lost the restart boundary"
8888        );
8889        let Some(StderrTailEntry::Line {
8890            text,
8891            truncated,
8892            at_ms,
8893        }) = tail.entries.iter().find(|entry| {
8894            matches!(
8895                entry,
8896                StderrTailEntry::Line {
8897                    truncated: true,
8898                    ..
8899                }
8900            )
8901        })
8902        else {
8903            panic!("the control response lost the truncated line");
8904        };
8905        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8906        assert!(*truncated);
8907        assert!(
8908            at_ms.is_some(),
8909            "the control response lost the line's capture time"
8910        );
8911    }
8912
8913    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8914    /// read done on the worker thread would stall every other task until it
8915    /// finished; the read must run off the worker so this test's own task keeps
8916    /// running while the read is paused.
8917    #[tokio::test(flavor = "current_thread")]
8918    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8919        let dir = ScratchDir::new("terminals-off-worker");
8920        let journal_path = dir.join("terminals.jsonl");
8921        let registry = Arc::new(Registry::default());
8922        let supervisor_handle = SupervisorHandle::new();
8923        let supervisor =
8924            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8925                .with_handle(supervisor_handle.clone())
8926                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8927        let module = supervisor
8928            .spawn(ModuleSpec {
8929                module_id: "terminal-off-worker".to_string(),
8930                program: fake_aft_stub_path(),
8931                args: Vec::new(),
8932                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8933                reserved: false,
8934                reserved_prefixes: Vec::new(),
8935                protocol: ModuleProtocol::Subc,
8936                overlap: Default::default(),
8937            })
8938            .unwrap();
8939        let deadline = Instant::now() + Duration::from_secs(5);
8940        while module.terminal_history().entries.len() != 2 {
8941            assert!(Instant::now() < deadline, "module did not record two exits");
8942            sleep(Duration::from_millis(10)).await;
8943        }
8944
8945        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8946        let handler =
8947            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8948        let frame = Frame::build(
8949            FrameType::Request,
8950            control_flags(),
8951            0,
8952            0,
8953            1,
8954            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8955                module_id: "terminal-off-worker".to_string(),
8956            })
8957            .unwrap(),
8958        )
8959        .unwrap();
8960        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8961        let spawned_at = std::time::Instant::now();
8962        let read = tokio::spawn({
8963            let handler = Arc::clone(&handler);
8964            async move { handler.handle_control_frame(&ctx, frame).await }
8965        });
8966        // Waiting for the pause from a blocking thread keeps this task pending,
8967        // so the runtime's single worker is free to run the read task.
8968        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8969            .await
8970            .unwrap()
8971            .expect("the history read reached its pause");
8972        let elapsed = spawned_at.elapsed();
8973        assert!(
8974            elapsed < Duration::from_secs(2) && !read.is_finished(),
8975            "this task could not run while the history read was paused \
8976             (resumed after {elapsed:?}, read finished: {})",
8977            read.is_finished()
8978        );
8979
8980        drop(release);
8981        let responses = read.await.unwrap().unwrap();
8982        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8983        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8984            panic!("expected supervisor.terminals response");
8985        };
8986        assert_eq!(terminals.entries.len(), 2);
8987        assert_eq!(terminals.journal_skipped_lines, 0);
8988        assert_eq!(terminals.journal_read_errors, 0);
8989    }
8990
8991    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8992    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8993        let registry = Arc::new(Registry::default());
8994        let supervisor_handle = SupervisorHandle::new();
8995        let supervisor =
8996            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8997                .with_handle(supervisor_handle.clone());
8998        let module = supervisor
8999            .spawn(ModuleSpec {
9000                module_id: "terminal-golden".to_string(),
9001                program: fake_aft_stub_path(),
9002                args: Vec::new(),
9003                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9004                reserved: false,
9005                reserved_prefixes: Vec::new(),
9006                protocol: ModuleProtocol::Subc,
9007                overlap: Default::default(),
9008            })
9009            .unwrap();
9010
9011        let deadline = Instant::now() + Duration::from_secs(5);
9012        while module.terminal_history().entries.len() != 2 {
9013            assert!(
9014                Instant::now() < deadline,
9015                "module did not retain two terminal exits: {:?}",
9016                module.terminal_history()
9017            );
9018            sleep(Duration::from_millis(10)).await;
9019        }
9020
9021        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9022        let request = ClientControlRequest::SupervisorTerminals {
9023            module_id: "terminal-golden".to_string(),
9024        };
9025        let frame = Frame::build(
9026            FrameType::Request,
9027            control_flags(),
9028            0,
9029            0,
9030            1,
9031            serde_json::to_vec(&request).unwrap(),
9032        )
9033        .unwrap();
9034        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9035        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9036        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9037        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9038            panic!("expected supervisor.terminals response");
9039        };
9040        assert_eq!(terminals.entries.len(), 2);
9041        assert_eq!(terminals.dropped, 0);
9042
9043        let mut rendered = serde_json::to_value(response).unwrap();
9044        // Wall-clock fields are the observation contract, but not stable fixture
9045        // bytes; normalize only them after the real handler has shaped the response.
9046        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9047        for (index, entry) in rendered["entries"]
9048            .as_array_mut()
9049            .expect("terminal response entries array")
9050            .iter_mut()
9051            .enumerate()
9052        {
9053            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9054        }
9055
9056        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9057            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9058        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9059        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9060            std::fs::write(&golden_path, &serialized).unwrap();
9061        }
9062        let expected: Value =
9063            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9064        assert_eq!(rendered, expected);
9065    }
9066
9067    #[test]
9068    fn hello_registers_manifest_and_returns_ack() {
9069        let registry = Arc::new(Registry::default());
9070        let handler = ControlHandler::new(Arc::clone(&registry));
9071        let conn = ConnectionId::new(1);
9072
9073        let responses = handler
9074            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9075            .unwrap();
9076
9077        assert_eq!(responses.len(), 1);
9078        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9079        assert_eq!(responses[0].header.channel, 0);
9080        assert_eq!(responses[0].header.corr, 7);
9081        let ack = parse_ack(&responses[0]);
9082        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9083        assert!(ack
9084            .subc_capabilities
9085            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9086        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9087        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9088        assert!(ack
9089            .subc_ops
9090            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9091        assert!(ack
9092            .subc_ops
9093            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9094
9095        let registration = registry.get_module("aft").unwrap().unwrap();
9096        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9097        assert_eq!(registration.state, ChannelState::Active);
9098        assert_eq!(registration.connection_id, conn);
9099        assert_eq!(registration.control_ops, module_baseline_control_ops());
9100    }
9101
9102    #[test]
9103    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9104        let invalid_identifiers = [
9105            ("case_change", "credentials-Provider/v1"),
9106            ("leading_zero", "credentials-provider/v01"),
9107            ("trailing_hyphen", "credentials-provider-/v1"),
9108            ("consecutive_hyphens", "credentials--provider/v1"),
9109            ("uppercase", "Credentials-provider/v1"),
9110            ("missing_v", "credentials-provider/1"),
9111            ("whitespace", "credentials provider/v1"),
9112            ("zero_version", "credentials-provider/v0"),
9113            ("out_of_range_version", "credentials-provider/v4294967296"),
9114            (
9115                "overlength_name",
9116                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9117            ),
9118        ];
9119        let mut cases = invalid_identifiers
9120            .into_iter()
9121            .map(|(name, identifier)| {
9122                (
9123                    format!("identifier_{name}"),
9124                    "capabilities.provides[0]".to_string(),
9125                    identifier.to_string(),
9126                    json!({ "provides": [identifier] }),
9127                    None,
9128                )
9129            })
9130            .collect::<Vec<_>>();
9131        cases.extend([
9132            (
9133                "unknown_need".to_string(),
9134                "capabilities.requires[0].need".to_string(),
9135                "deferred".to_string(),
9136                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9137                None,
9138            ),
9139            (
9140                "duplicate_provides".to_string(),
9141                "capabilities.provides[1]".to_string(),
9142                "credentials-provider/v1".to_string(),
9143                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9144                None,
9145            ),
9146            (
9147                "duplicate_must_never_reach".to_string(),
9148                "capabilities.must_never_reach[1]".to_string(),
9149                "credentials-provider/v1".to_string(),
9150                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9151                None,
9152            ),
9153            (
9154                "duplicate_requires_same_need".to_string(),
9155                "capabilities.requires[1]".to_string(),
9156                "credentials-provider/v1".to_string(),
9157                json!({ "requires": [
9158                    { "capability": "credentials-provider/v1", "need": "required" },
9159                    { "capability": "credentials-provider/v1", "need": "required" }
9160                ] }),
9161                None,
9162            ),
9163            (
9164                "duplicate_requires_conflicting_need".to_string(),
9165                "capabilities.requires[1]".to_string(),
9166                "credentials-provider/v1".to_string(),
9167                json!({ "requires": [
9168                    { "capability": "credentials-provider/v1", "need": "required" },
9169                    { "capability": "credentials-provider/v1", "need": "optional" }
9170                ] }),
9171                None,
9172            ),
9173            (
9174                "capabilities_root_pointer".to_string(),
9175                "runtime_computed[0]".to_string(),
9176                "/capabilities".to_string(),
9177                json!({}),
9178                Some(json!(["/capabilities"])),
9179            ),
9180            (
9181                "capabilities_descendant_pointer".to_string(),
9182                "runtime_computed[0]".to_string(),
9183                "/capabilities/provides".to_string(),
9184                json!({}),
9185                Some(json!(["/capabilities/provides"])),
9186            ),
9187            (
9188                "malformed_pointer_without_leading_slash".to_string(),
9189                "runtime_computed[0]".to_string(),
9190                "capabilities".to_string(),
9191                json!({}),
9192                Some(json!(["capabilities"])),
9193            ),
9194            (
9195                "malformed_pointer_escape".to_string(),
9196                "runtime_computed[0]".to_string(),
9197                "/roles/~2/tools".to_string(),
9198                json!({}),
9199                Some(json!(["/roles/~2/tools"])),
9200            ),
9201            (
9202                "unknown_capabilities_field".to_string(),
9203                "capabilities.future".to_string(),
9204                "<array>".to_string(),
9205                json!({ "future": [] }),
9206                None,
9207            ),
9208        ]);
9209
9210        for (index, (name, field, value, capabilities, runtime_computed)) in
9211            cases.into_iter().enumerate()
9212        {
9213            let registry = Arc::new(Registry::default());
9214            let handler = ControlHandler::new(Arc::clone(&registry));
9215            let response = handler
9216                .handle_control(
9217                    ConnectionId::new((index + 1) as u64),
9218                    capability_grammar_hello_frame(
9219                        capabilities,
9220                        runtime_computed,
9221                        index as u64 + 1,
9222                    ),
9223                )
9224                .expect("invalid HELLO returns a refusal");
9225
9226            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9227            let error = parse_error(&response[0]);
9228            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9229            let message = error["message"]
9230                .as_str()
9231                .expect("error message is a string");
9232            assert!(
9233                message.contains(&field),
9234                "{name}: field missing from {message}"
9235            );
9236            assert!(
9237                message.contains(&value),
9238                "{name}: value missing from {message}"
9239            );
9240            assert_eq!(
9241                registry
9242                    .active_registration_count()
9243                    .expect("registry reads"),
9244                0,
9245                "{name}: refused HELLO must not create a catalog entry"
9246            );
9247        }
9248    }
9249
9250    #[test]
9251    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9252        let registry = Arc::new(Registry::default());
9253        let handler = ControlHandler::new(Arc::clone(&registry));
9254        let capabilities = json!({
9255            "provides": ["credentials-provider/v1"],
9256            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9257            "must_never_reach": ["federation-transport/v1"]
9258        });
9259        let response = handler
9260            .handle_control(
9261                ConnectionId::new(99),
9262                capability_grammar_hello_frame(
9263                    capabilities.clone(),
9264                    Some(json!(["/roles/0/tools"])),
9265                    99,
9266                ),
9267            )
9268            .expect("valid HELLO registers");
9269        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9270
9271        let request = Frame::build(
9272            FrameType::Request,
9273            control_flags(),
9274            0,
9275            0,
9276            100,
9277            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9278                .expect("catalog request serializes"),
9279        )
9280        .expect("catalog request frame builds");
9281        let response = handler
9282            .handle_catalog_list(request, None)
9283            .expect("catalog list succeeds");
9284        let ClientControlResponse::CatalogList { modules, .. } =
9285            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9286        else {
9287            panic!("catalog request must return catalog.list");
9288        };
9289        assert_eq!(modules.len(), 1);
9290        assert_eq!(
9291            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9292            capabilities
9293        );
9294    }
9295
9296    #[test]
9297    fn catalog_list_mirrors_management_operation_description() {
9298        let registry = Arc::new(Registry::default());
9299        let handler = ControlHandler::new(Arc::clone(&registry));
9300        let description = "List managed records and return their identifiers and metadata.";
9301        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9302        manifest.provides = vec![ProviderRole::ManagementSurface {
9303            operations: vec![ManagementOperation {
9304                name: "records.list".to_string(),
9305                kind: ManagementOperationKind::Query,
9306                description: Some(description.to_string()),
9307            }],
9308            config_schema: json!({"type": "object"}),
9309            observability: vec![ObservabilitySurface {
9310                name: "records.stats".to_string(),
9311                kind: ObservabilityKind::Snapshot,
9312            }],
9313            identity_scope: vec![IdentityScope::Project],
9314            concurrency: Concurrency::ModuleManaged,
9315        }];
9316        registry
9317            .register_with_control_ops(
9318                manifest,
9319                PROTOCOL_VERSION,
9320                ConnectionId::new(99),
9321                Vec::new(),
9322            )
9323            .expect("described management manifest registers");
9324
9325        let request = Frame::build(
9326            FrameType::Request,
9327            control_flags(),
9328            0,
9329            0,
9330            100,
9331            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9332                .expect("catalog request serializes"),
9333        )
9334        .expect("catalog request frame builds");
9335        let response = handler
9336            .handle_catalog_list(request, None)
9337            .expect("catalog list succeeds");
9338        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9339        assert_eq!(
9340            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9341            "catalog.list must preserve the declared operation description verbatim"
9342        );
9343    }
9344
9345    #[test]
9346    fn catalog_list_mirrors_registered_operation_delivery() {
9347        let registry = Arc::new(Registry::default());
9348        let handler = ControlHandler::new(Arc::clone(&registry));
9349        let delivery =
9350            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9351        let mut body = serde_json::to_value(ModuleHelloBody {
9352            manifest: manifest("watch-management", PROTOCOL_VERSION),
9353            protocol_ver: PROTOCOL_VERSION,
9354            control_ops: None,
9355            launch_nonce: None,
9356        })
9357        .unwrap();
9358        body["manifest"]["provides"] = json!([{
9359            "role": "management_surface",
9360            "operations": [
9361                {"name": "status", "kind": "query"},
9362                {"name": "future", "kind": "query"},
9363                {"name": "reset", "kind": "mutate"}
9364            ],
9365            "config_schema": {}, "observability": [], "identity_scope": []
9366        }]);
9367        body["manifest"]["operation_delivery"] = delivery.clone();
9368        let hello = Frame::build(
9369            FrameType::Hello,
9370            control_flags(),
9371            0,
9372            0,
9373            99,
9374            serde_json::to_vec(&body).unwrap(),
9375        )
9376        .unwrap();
9377        let response = handler
9378            .handle_control(ConnectionId::new(99), hello)
9379            .unwrap();
9380        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9381
9382        let (_, registrations) = registry.list_modules().unwrap();
9383        let registered = &registrations[0].manifest;
9384        assert!(registered.is_watch_operation("status"));
9385        assert!(!registered.is_watch_operation("future"));
9386        assert!(!registered.is_watch_operation("reset"));
9387        assert!(!registered.is_watch_operation("missing"));
9388
9389        let request = Frame::build(
9390            FrameType::Request,
9391            control_flags(),
9392            0,
9393            0,
9394            100,
9395            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9396        )
9397        .unwrap();
9398        let response = handler.handle_catalog_list(request, None).unwrap();
9399        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9400        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9401        let ClientControlResponse::CatalogList { modules, .. } =
9402            serde_json::from_slice(&response[0].body).unwrap()
9403        else {
9404            panic!("expected catalog.list");
9405        };
9406        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9407    }
9408
9409    #[test]
9410    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9411        let registry = Arc::new(Registry::default());
9412        let supervisor = SupervisorHandle::new();
9413        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9414        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9415        let descriptor = json!({
9416            "schema": "ckios.richui.manifest/1",
9417            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9418            "bytes": 4096,
9419            "chunks": 4
9420        });
9421        let connection = ConnectionId::new(201);
9422        let hello = handler
9423            .handle_control(
9424                connection,
9425                hello_frame_with_rich_ui(
9426                    "vault",
9427                    PROTOCOL_VERSION,
9428                    201,
9429                    Some("the-real-nonce"),
9430                    descriptor.clone(),
9431                ),
9432            )
9433            .unwrap();
9434        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9435
9436        let request =
9437            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9438        let provides = registry
9439            .get_module_by_connection(connection)
9440            .unwrap()
9441            .unwrap()
9442            .manifest
9443            .provides;
9444        handler
9445            .handle_catalog_update(connection, request, provides, None, Some(false))
9446            .unwrap();
9447
9448        let request = Frame::build(
9449            FrameType::Request,
9450            control_flags(),
9451            0,
9452            0,
9453            203,
9454            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9455        )
9456        .unwrap();
9457        let response = handler.handle_catalog_list(request, None).unwrap();
9458        let ClientControlResponse::CatalogList { modules, .. } =
9459            serde_json::from_slice(&response[0].body).unwrap()
9460        else {
9461            panic!("expected catalog.list");
9462        };
9463        assert_eq!(
9464            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9465            descriptor
9466        );
9467        assert!(modules[0].nonce_verified);
9468    }
9469
9470    fn hello_with_role_declarations(declarations: Value) -> Frame {
9471        let mut body = serde_json::to_value(ModuleHelloBody {
9472            manifest: manifest("role-test", PROTOCOL_VERSION),
9473            protocol_ver: PROTOCOL_VERSION,
9474            control_ops: None,
9475            launch_nonce: None,
9476        })
9477        .unwrap();
9478        body["manifest"]["role_declarations"] = declarations;
9479        Frame::build(
9480            FrameType::Hello,
9481            control_flags(),
9482            0,
9483            0,
9484            301,
9485            serde_json::to_vec(&body).unwrap(),
9486        )
9487        .unwrap()
9488    }
9489
9490    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9491        let request = Frame::build(
9492            FrameType::Request,
9493            control_flags(),
9494            0,
9495            0,
9496            302,
9497            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9498        )
9499        .unwrap();
9500        let response = handler.handle_catalog_list(request, None).unwrap();
9501        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9502        let ClientControlResponse::CatalogList { mut modules, .. } =
9503            serde_json::from_slice(&response[0].body).unwrap()
9504        else {
9505            panic!("expected catalog.list");
9506        };
9507        (raw["modules"][0].clone(), modules.remove(0))
9508    }
9509
9510    #[test]
9511    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9512        let registry = Arc::new(Registry::default());
9513        let handler = ControlHandler::new(Arc::clone(&registry));
9514        let declarations = json!({
9515            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9516            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9517        });
9518        let connection = ConnectionId::new(301);
9519        let response = handler
9520            .handle_control(
9521                connection,
9522                hello_with_role_declarations(declarations.clone()),
9523            )
9524            .unwrap();
9525        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9526        let registered = registry
9527            .get_module_by_connection(connection)
9528            .unwrap()
9529            .unwrap()
9530            .manifest;
9531        // Compared as JSON values, not bytes: object key order is not part of
9532        // the relay's promise, and workspace builds unify serde_json's
9533        // preserve_order feature, which changes the order a literal keeps.
9534        assert_eq!(
9535            serde_json::to_value(&registered.role_declarations).unwrap(),
9536            declarations
9537        );
9538        let (raw, entry) = role_declarations_catalog(&handler);
9539        assert_eq!(raw["role_declarations"], declarations);
9540        assert_eq!(entry.role_declarations, registered.role_declarations);
9541
9542        let request =
9543            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9544        handler
9545            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9546            .unwrap();
9547        let (raw, entry) = role_declarations_catalog(&handler);
9548        assert_eq!(raw["role_declarations"], declarations);
9549        assert_eq!(entry.role_declarations, registered.role_declarations);
9550    }
9551
9552    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9553        let registry = Arc::new(Registry::default());
9554        let handler = ControlHandler::new(Arc::clone(&registry));
9555        let capture = EventCapture::default();
9556        let _subscriber =
9557            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9558        let response = handler
9559            .handle_control(
9560                ConnectionId::new(301),
9561                hello_with_role_declarations(declarations),
9562            )
9563            .unwrap();
9564        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9565        assert!(registry
9566            .get_module_by_connection(ConnectionId::new(301))
9567            .unwrap()
9568            .unwrap()
9569            .manifest
9570            .role_declarations
9571            .is_empty());
9572        let (raw, entry) = role_declarations_catalog(&handler);
9573        assert!(raw.get("role_declarations").is_none());
9574        assert!(entry.role_declarations.is_empty());
9575        let warnings: Vec<_> = capture
9576            .events()
9577            .into_iter()
9578            .filter(|event| {
9579                event.fields.get("message").map(String::as_str)
9580                    == Some("dropping whole role_declarations map from HELLO")
9581            })
9582            .collect();
9583        assert_eq!(warnings.len(), 1);
9584        assert_eq!(warnings[0].level, tracing::Level::WARN);
9585        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9586        assert_eq!(warnings[0].fields["reason"], reason);
9587    }
9588
9589    #[test]
9590    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9591        for value in [
9592            json!(null),
9593            json!(false),
9594            json!(0),
9595            json!("opaque"),
9596            json!([]),
9597        ] {
9598            assert_hello_drops_role_declarations_with_warning(
9599                json!({"routing/v1": {}, "broken/v1": value}),
9600                "role_declarations values must be objects",
9601            );
9602        }
9603        assert_hello_drops_role_declarations_with_warning(
9604            json!([]),
9605            "role_declarations must be an object",
9606        );
9607    }
9608
9609    #[test]
9610    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9611        assert_hello_drops_role_declarations_with_warning(
9612            json!({"routing/v1": {}, "bad key": {}}),
9613            "role_declarations key must use capability-name grammar",
9614        );
9615    }
9616
9617    #[test]
9618    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9619        assert_hello_drops_role_declarations_with_warning(
9620            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9621            "role_declarations exceeds 16384 compact JSON bytes",
9622        );
9623    }
9624
9625    #[test]
9626    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9627        let registry = Arc::new(Registry::default());
9628        let handler = ControlHandler::new(registry.clone());
9629        let descriptor = json!({
9630            "schema": "ckios.richui.manifest/1",
9631            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9632            "bytes": 128,
9633            "chunks": 2
9634        });
9635        let malformed = [
9636            ("schema", json!("")),
9637            ("schema", json!("a".repeat(65))),
9638            ("schema", json!("Bad schema")),
9639            (
9640                "manifest_digest",
9641                json!(format!("sha256:{}", "A".repeat(64))),
9642            ),
9643            ("manifest_digest", json!("sha256:abc")),
9644            ("bytes", json!(0)),
9645            ("bytes", json!(16_777_217)),
9646            ("bytes", json!("128")),
9647            ("chunks", json!(0)),
9648            ("chunks", json!(1025)),
9649            ("chunks", json!("2")),
9650        ];
9651        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9652            let mut invalid = descriptor.clone();
9653            invalid[field] = invalid_value;
9654            let response = handler
9655                .handle_control(
9656                    ConnectionId::new(210 + index as u64),
9657                    hello_frame_with_rich_ui(
9658                        &format!("bad-rich-ui-{index}"),
9659                        PROTOCOL_VERSION,
9660                        210 + index as u64,
9661                        None,
9662                        invalid,
9663                    ),
9664                )
9665                .unwrap();
9666            assert_eq!(response[0].header.ty, FrameType::Error);
9667            let error = parse_error(&response[0]);
9668            assert_eq!(error["code"], "invalid_manifest");
9669            assert!(error["message"]
9670                .as_str()
9671                .unwrap()
9672                .contains(&format!("rich_ui.{field}")));
9673            assert!(registry
9674                .get_module(&format!("bad-rich-ui-{index}"))
9675                .unwrap()
9676                .is_none());
9677        }
9678    }
9679
9680    #[test]
9681    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9682        let registry = Arc::new(Registry::default());
9683        let handler = ControlHandler::new(Arc::clone(&registry));
9684        let connection = ConnectionId::new(219);
9685        registry
9686            .register_with_control_ops(
9687                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9688                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9689                        "bad schema",
9690                        format!("sha256:{}", "a".repeat(64)),
9691                        128,
9692                        2,
9693                    )))
9694                    .build(),
9695                PROTOCOL_VERSION,
9696                connection,
9697                Vec::new(),
9698            )
9699            .unwrap();
9700        let request =
9701            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9702        let response = handler
9703            .handle_catalog_update(connection, request, Vec::new(), None, None)
9704            .unwrap();
9705        assert_eq!(response[0].header.ty, FrameType::Error);
9706        let error = parse_error(&response[0]);
9707        assert_eq!(error["code"], "invalid_manifest");
9708        assert!(error["message"]
9709            .as_str()
9710            .unwrap()
9711            .contains("rich_ui.schema"));
9712    }
9713
9714    #[test]
9715    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9716        let registry = Arc::new(Registry::default());
9717        let supervisor = SupervisorHandle::new();
9718        supervisor.open_swap("provider", "candidate-nonce".to_string());
9719        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9720        let connection = ConnectionId::new(218);
9721        let hello = handler
9722            .handle_control(
9723                connection,
9724                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9725            )
9726            .unwrap();
9727        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9728        assert!(registry.list_modules().unwrap().1.is_empty());
9729
9730        registry.promote_candidate("provider").unwrap().unwrap();
9731        let request = Frame::build(
9732            FrameType::Request,
9733            control_flags(),
9734            0,
9735            0,
9736            219,
9737            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9738        )
9739        .unwrap();
9740        let response = handler.handle_catalog_list(request, None).unwrap();
9741        let ClientControlResponse::CatalogList { modules, .. } =
9742            serde_json::from_slice(&response[0].body).unwrap()
9743        else {
9744            panic!("expected catalog.list");
9745        };
9746        assert_eq!(modules.len(), 1);
9747        assert!(modules[0].nonce_verified);
9748    }
9749
9750    #[test]
9751    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9752        let registry = Arc::new(Registry::default());
9753        let handler = ControlHandler::new(Arc::clone(&registry));
9754        let hello = Frame::build(
9755            FrameType::Hello,
9756            control_flags(),
9757            0,
9758            0,
9759            220,
9760            serde_json::to_vec(&ModuleHelloBody {
9761                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9762                protocol_ver: PROTOCOL_VERSION,
9763                control_ops: None,
9764                launch_nonce: Some("unverified".to_string()),
9765            })
9766            .unwrap(),
9767        )
9768        .unwrap();
9769        assert_eq!(
9770            handler
9771                .handle_control(ConnectionId::new(220), hello)
9772                .unwrap()[0]
9773                .header
9774                .ty,
9775            FrameType::HelloAck
9776        );
9777        let request = Frame::build(
9778            FrameType::Request,
9779            control_flags(),
9780            0,
9781            0,
9782            221,
9783            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9784        )
9785        .unwrap();
9786        let response = handler.handle_catalog_list(request, None).unwrap();
9787        let ClientControlResponse::CatalogList { modules, .. } =
9788            serde_json::from_slice(&response[0].body).unwrap()
9789        else {
9790            panic!("expected catalog.list");
9791        };
9792        assert_eq!(
9793            serde_json::to_vec(&modules[0]).unwrap(),
9794            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9795        );
9796        assert!(!modules[0].nonce_verified);
9797    }
9798
9799    #[test]
9800    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9801        let registry = Arc::new(Registry::default());
9802        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9803            [("vault".to_string(), true), ("squatter".to_string(), true)],
9804            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9805        );
9806        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9807        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9808            provides: vec!["credentials-provider/v1".to_string()],
9809            requires: Vec::new(),
9810            must_never_reach: Vec::new(),
9811        });
9812        let frame = Frame::build(
9813            FrameType::Hello,
9814            control_flags(),
9815            0,
9816            0,
9817            77,
9818            serde_json::to_vec(&ModuleHelloBody {
9819                manifest: squatter,
9820                protocol_ver: PROTOCOL_VERSION,
9821                control_ops: None,
9822                launch_nonce: None,
9823            })
9824            .expect("HELLO serializes"),
9825        )
9826        .expect("HELLO frame builds");
9827        let response = handler
9828            .handle_control(ConnectionId::new(77), frame)
9829            .expect("reserved claim receives a typed refusal");
9830        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9831        assert_eq!(
9832            registry
9833                .active_registration_count()
9834                .expect("registry reads"),
9835            0,
9836            "a reserved capability refusal must not leave a catalog entry"
9837        );
9838    }
9839
9840    #[test]
9841    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9842        for settlement in ["timeout", "inconclusive", "drop"] {
9843            let breakers = RouteBindBreakers::default();
9844            let RouteBindAdmission::Admitted {
9845                guard: mut old,
9846                probe: false,
9847            } = breakers.admit("prov")
9848            else {
9849                panic!("ordinary relay admitted")
9850            };
9851            let RouteBindAdmission::Admitted {
9852                guard: mut opener, ..
9853            } = breakers.admit("prov")
9854            else {
9855                panic!("second relay admitted")
9856            };
9857            assert!(
9858                !opener
9859                    .record_timeout(1, Duration::ZERO)
9860                    .unwrap()
9861                    .reopened_after_probe
9862            );
9863            let RouteBindAdmission::Admitted {
9864                guard: mut probe,
9865                probe: true,
9866            } = breakers.admit("prov")
9867            else {
9868                panic!("one cooldown probe admitted")
9869            };
9870            match settlement {
9871                "timeout" => assert!(
9872                    !old.record_timeout(1, Duration::ZERO)
9873                        .unwrap()
9874                        .reopened_after_probe
9875                ),
9876                "inconclusive" => old.record_inconclusive(),
9877                "drop" => drop(old),
9878                _ => unreachable!(),
9879            }
9880            assert!(
9881                matches!(
9882                    breakers.admit("prov"),
9883                    RouteBindAdmission::Refused {
9884                        probe_in_flight: true,
9885                        ..
9886                    }
9887                ),
9888                "{settlement} of a pre-open relay cannot release the real probe"
9889            );
9890            assert!(
9891                probe
9892                    .record_timeout(1, Duration::ZERO)
9893                    .unwrap()
9894                    .reopened_after_probe
9895            );
9896            assert!(matches!(
9897                breakers.admit("prov"),
9898                RouteBindAdmission::Admitted { probe: true, .. }
9899            ));
9900        }
9901        let breakers = RouteBindBreakers::default();
9902        let admit = || match breakers.admit("prov") {
9903            RouteBindAdmission::Admitted { guard, .. } => guard,
9904            _ => panic!("relay admitted"),
9905        };
9906        admit().record_timeout(1, Duration::ZERO);
9907        let mut old_probe = admit();
9908        breakers.reset_for_new_module_connection("prov");
9909        admit().record_timeout(1, Duration::ZERO);
9910        let _new_probe = admit();
9911        old_probe.record_inconclusive();
9912        assert!(matches!(
9913            breakers.admit("prov"),
9914            RouteBindAdmission::Refused {
9915                probe_in_flight: true,
9916                ..
9917            }
9918        ));
9919    }
9920
9921    #[tokio::test]
9922    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9923        for candidate in [false, true] {
9924            let registry = Arc::new(Registry::default());
9925            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9926                [("vault".to_string(), true), ("squatter".to_string(), true)],
9927                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9928            );
9929            let conn = ConnectionId::new(77);
9930            let (ctx, mut rx) = route_ctx(conn);
9931            let initial = capability_manifest("squatter", &[], &[]);
9932            if candidate {
9933                registry
9934                    .register_candidate_with_control_ops(
9935                        initial.clone(),
9936                        PROTOCOL_VERSION,
9937                        conn,
9938                        module_baseline_control_ops(),
9939                    )
9940                    .unwrap();
9941                handler
9942                    .forwarding
9943                    .register_candidate_module_connection(
9944                        conn,
9945                        "squatter".to_string(),
9946                        PROTOCOL_VERSION,
9947                        manifest_concurrency(&initial),
9948                        ctx.egress.clone(),
9949                    )
9950                    .unwrap();
9951            } else {
9952                hello_via_sink(
9953                    &handler,
9954                    &ctx,
9955                    &mut rx,
9956                    hello_frame_with_manifest(initial.clone(), 1),
9957                )
9958                .await;
9959            }
9960            let response = handler
9961                .handle_control_frame(
9962                    &ctx,
9963                    catalog_update_with_capabilities_frame(
9964                        2,
9965                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
9966                            .capabilities
9967                            .unwrap(),
9968                    ),
9969                )
9970                .await
9971                .unwrap();
9972            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9973            assert_eq!(
9974                registry
9975                    .get_module_by_connection(conn)
9976                    .unwrap()
9977                    .unwrap()
9978                    .manifest,
9979                initial
9980            );
9981        }
9982    }
9983
9984    #[test]
9985    fn server_describe_surfaces_required_capability_verdict_fields() {
9986        let registry = Arc::new(Registry::default());
9987        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9988            [
9989                ("consumer".to_string(), true),
9990                ("provider".to_string(), false),
9991            ],
9992            BTreeMap::new(),
9993        );
9994        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9995        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9996            provides: Vec::new(),
9997            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9998                capability: "credentials-provider/v1".to_string(),
9999                need: subc_protocol::manifest::CapabilityNeed::Required,
10000            }],
10001            must_never_reach: Vec::new(),
10002        });
10003        let hello = Frame::build(
10004            FrameType::Hello,
10005            control_flags(),
10006            0,
10007            0,
10008            78,
10009            serde_json::to_vec(&ModuleHelloBody {
10010                manifest: consumer,
10011                protocol_ver: PROTOCOL_VERSION,
10012                control_ops: None,
10013                launch_nonce: None,
10014            })
10015            .expect("HELLO serializes"),
10016        )
10017        .expect("HELLO frame builds");
10018        handler
10019            .handle_control(ConnectionId::new(78), hello)
10020            .expect("consumer registers");
10021        let describe = Frame::build(
10022            FrameType::Request,
10023            control_flags(),
10024            0,
10025            0,
10026            79,
10027            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10028                .expect("request serializes"),
10029        )
10030        .expect("describe frame builds");
10031        let response = handler
10032            .handle_server_describe(describe)
10033            .expect("server.describe succeeds");
10034        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10035        let requirement = &rendered["capability_requirements"][0];
10036        assert_eq!(requirement["consumer"], "consumer");
10037        assert_eq!(requirement["verdict"], "never_provided");
10038        assert_eq!(requirement["episode_seq"], 1);
10039        assert_eq!(requirement["config_satisfiable"], false);
10040        assert_eq!(requirement["runtime_available"], false);
10041        assert!(requirement["detail"]
10042            .as_str()
10043            .expect("detail string")
10044            .contains("credentials-provider/v1"));
10045    }
10046
10047    /// Register a consumer that requires `project-identity/v1` and the provider
10048    /// of it, optionally mark daemon shutdown as begun, then deregister the
10049    /// provider as its connection closes. Returns the `capability.requirement`
10050    /// events that deregistration logged and the verdict the evaluator then
10051    /// holds for the requirement (the one `route.open` refusals read).
10052    fn deregister_required_provider(
10053        daemon_shutdown_begun: bool,
10054    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10055        const CAPABILITY: &str = "project-identity/v1";
10056        let registry = Arc::new(Registry::default());
10057        let daemon_shutdown = DaemonShutdownFlag::default();
10058        let handler = ControlHandler::new(Arc::clone(&registry))
10059            .with_capability_config(
10060                [
10061                    ("consumer".to_string(), true),
10062                    ("provider".to_string(), true),
10063                ],
10064                BTreeMap::new(),
10065            )
10066            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10067        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10068        consumer.capabilities = Some(CapabilityDeclarations {
10069            provides: Vec::new(),
10070            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10071                capability: CAPABILITY.to_string(),
10072                need: subc_protocol::manifest::CapabilityNeed::Required,
10073            }],
10074            must_never_reach: Vec::new(),
10075        });
10076        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10077        let provider_conn = ConnectionId::new(182);
10078        for (conn, manifest, corr) in [
10079            (ConnectionId::new(181), consumer, 181),
10080            (provider_conn, provider, 182),
10081        ] {
10082            let hello = handler
10083                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10084                .expect("module registers");
10085            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10086        }
10087        assert_eq!(
10088            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10089            Some(CapabilityVerdict::Provided),
10090            "the registered provider must satisfy the requirement first"
10091        );
10092        if daemon_shutdown_begun {
10093            daemon_shutdown.set_for_test();
10094        }
10095
10096        let capture = EventCapture::default();
10097        let _subscriber =
10098            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10099        handler
10100            .cleanup_connection(provider_conn)
10101            .expect("provider deregisters");
10102        let events = capture
10103            .events()
10104            .into_iter()
10105            .filter(|event| {
10106                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10107            })
10108            .collect();
10109        (
10110            events,
10111            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10112        )
10113    }
10114
10115    #[test]
10116    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10117        let (events, verdict) = deregister_required_provider(true);
10118        assert_eq!(
10119            verdict,
10120            Some(CapabilityVerdict::NeverProvided),
10121            "shutdown must not change the verdict route.open refusals read"
10122        );
10123        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10124        let event = &events[0];
10125        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10126        assert_eq!(
10127            event.fields.get("cause").map(String::as_str),
10128            Some("\"daemon_shutdown\""),
10129            "{event:?}"
10130        );
10131        assert_eq!(
10132            event.fields.get("verdict").map(String::as_str),
10133            Some("\"never_provided\""),
10134            "{event:?}"
10135        );
10136    }
10137
10138    #[test]
10139    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10140        let (events, verdict) = deregister_required_provider(false);
10141        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10142        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10143        let event = &events[0];
10144        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10145        assert_eq!(event.fields.get("cause"), None, "{event:?}");
10146        assert_eq!(
10147            event.fields.get("verdict").map(String::as_str),
10148            Some("\"never_provided\""),
10149            "{event:?}"
10150        );
10151    }
10152
10153    #[test]
10154    fn catalog_list_omits_capabilities_for_legacy_manifest() {
10155        let registry = Arc::new(Registry::default());
10156        let handler = ControlHandler::new(Arc::clone(&registry));
10157        let hello = handler
10158            .handle_control(
10159                ConnectionId::new(101),
10160                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10161            )
10162            .expect("legacy HELLO registers");
10163        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10164
10165        let request = Frame::build(
10166            FrameType::Request,
10167            control_flags(),
10168            0,
10169            0,
10170            102,
10171            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10172                .expect("catalog request serializes"),
10173        )
10174        .expect("catalog request frame builds");
10175        let response = handler
10176            .handle_catalog_list(request, None)
10177            .expect("catalog list succeeds");
10178        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10179        assert!(
10180            body["modules"][0].get("capabilities").is_none(),
10181            "legacy manifest must retain an absent capabilities field on catalog.list"
10182        );
10183    }
10184
10185    #[test]
10186    fn hello_ack_omits_storage_when_no_storage_config() {
10187        let registry = Arc::new(Registry::default());
10188        let handler = ControlHandler::new(Arc::clone(&registry));
10189        let responses = handler
10190            .handle_control(
10191                ConnectionId::new(1),
10192                hello_frame("aft", PROTOCOL_VERSION, 7),
10193            )
10194            .unwrap();
10195        let ack = parse_ack(&responses[0]);
10196        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10197        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10198    }
10199
10200    #[tokio::test]
10201    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10202        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10203        let registry = Arc::new(Registry::default());
10204        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
10205        let responses = handler
10206            .handle_control(
10207                ConnectionId::new(1),
10208                hello_frame("aft", PROTOCOL_VERSION, 7),
10209            )
10210            .unwrap();
10211        let ack = parse_ack(&responses[0]);
10212        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10213
10214        let described = handler
10215            .handle_control_frame(
10216                &route_ctx(ConnectionId::new(2)).0,
10217                Frame::build(
10218                    FrameType::Request,
10219                    control_flags(),
10220                    0,
10221                    0,
10222                    9,
10223                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10224                )
10225                .unwrap(),
10226            )
10227            .await
10228            .unwrap();
10229        let ClientControlResponse::ServerDescribe { machine_id, .. } =
10230            serde_json::from_slice(&described[0].body).unwrap()
10231        else {
10232            panic!("server.describe answered with another shape");
10233        };
10234        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10235    }
10236
10237    #[test]
10238    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10239        // With a central sqlite storage policy, each registering module gets its
10240        // own resolved descriptor in HELLO_ACK, keyed by its module id.
10241        let registry = Arc::new(Registry::default());
10242        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
10243            crate::daemon_config::StorageConfig::Sqlite {
10244                data_home: std::path::PathBuf::from("/data"),
10245            },
10246        ));
10247
10248        let responses = handler
10249            .handle_control(
10250                ConnectionId::new(1),
10251                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10252            )
10253            .unwrap();
10254        let ack = parse_ack(&responses[0]);
10255        assert_eq!(
10256            ack.storage,
10257            Some(serde_json::json!({
10258                "module_id": "alfonso-routing",
10259                "storage_namespace": "default",
10260                "isolation": { "kind": "module" },
10261                "backend": {
10262                    "backend": "sqlite",
10263                    "path": "/data/cortexkit/alfonso-routing/store.db"
10264                }
10265            })),
10266            "the delivered descriptor is the module's own sqlite store path"
10267        );
10268    }
10269
10270    #[test]
10271    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10272        let registry = Arc::new(Registry::default());
10273        let handler = ControlHandler::new(Arc::clone(&registry));
10274        let conn = ConnectionId::new(1);
10275        let responses = handler
10276            .handle_control(
10277                conn,
10278                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10279            )
10280            .unwrap();
10281        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10282        let registration = registry.get_module("aft").unwrap().unwrap();
10283        assert_eq!(registration.control_ops, module_baseline_control_ops());
10284
10285        let frame =
10286            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10287        assert!(handler
10288            .guard_module_control_op(&frame, "aft", "route.bind")
10289            .unwrap()
10290            .is_none());
10291        let error = handler
10292            .guard_module_control_op(&frame, "aft", "test.synthetic")
10293            .unwrap()
10294            .expect("synthetic ungranted op should be rejected");
10295        assert_eq!(error.header.ty, FrameType::Error);
10296        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10297    }
10298
10299    #[test]
10300    fn hello_control_ops_some_adds_optional_grants() {
10301        let registry = Arc::new(Registry::default());
10302        let handler = ControlHandler::new(Arc::clone(&registry));
10303        handler
10304            .handle_control(
10305                ConnectionId::new(1),
10306                hello_frame_with_control_ops(
10307                    "aft",
10308                    PROTOCOL_VERSION,
10309                    7,
10310                    Some(vec![
10311                        "future.synthetic".to_string(),
10312                        "route.bind".to_string(),
10313                    ]),
10314                ),
10315            )
10316            .unwrap();
10317        let registration = registry.get_module("aft").unwrap().unwrap();
10318        assert_eq!(
10319            registration.control_ops,
10320            vec![
10321                "route.bind".to_string(),
10322                "route.status".to_string(),
10323                "future.synthetic".to_string(),
10324            ]
10325        );
10326        let frame =
10327            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10328        assert!(handler
10329            .guard_module_control_op(&frame, "aft", "future.synthetic")
10330            .unwrap()
10331            .is_none());
10332    }
10333
10334    #[tokio::test]
10335    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10336        let registry = Arc::new(Registry::default());
10337        let forwarding = Arc::new(ForwardingTable::default());
10338        let handler =
10339            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10340        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10341        hello_via_sink(
10342            &handler,
10343            &module_ctx,
10344            &mut module_rx,
10345            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10346        )
10347        .await;
10348
10349        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10350        let responses = handler
10351            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10352            .await
10353            .unwrap();
10354        assert_eq!(responses.len(), 1);
10355        assert_eq!(responses[0].header.ty, FrameType::Error);
10356        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10357        assert!(module_rx.try_recv().is_err());
10358    }
10359
10360    #[tokio::test]
10361    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10362        let registry = Arc::new(Registry::default());
10363        let forwarding = Arc::new(ForwardingTable::default());
10364        let handler =
10365            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10366        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10367        hello_via_sink(
10368            &handler,
10369            &module_ctx,
10370            &mut module_rx,
10371            hello_frame_with_control_ops(
10372                "aft",
10373                PROTOCOL_VERSION,
10374                7,
10375                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10376            ),
10377        )
10378        .await;
10379
10380        let project_root = unique_project_root("demux");
10381        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10382        let route_handler = handler.clone();
10383        let route_task = tokio::spawn(async move {
10384            route_handler
10385                .handle_control_frame(
10386                    &route_client_ctx,
10387                    route_open_frame(100, "aft", project_root),
10388                )
10389                .await
10390                .unwrap()
10391        });
10392        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10393            .await
10394            .unwrap()
10395            .unwrap();
10396        assert!(matches!(
10397            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10398            ModuleControlRequest::RouteBind { .. }
10399        ));
10400
10401        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10402        let health_handler = handler.clone();
10403        let health_task = tokio::spawn(async move {
10404            health_handler
10405                .handle_control_frame(
10406                    &health_client_ctx,
10407                    supervisor_health_probe_frame(101, "aft"),
10408                )
10409                .await
10410                .unwrap()
10411        });
10412        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10413            .await
10414            .unwrap()
10415            .unwrap();
10416        assert_eq!(
10417            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10418            ModuleControlRequest::HealthCheck {}
10419        );
10420
10421        handler
10422            .handle_control_frame(
10423                &module_ctx,
10424                health_response(health_frame.header.corr, HealthStatus::Degraded),
10425            )
10426            .await
10427            .unwrap();
10428        let health_response = health_task.await.unwrap();
10429        assert_eq!(health_response.len(), 1);
10430        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10431            ClientControlResponse::SupervisorHealthProbe {
10432                module_id,
10433                status,
10434                detail,
10435                metrics,
10436            } => {
10437                assert_eq!(module_id, "aft");
10438                assert_eq!(status, HealthStatus::Degraded);
10439                assert_eq!(detail.as_deref(), Some("warming"));
10440                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10441            }
10442            other => panic!("unexpected health response: {other:?}"),
10443        }
10444
10445        handler
10446            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10447            .await
10448            .unwrap();
10449        let route_response = route_task.await.unwrap();
10450        assert!(route_response.is_empty());
10451        let published = route_client_rx.recv().await.unwrap();
10452        assert!(matches!(
10453            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10454            ClientControlResponse::RouteOpen { .. }
10455        ));
10456    }
10457
10458    /// Start one `route.open` on `client_connection` and return its still-running
10459    /// handler task together with the `route.bind` the module received for it.
10460    /// The handler blocks until the module answers, so it has to run as a task
10461    /// while the test drives the module side.
10462    async fn relay_route_open(
10463        handler: &ControlHandler,
10464        client_connection: ConnectionId,
10465        client_egress: &FrameSink,
10466        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10467        corr: u64,
10468        module_id: &str,
10469        project_root_label: &str,
10470    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10471        let ctx = RouteCtx {
10472            connection_id: client_connection,
10473            egress: client_egress.clone(),
10474        };
10475        let handler = handler.clone();
10476        let project_root = unique_project_root(project_root_label);
10477        let module_id = module_id.to_string();
10478        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10479        let task = tokio::spawn(async move {
10480            let _guard = tracing::dispatcher::set_default(&dispatch);
10481            handler
10482                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10483                .await
10484                .unwrap()
10485        });
10486        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10487            .await
10488            .expect("module receives the relayed route.bind")
10489            .expect("module egress is open");
10490        (task, bind.frame)
10491    }
10492
10493    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10494        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10495            ModuleControlRequest::RouteBind {
10496                route_channel,
10497                epoch,
10498                ..
10499            } => (route_channel, epoch),
10500            other => panic!("expected a route.bind request, got {other:?}"),
10501        }
10502    }
10503
10504    fn published_route(frame: &Frame) -> (u16, u32) {
10505        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10506            ClientControlResponse::RouteOpen {
10507                route_channel,
10508                route_epoch,
10509            } => (route_channel, route_epoch),
10510            other => panic!("expected a route.open response, got {other:?}"),
10511        }
10512    }
10513
10514    /// Reproduction of a production outage. A client had `route.open`s in
10515    /// flight to a module and was already marked closing -- its egress had refused a
10516    /// module frame, so the daemon asked its connection to end -- while its sink
10517    /// was still open. When the module acked those binds, the daemon refused to
10518    /// commit a route for a closing client, and that refusal was returned from
10519    /// the MODULE connection's frame handler, where a router error that has no
10520    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
10521    /// the supervisor correctly did not respawn a clean exit, and every seat lost
10522    /// its tools for hours -- one client's teardown took down a connection
10523    /// carrying ~170 other routes.
10524    ///
10525    /// The window is opened here by calling the production path that opens it
10526    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
10527    /// state that matters is "in `closing_connections`, sink still open, relay
10528    /// still pending", and it lasts only from the close request until the
10529    /// connection loop reacts to it; a socket-level test can flood a client into
10530    /// that escalation but cannot pin the module's ack inside the window. Closing
10531    /// the socket instead takes the other path entirely -- connection teardown
10532    /// removes the pending relay under the same lock, so the ack finds nothing.
10533    #[tokio::test]
10534    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10535        let registry = Arc::new(Registry::default());
10536        let forwarding = Arc::new(ForwardingTable::default());
10537        let handler =
10538            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10539
10540        let module_connection = ConnectionId::new(30);
10541        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10542        hello_via_sink(
10543            &handler,
10544            &module_ctx,
10545            &mut module_rx,
10546            hello_frame("aft", PROTOCOL_VERSION, 7),
10547        )
10548        .await;
10549
10550        let dying_client = ConnectionId::new(31);
10551        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10552
10553        // A published route on the dying client. The escalation below only marks
10554        // a connection closing for a route it has already published.
10555        let (first_task, first_bind) = relay_route_open(
10556            &handler,
10557            dying_client,
10558            &dying_ctx.egress,
10559            &mut module_rx,
10560            100,
10561            "aft",
10562            "closing-first",
10563        )
10564        .await;
10565        handler
10566            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10567            .await
10568            .unwrap();
10569        assert!(first_task.await.unwrap().is_empty());
10570        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10571
10572        // A second route.open from the same client, relayed and awaiting its ack.
10573        let (second_task, second_bind) = relay_route_open(
10574            &handler,
10575            dying_client,
10576            &dying_ctx.egress,
10577            &mut module_rx,
10578            101,
10579            "aft",
10580            "closing-second",
10581        )
10582        .await;
10583        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10584
10585        // The window: the client is closing, its sink is still open, and its
10586        // second bind is still pending.
10587        assert!(forwarding
10588            .escalate_client_delivery_failure(
10589                dying_client,
10590                first_channel,
10591                first_epoch,
10592                CloseReason::new(
10593                    "module_to_client_delivery_failed",
10594                    "client egress refused a module frame",
10595                ),
10596                crate::forwarding::UndeliveredFrame {
10597                    module_id: None,
10598                    sink: &dying_ctx.egress,
10599                },
10600            )
10601            .unwrap());
10602        assert!(!dying_ctx.egress.is_closed());
10603
10604        // The frame that used to end the module connection.
10605        let ack = handler
10606            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10607            .await;
10608        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10609        if module_loop_error.is_some() {
10610            // What the server's connection loop does with a router error that has
10611            // no ERROR-frame translation: end the connection, which releases the
10612            // module's registration and every route on it.
10613            handler.cleanup_connection(module_connection).unwrap();
10614        }
10615        // Read the module's next frame before opening the co-tenant's route, so
10616        // the GOODBYE assertion below is about THIS ack and not about later
10617        // traffic. `None` means the module was told nothing.
10618        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10619            .await
10620            .ok()
10621            .flatten();
10622
10623        // 1. The module connection is still registered.
10624        assert!(
10625            registry
10626                .get_module_by_connection(module_connection)
10627                .unwrap()
10628                .is_some(),
10629            "one client's closing connection ended the shared module connection: \
10630             {module_loop_error:?}"
10631        );
10632        // ...and still serving: another client can open and use a route on it.
10633        let cotenant = ConnectionId::new(32);
10634        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10635        let (cotenant_task, cotenant_bind) = relay_route_open(
10636            &handler,
10637            cotenant,
10638            &cotenant_ctx.egress,
10639            &mut module_rx,
10640            102,
10641            "aft",
10642            "closing-cotenant",
10643        )
10644        .await;
10645        handler
10646            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10647            .await
10648            .unwrap();
10649        assert!(cotenant_task.await.unwrap().is_empty());
10650        let (cotenant_channel, cotenant_epoch) =
10651            published_route(&cotenant_rx.recv().await.unwrap());
10652        assert!(matches!(
10653            forwarding
10654                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10655                .unwrap(),
10656            DataRoute::Client(DataRouteState::Bound(_))
10657        ));
10658
10659        // 2. The module was told to drop the binding it created for the route
10660        //    that will never be published.
10661        let goodbye = post_ack_module_frame
10662            .expect("module receives a GOODBYE for the abandoned route channel");
10663        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10664        assert_eq!(goodbye.header.channel, abandoned_channel);
10665        assert_eq!(goodbye.header.epoch, abandoned_epoch);
10666
10667        // 3. The dying client received nothing: no route was ever published to
10668        //    it. Its route.open is answered as unavailable, which the connection
10669        //    loop would write to a socket that is already going away.
10670        assert!(dying_rx.try_recv().is_err());
10671        let second_response = second_task.await.unwrap();
10672        assert_eq!(second_response.len(), 1);
10673        assert_eq!(
10674            parse_error(&second_response[0])["code"],
10675            "target_unavailable"
10676        );
10677    }
10678
10679    /// One `catalog.list` as a client sees it: the generation and the listed
10680    /// module ids with their readiness.
10681    async fn catalog_snapshot(
10682        handler: &ControlHandler,
10683        client_ctx: &RouteCtx,
10684        corr: u64,
10685    ) -> (u64, Vec<(String, bool)>) {
10686        let body =
10687            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10688        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10689        let reply = handler
10690            .handle_control_frame(client_ctx, request)
10691            .await
10692            .unwrap()
10693            .pop()
10694            .unwrap();
10695        let ClientControlResponse::CatalogList {
10696            generation,
10697            modules,
10698            ..
10699        } = serde_json::from_slice(&reply.body).unwrap()
10700        else {
10701            panic!("not a catalog.list reply");
10702        };
10703        (
10704            generation,
10705            modules
10706                .into_iter()
10707                .map(|entry| (entry.module_id, entry.ready))
10708                .collect(),
10709        )
10710    }
10711
10712    /// A handler whose HELLO for module "aft" on connection 40 is parked
10713    /// between reserving its registry entry and installing its forwarding
10714    /// endpoint. Returns the handler, the two-party barrier (wait once to know
10715    /// the HELLO is parked, once more to release it), the HELLO's thread, and
10716    /// the module's context and receiver.
10717    fn hello_held_before_forwarding() -> (
10718        ControlHandler,
10719        Arc<std::sync::Barrier>,
10720        std::thread::JoinHandle<Vec<Frame>>,
10721        RouteCtx,
10722        mpsc::Receiver<crate::router::OutboundFrame>,
10723    ) {
10724        let hold = Arc::new(std::sync::Barrier::new(2));
10725        let handler = ControlHandler::with_forwarding(
10726            Arc::new(Registry::default()),
10727            Arc::new(ForwardingTable::default()),
10728        )
10729        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10730        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10731        let hello = {
10732            let handler = handler.clone();
10733            let module_ctx = module_ctx.clone();
10734            std::thread::spawn(move || {
10735                handler
10736                    .handle_hello(
10737                        module_ctx.connection_id,
10738                        Some(module_ctx.egress.clone()),
10739                        hello_frame("aft", PROTOCOL_VERSION, 7),
10740                    )
10741                    .unwrap()
10742            })
10743        };
10744        (handler, hold, hello, module_ctx, module_rx)
10745    }
10746
10747    /// While a HELLO has reserved its registry entry but its forwarding
10748    /// endpoint is not installed yet, `route.open` to the module can only be
10749    /// refused. `catalog.list` must not list the module in that window: a
10750    /// client that opens a route as soon as it sees a module listed and ready
10751    /// would be refused for a module the daemon had just advertised. The
10752    /// refusal in the window stays the retryable `target_unavailable`, never
10753    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10754    /// Once the HELLO completes, the module is listed and a route to it opens.
10755    #[tokio::test]
10756    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10757        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10758        let client = ConnectionId::new(41);
10759        let (client_ctx, mut client_rx) = route_ctx(client);
10760
10761        hold.wait();
10762        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10763        assert!(handler.registry.get_module("aft").unwrap().is_some());
10764        assert!(!handler
10765            .forwarding
10766            .has_live_module_connection("aft")
10767            .unwrap());
10768        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10769        assert!(
10770            listed.is_empty(),
10771            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10772        );
10773        let refused = handler
10774            .handle_control_frame(
10775                &client_ctx,
10776                route_open_frame(301, "aft", unique_project_root("listed-window")),
10777            )
10778            .await
10779            .unwrap();
10780        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10781        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10782        let duplicate = handler
10783            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10784            .await
10785            .unwrap();
10786        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10787
10788        hold.wait();
10789        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10790        let ack = module_rx.try_recv().unwrap().frame;
10791        assert_eq!(ack.header.ty, FrameType::HelloAck);
10792        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10793        assert_eq!(listed, vec![("aft".to_string(), true)]);
10794
10795        let (open_task, bind) = relay_route_open(
10796            &handler,
10797            client,
10798            &client_ctx.egress,
10799            &mut module_rx,
10800            303,
10801            "aft",
10802            "listed-after",
10803        )
10804        .await;
10805        assert!(handler
10806            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10807            .await
10808            .unwrap()
10809            .is_empty());
10810        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10811        published_route(&client_rx.try_recv().unwrap().frame);
10812    }
10813
10814    /// A catalog watcher keys on the generation: when it changes, it re-reads
10815    /// the listing. So the generation must advance exactly when the module
10816    /// appears. If it advanced while the module was still hidden, a watcher
10817    /// would read the new generation without the module, and the module would
10818    /// later appear under that same generation with nothing telling the
10819    /// watcher to look again.
10820    #[tokio::test]
10821    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10822        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10823        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10824        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10825
10826        hold.wait();
10827        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10828        assert!(listed.is_empty());
10829        assert_eq!(
10830            during, before,
10831            "the generation moved while the module was not listed"
10832        );
10833
10834        hold.wait();
10835        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10836        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10837        assert_eq!(listed, vec![("aft".to_string(), true)]);
10838        assert_eq!(
10839            after,
10840            before + 1,
10841            "the module appeared without exactly one generation bump"
10842        );
10843    }
10844
10845    /// A HELLO whose forwarding endpoint cannot be installed (here the
10846    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
10847    /// refused and leaves nothing behind: no registry entry, no catalog entry,
10848    /// and the catalog generation where it was.
10849    #[tokio::test]
10850    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10851        let registry = Arc::new(Registry::default());
10852        let handler = ControlHandler::with_forwarding(
10853            Arc::clone(&registry),
10854            Arc::new(ForwardingTable::default()),
10855        );
10856        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10857        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10858        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10859        drop(module_rx);
10860
10861        let replies = handler
10862            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10863            .await
10864            .unwrap();
10865        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10866        assert!(registry.get_module("aft").unwrap().is_none());
10867        assert!(registry
10868            .get_module_by_connection(module_ctx.connection_id)
10869            .unwrap()
10870            .is_none());
10871        assert_eq!(
10872            catalog_snapshot(&handler, &client_ctx, 301).await,
10873            (before, Vec::new())
10874        );
10875    }
10876
10877    /// A handler whose `route.open` parks after queueing the route.bind and
10878    /// before marking it enqueued, with module "aft" registered on connection
10879    /// 40. Returns the two-party barrier that parks and releases it.
10880    async fn route_open_held_after_bind_send() -> (
10881        ControlHandler,
10882        Arc<tokio::sync::Barrier>,
10883        RouteCtx,
10884        mpsc::Receiver<crate::router::OutboundFrame>,
10885    ) {
10886        let hold = Arc::new(tokio::sync::Barrier::new(2));
10887        let handler = ControlHandler::with_forwarding(
10888            Arc::new(Registry::default()),
10889            Arc::new(ForwardingTable::default()),
10890        )
10891        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10892        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10893        hello_via_sink(
10894            &handler,
10895            &module_ctx,
10896            &mut module_rx,
10897            hello_frame("aft", PROTOCOL_VERSION, 7),
10898        )
10899        .await;
10900        (handler, hold, module_ctx, module_rx)
10901    }
10902
10903    /// A fast module acks the route.bind, and the reader commits the route,
10904    /// before `route.open` has marked the relay enqueued. The daemon used to
10905    /// read the missing relay as an abandoned bind and send the module a
10906    /// GOODBYE for the route it had just committed: the client was told its
10907    /// route was open, the module unbound the channel, and the client's first
10908    /// request was dropped on the module side as unbound.
10909    #[tokio::test]
10910    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10911        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10912        let client = ConnectionId::new(41);
10913        let (client_ctx, mut client_rx) = route_ctx(client);
10914
10915        let (open_task, bind) = relay_route_open(
10916            &handler,
10917            client,
10918            &client_ctx.egress,
10919            &mut module_rx,
10920            200,
10921            "aft",
10922            "acked-before-mark",
10923        )
10924        .await;
10925        let (module_channel, module_epoch) = route_bind_channel(&bind);
10926        // route.open is now parked between queueing the bind and marking it.
10927        hold.wait().await;
10928
10929        // The module answers inside that window, and the route commits.
10930        assert!(handler
10931            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10932            .await
10933            .unwrap()
10934            .is_empty());
10935        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10936
10937        hold.wait().await;
10938        assert!(
10939            open_task.await.unwrap().is_empty(),
10940            "route.open answers accepted"
10941        );
10942
10943        // Every frame route.open could have sent is already queued, because it
10944        // has returned; the module must have been sent nothing.
10945        match module_rx.try_recv() {
10946            Err(mpsc::error::TryRecvError::Empty) => {}
10947            Ok(outbound) => panic!(
10948                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10949                outbound.frame.header.ty,
10950                outbound.frame.header.channel,
10951                outbound.frame.header.epoch
10952            ),
10953            Err(err) => panic!("module egress closed: {err}"),
10954        }
10955
10956        // The client's first request on the route is the next thing the module
10957        // reads, on the binding it just created.
10958        let router = Router::with_control_handler(Arc::new(handler.clone()));
10959        let request = Frame::build(
10960            FrameType::Request,
10961            Flags::new(true, Priority::Interactive, false),
10962            client_channel,
10963            client_epoch,
10964            201,
10965            b"first request".to_vec(),
10966        )
10967        .unwrap();
10968        router
10969            .route_for_connection(&client_ctx, request)
10970            .await
10971            .unwrap();
10972        let delivered = module_rx.try_recv().unwrap().frame;
10973        assert_eq!(delivered.header.ty, FrameType::Request);
10974        assert_eq!(
10975            (delivered.header.channel, delivered.header.epoch),
10976            (module_channel, module_epoch)
10977        );
10978        assert_eq!(delivered.body, b"first request");
10979    }
10980
10981    /// The client goes away while `route.open` is still queueing the bind, so
10982    /// the relay is dropped with no module reply. The teardown cannot send the
10983    /// GOODBYE itself (it could reach the module ahead of the bind), so
10984    /// `route.open` must send it once the bind is queued; otherwise the module
10985    /// keeps a binding the daemon will never commit.
10986    #[tokio::test]
10987    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10988        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10989        let client = ConnectionId::new(42);
10990        let (client_ctx, _client_rx) = route_ctx(client);
10991
10992        let (open_task, bind) = relay_route_open(
10993            &handler,
10994            client,
10995            &client_ctx.egress,
10996            &mut module_rx,
10997            300,
10998            "aft",
10999            "released-before-mark",
11000        )
11001        .await;
11002        let (module_channel, module_epoch) = route_bind_channel(&bind);
11003        hold.wait().await;
11004
11005        handler.cleanup_connection(client).unwrap();
11006        assert!(
11007            module_rx.try_recv().is_err(),
11008            "client teardown must not GOODBYE a bind route.open is still queueing"
11009        );
11010
11011        hold.wait().await;
11012        let response = open_task.await.unwrap();
11013        assert_eq!(response.len(), 1);
11014        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11015
11016        let goodbye = module_rx
11017            .try_recv()
11018            .expect("module is told to drop the binding the daemon abandoned")
11019            .frame;
11020        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11021        assert_eq!(
11022            (goodbye.header.channel, goodbye.header.epoch),
11023            (module_channel, module_epoch)
11024        );
11025        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11026    }
11027
11028    /// The module acks inside the same window, but the client's egress closed
11029    /// first, so the reader refuses to commit. The ack proves the module holds
11030    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
11031    /// send a second one.
11032    #[tokio::test]
11033    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11034        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11035        let client = ConnectionId::new(43);
11036        let (client_ctx, client_rx) = route_ctx(client);
11037
11038        let (open_task, bind) = relay_route_open(
11039            &handler,
11040            client,
11041            &client_ctx.egress,
11042            &mut module_rx,
11043            400,
11044            "aft",
11045            "gone-client-before-mark",
11046        )
11047        .await;
11048        let (module_channel, module_epoch) = route_bind_channel(&bind);
11049        hold.wait().await;
11050
11051        drop(client_rx);
11052        assert!(client_ctx.egress.is_closed());
11053        handler
11054            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11055            .await
11056            .unwrap();
11057
11058        hold.wait().await;
11059        let response = open_task.await.unwrap();
11060        assert_eq!(response.len(), 1);
11061        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11062
11063        let goodbye = module_rx
11064            .try_recv()
11065            .expect("module is told to drop the binding it acked")
11066            .frame;
11067        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11068        assert_eq!(
11069            (goodbye.header.channel, goodbye.header.epoch),
11070            (module_channel, module_epoch)
11071        );
11072        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11073        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11074    }
11075
11076    /// The fence at the module-loop boundary, stated as its own contract: which
11077    /// forwarding failures are allowed to end the module connection that is being
11078    /// served. A `ConnectionClosing` naming some client is about that client, and
11079    /// a module connection is shared; the same error naming the module's own
11080    /// connection is about this connection and must stay fatal, as must failures
11081    /// that are about the forwarding table itself.
11082    #[test]
11083    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11084        let handler = ControlHandler::default();
11085        let module_connection = ConnectionId::new(30);
11086        let client_connection = ConnectionId::new(31);
11087
11088        handler
11089            .refuse_to_end_module_connection_for_a_client(
11090                module_connection,
11091                77,
11092                ForwardingError::ConnectionClosing {
11093                    connection_id: client_connection,
11094                },
11095            )
11096            .expect("a closing client must never end the module connection");
11097
11098        assert!(matches!(
11099            handler.refuse_to_end_module_connection_for_a_client(
11100                module_connection,
11101                78,
11102                ForwardingError::ConnectionClosing {
11103                    connection_id: module_connection,
11104                },
11105            ),
11106            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11107                connection_id
11108            })) if connection_id == module_connection
11109        ));
11110        assert!(matches!(
11111            handler.refuse_to_end_module_connection_for_a_client(
11112                module_connection,
11113                79,
11114                ForwardingError::Poisoned,
11115            ),
11116            Err(RouterError::Forwarding(ForwardingError::Poisoned))
11117        ));
11118        assert!(matches!(
11119            handler.refuse_to_end_module_connection_for_a_client(
11120                module_connection,
11121                80,
11122                ForwardingError::StaleModuleEndpoint,
11123            ),
11124            Err(RouterError::Forwarding(
11125                ForwardingError::StaleModuleEndpoint
11126            ))
11127        ));
11128    }
11129
11130    /// The spawn-attestation guard is what stops a connected module from claiming
11131    /// another module's identity and being stamped `Reserved` for it. Every other
11132    /// test that supplies a consumer_identity supplies a CORRECT one, because a
11133    /// correct one is what the rest of the flow needs -- so the guard's rejection
11134    /// branch was never the subject of an assertion, only its acceptance branch.
11135    ///
11136    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
11137    /// whole subc-core library suite green; only the forwarding integration tests
11138    /// notice, and they notice for unrelated reasons. This test exists so the
11139    /// refusal itself is asserted where the guard lives: it fails if the identity
11140    /// check stops refusing, which is the direction that matters, since a guard
11141    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
11142    #[tokio::test]
11143    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11144        let registry = Arc::new(Registry::default());
11145        let forwarding = Arc::new(ForwardingTable::default());
11146        let supervisor = SupervisorHandle::new();
11147        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11148        let handler =
11149            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11150                .with_supervisor(supervisor);
11151
11152        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11153        hello_via_sink(
11154            &handler,
11155            &target_ctx,
11156            &mut target_rx,
11157            hello_frame("target", PROTOCOL_VERSION, 1),
11158        )
11159        .await;
11160
11161        // A real supervised module id presenting the wrong nonce. This is the
11162        // impersonation case: the attacker knows a privileged module_id, which is
11163        // public, and guesses at the nonce, which is not.
11164        let wrong_nonce = handler
11165            .handle_control_frame(
11166                &route_ctx(ConnectionId::new(91)).0,
11167                route_open_frame_with_admission_facts(
11168                    20,
11169                    "target",
11170                    unique_project_root("admission-facts"),
11171                    Some(subc_control::ConsumerIdentity {
11172                        module_id: "fed".to_string(),
11173                        launch_nonce: "not-the-real-nonce".to_string(),
11174                    }),
11175                    None,
11176                ),
11177            )
11178            .await
11179            .unwrap();
11180        assert_eq!(
11181            parse_error(&wrong_nonce[0])["code"],
11182            "bad_consumer_identity",
11183            "a mismatched launch nonce must be refused, not stamped Reserved"
11184        );
11185
11186        // A module id the supervisor never spawned at all, so no nonce exists to
11187        // compare against. An implementation that treats "no record" as "nothing
11188        // to check" fails open here while passing the case above.
11189        let never_spawned = handler
11190            .handle_control_frame(
11191                &route_ctx(ConnectionId::new(92)).0,
11192                route_open_frame_with_admission_facts(
11193                    21,
11194                    "target",
11195                    unique_project_root("admission-facts"),
11196                    Some(subc_control::ConsumerIdentity {
11197                        module_id: "never-spawned".to_string(),
11198                        launch_nonce: "any-nonce".to_string(),
11199                    }),
11200                    None,
11201                ),
11202            )
11203            .await
11204            .unwrap();
11205        assert_eq!(
11206            parse_error(&never_spawned[0])["code"],
11207            "bad_consumer_identity",
11208            "an unspawned module_id must be refused rather than accepted for lack of a record"
11209        );
11210    }
11211
11212    /// The refusal test above proves the guard says NO. Nothing proved it can say
11213    /// YES, and the difference is not academic: replacing the whole authorization
11214    /// with `false` -- admitting no consumer identity at all, revoking Reserved
11215    /// standing for every supervised module in the fleet -- leaves 110 of the 111
11216    /// library tests GREEN. The one that notices does so by HANGING, because it
11217    /// waits for a bind that can no longer happen.
11218    ///
11219    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
11220    /// or flaky test, invites a RETRY rather than an investigation, and the retry
11221    /// hangs too and gets blamed on the runner. So a total revocation of the
11222    /// daemon's trust grant would have shipped behind a symptom nobody attributes
11223    /// to code.
11224    ///
11225    /// The bias is structural rather than accidental. A REFUSAL looks like a
11226    /// failure someone writes a test for; a GRANT looks like the happy path. Every
11227    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
11228    /// suite for that reason, and this one is the purest case in the daemon.
11229    ///
11230    /// This test asserts the EFFECT rather than the absence of an error: the module
11231    /// receives a RouteBind and it carries `Reserved` naming the attested module.
11232    /// A guard that admitted nobody would produce no bind at all; one that admitted
11233    /// everybody would stamp the wrong principal, which the refusal test catches.
11234    #[tokio::test]
11235    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11236        let registry = Arc::new(Registry::default());
11237        let forwarding = Arc::new(ForwardingTable::default());
11238        let supervisor = SupervisorHandle::new();
11239        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11240        let handler =
11241            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11242                .with_supervisor(supervisor);
11243
11244        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11245        hello_via_sink(
11246            &handler,
11247            &target_ctx,
11248            &mut target_rx,
11249            hello_frame("target", PROTOCOL_VERSION, 1),
11250        )
11251        .await;
11252
11253        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11254        let route_handler = handler.clone();
11255        let route_task = tokio::spawn(async move {
11256            route_handler
11257                .handle_control_frame(
11258                    &client_ctx,
11259                    route_open_frame_with_admission_facts(
11260                        30,
11261                        "target",
11262                        unique_project_root("admission-facts"),
11263                        Some(subc_control::ConsumerIdentity {
11264                            module_id: "fed".to_string(),
11265                            launch_nonce: "fed-nonce".to_string(),
11266                        }),
11267                        None,
11268                    ),
11269                )
11270                .await
11271                .unwrap()
11272        });
11273
11274        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
11275        // the very mutation it exists to catch -- a guard that admits nobody -- no
11276        // bind is ever sent, so it HUNG rather than failing. That reproduces the
11277        // exact defect being fixed: a total revocation detected only as a stalled
11278        // suite, which reads as flakiness and invites a retry. An acceptance test
11279        // that waits for an effect must bound the wait, or a red becomes a hang.
11280        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11281            .await
11282            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11283            .expect("module control channel closed before route.bind");
11284        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11285        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11286            panic!("expected route.bind")
11287        };
11288        assert_eq!(
11289            principal,
11290            Some(Principal::Reserved {
11291                module_id: "fed".to_string()
11292            }),
11293            "a correctly attested consumer must be stamped Reserved for its own id"
11294        );
11295
11296        handler
11297            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11298            .await
11299            .unwrap();
11300        assert!(route_task.await.unwrap().is_empty());
11301        assert!(
11302            matches!(
11303                serde_json::from_slice::<ClientControlResponse>(
11304                    &client_rx.recv().await.unwrap().body
11305                )
11306                .unwrap(),
11307                ClientControlResponse::RouteOpen { .. }
11308            ),
11309            "the route must actually open, not merely avoid an error"
11310        );
11311    }
11312
11313    #[tokio::test(start_paused = true)]
11314    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11315        let registry = Arc::new(Registry::default());
11316        let forwarding = Arc::new(ForwardingTable::default());
11317        let supervisor = SupervisorHandle::new();
11318        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11319        let handler =
11320            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11321                .with_supervisor(supervisor);
11322
11323        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11324        hello_via_sink(
11325            &handler,
11326            &target_ctx,
11327            &mut target_rx,
11328            hello_frame("target", PROTOCOL_VERSION, 1),
11329        )
11330        .await;
11331
11332        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11333        let direct_handler = handler.clone();
11334        let direct_open = tokio::spawn(async move {
11335            direct_handler
11336                .handle_control_frame(
11337                    &direct_ctx,
11338                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
11339                )
11340                .await
11341                .unwrap()
11342        });
11343        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11344            .await
11345            .expect("no direct route.bind within 5s")
11346            .expect("target control channel closed before direct route.bind");
11347        handler
11348            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11349            .await
11350            .unwrap();
11351        assert!(direct_open.await.unwrap().is_empty());
11352        let _ = direct_rx.recv().await.unwrap();
11353
11354        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11355        let reserved_handler = handler.clone();
11356        let reserved_open = tokio::spawn(async move {
11357            reserved_handler
11358                .handle_control_frame(
11359                    &reserved_ctx,
11360                    route_open_frame_with_admission_facts(
11361                        3,
11362                        "target",
11363                        unique_project_root("admission-facts"),
11364                        Some(ConsumerIdentity {
11365                            module_id: "fed".to_string(),
11366                            launch_nonce: "fed-nonce".to_string(),
11367                        }),
11368                        None,
11369                    ),
11370                )
11371                .await
11372                .unwrap()
11373        });
11374        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11375            .await
11376            .expect("no reserved route.bind within 5s")
11377            .expect("target control channel closed before reserved route.bind");
11378        handler
11379            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11380            .await
11381            .unwrap();
11382        assert!(reserved_open.await.unwrap().is_empty());
11383        let _ = reserved_rx.recv().await.unwrap();
11384
11385        forwarding
11386            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11387            .unwrap();
11388        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11389        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11390            module_id: Some("target".to_string()),
11391        })
11392        .unwrap();
11393        let census_frame =
11394            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11395        let response = handler
11396            .handle_control_frame(&census_ctx, census_frame)
11397            .await
11398            .unwrap()
11399            .pop()
11400            .unwrap();
11401        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11402        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11403        assert!(matches!(
11404            decoded,
11405            ClientControlResponse::SupervisorRoutes { .. }
11406        ));
11407        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11408        assert_eq!(routes.len(), 2);
11409        assert!(routes.iter().all(|route| route["draining"] == true));
11410        // The census carries WHY: the reason the drain was begun with, in the
11411        // route.closing vocabulary, on every draining route this drain marked.
11412        assert!(
11413            routes.iter().all(|route| route["drain_reason"] == "reload"),
11414            "draining routes must name the drain's reason: {routes:?}"
11415        );
11416        assert!(routes.iter().any(|route| {
11417            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11418        }));
11419        assert!(routes.iter().any(|route| {
11420            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11421        }));
11422
11423        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11424            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11425        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11426            std::fs::write(
11427                &golden_path,
11428                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11429            )
11430            .unwrap();
11431        }
11432        let expected: Value =
11433            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11434        assert_eq!(actual, expected);
11435    }
11436
11437    async fn query_live_roots(
11438        handler: &ControlHandler,
11439        module_ctx: &RouteCtx,
11440    ) -> ModuleControlResponseToModule {
11441        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11442        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11443        let response = handler
11444            .handle_control_frame(module_ctx, frame)
11445            .await
11446            .unwrap()
11447            .pop()
11448            .unwrap();
11449        serde_json::from_slice(&response.body).unwrap()
11450    }
11451
11452    #[tokio::test(start_paused = true)]
11453    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11454        let registry = Arc::new(Registry::default());
11455        let forwarding = Arc::new(ForwardingTable::default());
11456        let handler = ControlHandler::with_forwarding(registry, forwarding);
11457        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11458        hello_via_sink(
11459            &handler,
11460            &target_ctx,
11461            &mut target_rx,
11462            hello_frame("target", PROTOCOL_VERSION, 1),
11463        )
11464        .await;
11465        let root = unique_project_root("live-roots-known");
11466        let path = ProjectRootId::from_path_allowing_missing(root.path())
11467            .unwrap()
11468            .as_path()
11469            .to_path_buf();
11470        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11471        let open_handler = handler.clone();
11472        let opened = tokio::spawn(async move {
11473            open_handler
11474                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11475                .await
11476                .unwrap()
11477        });
11478        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11479            .await
11480            .unwrap()
11481            .unwrap();
11482        handler
11483            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11484            .await
11485            .unwrap();
11486        assert!(opened.await.unwrap().is_empty());
11487        let _ = client_rx.recv().await.unwrap();
11488
11489        let root = unique_project_root("live-roots-pending");
11490        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11491            .unwrap()
11492            .as_path()
11493            .to_path_buf();
11494        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11495        let open_handler = handler.clone();
11496        let pending = tokio::spawn(async move {
11497            open_handler
11498                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11499                .await
11500                .unwrap()
11501        });
11502        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11503            .await
11504            .unwrap()
11505            .unwrap();
11506        let actual = query_live_roots(&handler, &target_ctx).await;
11507        let ModuleControlResponseToModule::LiveRoots {
11508            roots,
11509            unknown_root_bindings,
11510            total_bindings,
11511        } = actual
11512        else {
11513            panic!("expected live roots")
11514        };
11515        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11516        assert_eq!(unknown_root_bindings, 0);
11517        assert_eq!(
11518            roots.len(),
11519            2,
11520            "root-known arm must retain each canonical root"
11521        );
11522        assert_eq!(
11523            total_bindings,
11524            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11525        );
11526        let counts = roots
11527            .iter()
11528            .map(|root| (root.project_root.clone(), root.bound, root.pending))
11529            .collect::<Vec<_>>();
11530        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11531        expected.sort_by(|a, b| a.0.cmp(&b.0));
11532        assert_eq!(
11533            counts, expected,
11534            "roots must sort by path and count pending separately"
11535        );
11536        handler
11537            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11538            .await
11539            .unwrap();
11540        assert!(pending.await.unwrap().is_empty());
11541    }
11542
11543    #[tokio::test(start_paused = true)]
11544    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11545        let forwarding = Arc::new(ForwardingTable::default());
11546        let handler =
11547            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11548        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11549        hello_via_sink(
11550            &handler,
11551            &target_ctx,
11552            &mut target_rx,
11553            hello_frame("target", PROTOCOL_VERSION, 1),
11554        )
11555        .await;
11556        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11557        let pending = forwarding
11558            .begin_route_bind_relay_for_test(
11559                client_ctx.connection_id,
11560                client_ctx.egress.clone(),
11561                2,
11562                "target",
11563            )
11564            .unwrap();
11565        forwarding
11566            .complete_pending_relay(
11567                target_ctx.connection_id,
11568                pending.corr,
11569                RouteBindRelayOutcome::Accepted,
11570            )
11571            .unwrap();
11572        let actual = query_live_roots(&handler, &target_ctx).await;
11573        let ModuleControlResponseToModule::LiveRoots {
11574            roots,
11575            unknown_root_bindings,
11576            total_bindings,
11577        } = actual
11578        else {
11579            panic!("expected live roots")
11580        };
11581        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11582        assert_eq!(
11583            unknown_root_bindings, 1,
11584            "unknown-root arm must not read as no bindings"
11585        );
11586        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11587        assert_eq!(
11588            total_bindings,
11589            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11590        );
11591    }
11592
11593    /// A module reads HELLO_ACK as its first frame and exits on anything else,
11594    /// so the ack has to be on its outbound queue before the module is
11595    /// routable. The connection loop writes a handler's replies only after the
11596    /// handler returns; this test stops in exactly that gap, runs a real
11597    /// route.open from another connection, and only then writes whatever the
11598    /// HELLO handler returned, the way the loop would. If the ack were still a
11599    /// reply, the route.bind request would reach the module first.
11600    #[tokio::test(start_paused = true)]
11601    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11602        let forwarding = Arc::new(ForwardingTable::default());
11603        let handler =
11604            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11605        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11606        let replies = handler
11607            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11608            .await
11609            .unwrap();
11610        let queued_by_hello = module_rx.len();
11611
11612        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11613        let open_handler = handler.clone();
11614        let open = tokio::spawn(async move {
11615            open_handler
11616                .handle_control_frame(
11617                    &client_ctx,
11618                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11619                )
11620                .await
11621                .unwrap()
11622        });
11623        // Let the route.open run until its route.bind is on the module's queue.
11624        let mut spins = 0;
11625        while module_rx.len() == queued_by_hello {
11626            spins += 1;
11627            assert!(spins < 10_000, "route.open never queued a route.bind");
11628            tokio::task::yield_now().await;
11629        }
11630
11631        // Now the connection loop's half: write the HELLO handler's replies.
11632        for reply in replies {
11633            module_ctx.egress.send(reply).await.unwrap();
11634        }
11635
11636        let first = module_rx.recv().await.unwrap().frame;
11637        assert_eq!(
11638            first.header.ty,
11639            FrameType::HelloAck,
11640            "the first frame a registering module reads must be its HELLO_ACK"
11641        );
11642        assert_eq!(first.header.corr, 7);
11643        let second = module_rx.recv().await.unwrap().frame;
11644        assert_eq!(second.header.ty, FrameType::Request);
11645        assert!(
11646            matches!(
11647                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11648                ModuleControlRequest::RouteBind { .. }
11649            ),
11650            "the route.bind follows the ack"
11651        );
11652        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11653
11654        handler
11655            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11656            .await
11657            .unwrap();
11658        assert!(open.await.unwrap().is_empty());
11659        let _ = client_rx.recv().await.unwrap();
11660    }
11661
11662    #[tokio::test(start_paused = true)]
11663    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11664        let handler = ControlHandler::with_forwarding(
11665            Arc::new(Registry::default()),
11666            Arc::new(ForwardingTable::default()),
11667        );
11668        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11669        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11670        hello_via_sink(
11671            &handler,
11672            &first_ctx,
11673            &mut first_rx,
11674            hello_frame("first", PROTOCOL_VERSION, 1),
11675        )
11676        .await;
11677        hello_via_sink(
11678            &handler,
11679            &second_ctx,
11680            &mut second_rx,
11681            hello_frame("second", PROTOCOL_VERSION, 2),
11682        )
11683        .await;
11684        let root = unique_project_root("second-only");
11685        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11686        let cloned = handler.clone();
11687        let open = tokio::spawn(async move {
11688            cloned
11689                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11690                .await
11691                .unwrap()
11692        });
11693        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11694            .await
11695            .unwrap()
11696            .unwrap();
11697        let first = query_live_roots(&handler, &first_ctx).await;
11698        let second = query_live_roots(&handler, &second_ctx).await;
11699        assert!(
11700            matches!(
11701                first,
11702                ModuleControlResponseToModule::LiveRoots {
11703                    total_bindings: 0,
11704                    ..
11705                }
11706            ),
11707            "cross-module scope must not expose another module's roots"
11708        );
11709        assert!(
11710            matches!(
11711                second,
11712                ModuleControlResponseToModule::LiveRoots {
11713                    total_bindings: 1,
11714                    ..
11715                }
11716            ),
11717            "second module must see its pending route"
11718        );
11719        handler
11720            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11721            .await
11722            .unwrap();
11723        assert!(open.await.unwrap().is_empty());
11724    }
11725
11726    #[tokio::test(start_paused = true)]
11727    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11728        let handler = ControlHandler::with_forwarding(
11729            Arc::new(Registry::default()),
11730            Arc::new(ForwardingTable::default()),
11731        );
11732        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11733        hello_via_sink(
11734            &handler,
11735            &target_ctx,
11736            &mut target_rx,
11737            hello_frame("target", PROTOCOL_VERSION, 1),
11738        )
11739        .await;
11740        let actual = query_live_roots(&handler, &target_ctx).await;
11741        let ModuleControlResponseToModule::LiveRoots {
11742            roots,
11743            unknown_root_bindings,
11744            total_bindings,
11745        } = actual
11746        else {
11747            panic!("expected live roots")
11748        };
11749        assert!(roots.is_empty());
11750        assert_eq!(unknown_root_bindings, 0);
11751        assert_eq!(total_bindings, 0);
11752        assert_eq!(
11753            total_bindings,
11754            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11755        );
11756    }
11757
11758    /// Read the vendored fed corpus rather than hand-building a package.
11759    ///
11760    /// A hand-built object encodes what the test author believed the carrier
11761    /// emits. These vectors are what it actually emits, and one of them exists
11762    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11763    /// ignores additive unknown fields at the traversal emit terminus."
11764    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11765        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11766            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11767        let text = std::fs::read_to_string(&path)
11768            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11769        let vectors: Vec<(String, Value)> = text
11770            .lines()
11771            .filter(|line| !line.trim().is_empty())
11772            .map(|line| {
11773                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11774                let id = entry["corpus_id"]
11775                    .as_str()
11776                    .expect("every vector carries a corpus_id")
11777                    .to_string();
11778                (id, entry["package"].clone())
11779            })
11780            .collect();
11781        // Pin the count: a corpus that silently shrinks would take its coverage
11782        // with it, and a suite reading N-1 vectors reports the same clean pass
11783        // as one reading N.
11784        assert_eq!(
11785            vectors.len(),
11786            3,
11787            "vendored fed corpus changed size; re-sync from subc-federation"
11788        );
11789
11790        // Pin what makes the corpus DISCRIMINATING, not just present.
11791        //
11792        // The relay test below takes its expected value from the corpus, so the
11793        // corpus supplies the test's power to detect a lossy relay rather than
11794        // its correctness. A relay that dropped unrecognised fields would still
11795        // be caught -- but only by a package carrying fields it does not know.
11796        // Shrink every package to the handful of keys any implementation would
11797        // recognise and the test keeps passing over an input that can no longer
11798        // fail, which is the same clean green as a corpus that shrank away.
11799        //
11800        // So assert the precondition rather than duplicating the packages here:
11801        // at least one vector must carry a field beyond the small common set.
11802        // That is one claim to maintain instead of nine, and it fails loudly if
11803        // a re-sync ever flattens the corpus.
11804        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11805        let richest = vectors
11806            .iter()
11807            .filter_map(|(_, package)| package.as_object())
11808            .map(|object| {
11809                object
11810                    .keys()
11811                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11812                    .count()
11813            })
11814            .max()
11815            .unwrap_or(0);
11816        assert!(
11817            richest >= 2,
11818            "vendored corpus no longer carries a package with unmodelled fields, \
11819             so the relay test can no longer distinguish a verbatim relay from a lossy one"
11820        );
11821
11822        vectors
11823    }
11824
11825    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
11826    ///
11827    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
11828    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
11829    /// three-key object, so a relay that quietly dropped fields it did not
11830    /// recognise would satisfy it. These vectors carry nine keys including ones
11831    /// this crate has no type for, so a typed relay fails here and only here.
11832    #[tokio::test]
11833    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11834        for (corpus_id, package) in fed_admission_facts_vectors() {
11835            let registry = Arc::new(Registry::default());
11836            let forwarding = Arc::new(ForwardingTable::default());
11837            let supervisor = SupervisorHandle::new();
11838            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11839            let handler =
11840                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11841                    .with_supervisor(supervisor)
11842                    .with_admission_facts_config(
11843                        Some("fed".to_string()),
11844                        Some(vec!["target".to_string()]),
11845                    );
11846
11847            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11848            hello_via_sink(
11849                &handler,
11850                &target_ctx,
11851                &mut target_rx,
11852                hello_frame("target", PROTOCOL_VERSION, 1),
11853            )
11854            .await;
11855
11856            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11857            let route_handler = handler.clone();
11858            let expected = package.clone();
11859            let route_task = tokio::spawn(async move {
11860                route_handler
11861                    .handle_control_frame(
11862                        &client_ctx,
11863                        route_open_frame_with_admission_facts(
11864                            20,
11865                            "target",
11866                            unique_project_root("admission-facts"),
11867                            Some(subc_control::ConsumerIdentity {
11868                                module_id: "fed".to_string(),
11869                                launch_nonce: "fed-nonce".to_string(),
11870                            }),
11871                            Some(package),
11872                        ),
11873                    )
11874                    .await
11875                    .unwrap()
11876            });
11877
11878            let bind_frame = target_rx.recv().await.unwrap();
11879            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11880            let ModuleControlRequest::RouteBind {
11881                admission_facts, ..
11882            } = bind
11883            else {
11884                panic!("{corpus_id}: expected route.bind")
11885            };
11886            assert_eq!(
11887                admission_facts,
11888                Some(expected),
11889                "{corpus_id}: relay must not add, drop or reshape any field"
11890            );
11891
11892            handler
11893                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11894                .await
11895                .unwrap();
11896            route_task.await.unwrap();
11897        }
11898    }
11899
11900    #[tokio::test]
11901    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11902        let registry = Arc::new(Registry::default());
11903        let forwarding = Arc::new(ForwardingTable::default());
11904        let supervisor = SupervisorHandle::new();
11905        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11906        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11907        let handler =
11908            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11909                .with_supervisor(supervisor)
11910                .with_admission_facts_config(
11911                    Some("fed".to_string()),
11912                    Some(vec!["target".to_string()]),
11913                );
11914
11915        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11916        hello_via_sink(
11917            &handler,
11918            &target_ctx,
11919            &mut target_rx,
11920            hello_frame("target", PROTOCOL_VERSION, 1),
11921        )
11922        .await;
11923        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11924        hello_via_sink(
11925            &handler,
11926            &other_ctx,
11927            &mut other_rx,
11928            hello_frame("other", PROTOCOL_VERSION, 2),
11929        )
11930        .await;
11931
11932        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11933        let expected_facts = facts.clone();
11934        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11935        let route_handler = handler.clone();
11936        let route_task = tokio::spawn(async move {
11937            route_handler
11938                .handle_control_frame(
11939                    &client_ctx,
11940                    route_open_frame_with_admission_facts(
11941                        10,
11942                        "target",
11943                        unique_project_root("admission-facts"),
11944                        Some(subc_control::ConsumerIdentity {
11945                            module_id: "fed".to_string(),
11946                            launch_nonce: "fed-nonce".to_string(),
11947                        }),
11948                        Some(facts.clone()),
11949                    ),
11950                )
11951                .await
11952                .unwrap()
11953        });
11954        let bind_frame = target_rx.recv().await.unwrap();
11955        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11956        let ModuleControlRequest::RouteBind {
11957            admission_facts, ..
11958        } = bind
11959        else {
11960            panic!("expected route.bind")
11961        };
11962        assert_eq!(admission_facts, Some(expected_facts));
11963        handler
11964            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11965            .await
11966            .unwrap();
11967        assert!(route_task.await.unwrap().is_empty());
11968        assert!(matches!(
11969            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11970                .unwrap(),
11971            ClientControlResponse::RouteOpen { .. }
11972        ));
11973
11974        let direct = handler
11975            .handle_control_frame(
11976                &route_ctx(ConnectionId::new(73)).0,
11977                route_open_frame_with_admission_facts(
11978                    11,
11979                    "target",
11980                    unique_project_root("admission-facts"),
11981                    None,
11982                    Some(json!({"x": 1})),
11983                ),
11984            )
11985            .await
11986            .unwrap();
11987        assert_eq!(
11988            parse_error(&direct[0])["code"],
11989            "admission_facts_not_permitted"
11990        );
11991
11992        let different_reserved = handler
11993            .handle_control_frame(
11994                &route_ctx(ConnectionId::new(77)).0,
11995                route_open_frame_with_admission_facts(
11996                    15,
11997                    "target",
11998                    unique_project_root("admission-facts"),
11999                    Some(subc_control::ConsumerIdentity {
12000                        module_id: "other".to_string(),
12001                        launch_nonce: "other-nonce".to_string(),
12002                    }),
12003                    Some(json!({"x": 1})),
12004                ),
12005            )
12006            .await
12007            .unwrap();
12008        assert_eq!(
12009            parse_error(&different_reserved[0])["code"],
12010            "admission_facts_not_permitted"
12011        );
12012
12013        let other_target = handler
12014            .handle_control_frame(
12015                &route_ctx(ConnectionId::new(74)).0,
12016                route_open_frame_with_admission_facts(
12017                    12,
12018                    "other",
12019                    unique_project_root("admission-facts"),
12020                    Some(subc_control::ConsumerIdentity {
12021                        module_id: "fed".to_string(),
12022                        launch_nonce: "fed-nonce".to_string(),
12023                    }),
12024                    Some(json!({"x": 1})),
12025                ),
12026            )
12027            .await
12028            .unwrap();
12029        assert_eq!(
12030            parse_error(&other_target[0])["code"],
12031            "admission_facts_target_not_allowed"
12032        );
12033
12034        let nonexistent = handler
12035            .handle_control_frame(
12036                &route_ctx(ConnectionId::new(75)).0,
12037                route_open_frame_with_admission_facts(
12038                    13,
12039                    "missing",
12040                    unique_project_root("admission-facts"),
12041                    None,
12042                    Some(json!({"x": 1})),
12043                ),
12044            )
12045            .await
12046            .unwrap();
12047        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12048
12049        let described = handler
12050            .handle_control_frame(
12051                &route_ctx(ConnectionId::new(76)).0,
12052                Frame::build(
12053                    FrameType::Request,
12054                    control_flags(),
12055                    0,
12056                    0,
12057                    14,
12058                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12059                )
12060                .unwrap(),
12061            )
12062            .await
12063            .unwrap();
12064        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12065            serde_json::from_slice(&described[0].body).unwrap()
12066        else {
12067            panic!("expected server.describe response")
12068        };
12069        assert!(capabilities
12070            .iter()
12071            .any(|cap| cap == "admission_facts_relay_v1"));
12072    }
12073
12074    #[tokio::test]
12075    async fn admission_facts_without_configured_carrier_are_rejected() {
12076        let registry = Arc::new(Registry::default());
12077        let forwarding = Arc::new(ForwardingTable::default());
12078        let handler = ControlHandler::with_forwarding(registry, forwarding);
12079        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12080        hello_via_sink(
12081            &handler,
12082            &target_ctx,
12083            &mut target_rx,
12084            hello_frame("target", PROTOCOL_VERSION, 1),
12085        )
12086        .await;
12087
12088        let responses = handler
12089            .handle_control_frame(
12090                &route_ctx(ConnectionId::new(79)).0,
12091                route_open_frame_with_admission_facts(
12092                    16,
12093                    "target",
12094                    unique_project_root("admission-facts"),
12095                    None,
12096                    Some(json!({"x": 1})),
12097                ),
12098            )
12099            .await
12100            .unwrap();
12101        assert_eq!(
12102            parse_error(&responses[0])["code"],
12103            "admission_facts_not_permitted"
12104        );
12105    }
12106
12107    #[tokio::test]
12108    async fn route_open_relays_consumer_capabilities_verbatim() {
12109        let registry = Arc::new(Registry::default());
12110        let forwarding = Arc::new(ForwardingTable::default());
12111        let handler =
12112            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12113        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12114        hello_via_sink(
12115            &handler,
12116            &module_ctx,
12117            &mut module_rx,
12118            hello_frame("aft", PROTOCOL_VERSION, 7),
12119        )
12120        .await;
12121
12122        let expected = vec!["elicitation".to_string(), "roots".to_string()];
12123        let expected_for_request = expected.clone();
12124        let project_root = unique_project_root("consumer-capabilities-present");
12125        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12126        let route_handler = handler.clone();
12127        let route_task = tokio::spawn(async move {
12128            route_handler
12129                .handle_control_frame(
12130                    &client_ctx,
12131                    route_open_frame_with_consumer_capabilities(
12132                        401,
12133                        "aft",
12134                        project_root,
12135                        Some(expected_for_request),
12136                    ),
12137                )
12138                .await
12139                .unwrap()
12140        });
12141        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12142            .await
12143            .unwrap()
12144            .unwrap();
12145        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12146        let ModuleControlRequest::RouteBind {
12147            consumer_capabilities,
12148            ..
12149        } = bind
12150        else {
12151            panic!("expected route.bind request, got {bind:?}");
12152        };
12153        assert_eq!(consumer_capabilities, Some(expected.clone()));
12154
12155        handler
12156            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12157            .await
12158            .unwrap();
12159        let route_response = route_task.await.unwrap();
12160        assert!(route_response.is_empty());
12161        let published = client_rx.recv().await.unwrap();
12162        assert!(matches!(
12163            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12164            ClientControlResponse::RouteOpen { .. }
12165        ));
12166    }
12167
12168    #[tokio::test]
12169    async fn route_open_without_consumer_capabilities_relays_none() {
12170        let registry = Arc::new(Registry::default());
12171        let forwarding = Arc::new(ForwardingTable::default());
12172        let handler =
12173            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12174        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12175        hello_via_sink(
12176            &handler,
12177            &module_ctx,
12178            &mut module_rx,
12179            hello_frame("aft", PROTOCOL_VERSION, 7),
12180        )
12181        .await;
12182
12183        let project_root = unique_project_root("consumer-capabilities-absent");
12184        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12185        let route_handler = handler.clone();
12186        let route_task = tokio::spawn(async move {
12187            route_handler
12188                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12189                .await
12190                .unwrap()
12191        });
12192        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12193            .await
12194            .unwrap()
12195            .unwrap();
12196        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12197        let ModuleControlRequest::RouteBind {
12198            consumer_capabilities,
12199            ..
12200        } = bind
12201        else {
12202            panic!("expected route.bind request, got {bind:?}");
12203        };
12204        assert_eq!(consumer_capabilities, None);
12205
12206        handler
12207            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12208            .await
12209            .unwrap();
12210        let route_response = route_task.await.unwrap();
12211        assert!(route_response.is_empty());
12212        let published = client_rx.recv().await.unwrap();
12213        assert!(matches!(
12214            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12215            ClientControlResponse::RouteOpen { .. }
12216        ));
12217    }
12218
12219    /// Opens a route to a freshly registered `aft` with `sent` as the
12220    /// route.open's role_versions, acks the bind, and returns the role_versions
12221    /// the module's bind carried.
12222    async fn bind_role_versions_for(
12223        sent: Option<BTreeMap<String, String>>,
12224        connection: u64,
12225    ) -> Option<BTreeMap<String, String>> {
12226        let registry = Arc::new(Registry::default());
12227        let forwarding = Arc::new(ForwardingTable::default());
12228        let handler =
12229            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12230        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12231        hello_via_sink(
12232            &handler,
12233            &module_ctx,
12234            &mut module_rx,
12235            hello_frame("aft", PROTOCOL_VERSION, 7),
12236        )
12237        .await;
12238        let project_root = unique_project_root("role-versions");
12239        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12240        let route_handler = handler.clone();
12241        let route_task = tokio::spawn(async move {
12242            route_handler
12243                .handle_control_frame(
12244                    &client_ctx,
12245                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
12246                )
12247                .await
12248                .unwrap()
12249        });
12250        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12251            .await
12252            .expect("a well-formed route.open reaches the module as a bind")
12253            .unwrap();
12254        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12255        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12256            panic!("expected route.bind request, got {bind:?}");
12257        };
12258        handler
12259            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12260            .await
12261            .unwrap();
12262        assert!(route_task.await.unwrap().is_empty());
12263        let published = client_rx.recv().await.unwrap();
12264        assert!(matches!(
12265            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12266            ClientControlResponse::RouteOpen { .. }
12267        ));
12268        role_versions
12269    }
12270
12271    #[tokio::test]
12272    async fn route_open_relays_role_versions_verbatim() {
12273        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12274        assert_eq!(
12275            bind_role_versions_for(Some(sent.clone()), 141).await,
12276            Some(sent)
12277        );
12278    }
12279
12280    /// An empty map declares nothing, so the provider sees no field rather
12281    /// than an empty object it would have to treat as a second "none".
12282    #[tokio::test]
12283    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12284        assert_eq!(bind_role_versions_for(None, 143).await, None);
12285        assert_eq!(
12286            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12287            None
12288        );
12289    }
12290
12291    #[tokio::test]
12292    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12293        let registry = Arc::new(Registry::default());
12294        let forwarding = Arc::new(ForwardingTable::default());
12295        let handler =
12296            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12297        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12298        hello_via_sink(
12299            &handler,
12300            &module_ctx,
12301            &mut module_rx,
12302            hello_frame("aft", PROTOCOL_VERSION, 7),
12303        )
12304        .await;
12305
12306        let nine: BTreeMap<String, String> = (0..9)
12307            .map(|index| (format!("role-{index}"), "v1".to_string()))
12308            .collect();
12309        for (label, malformed) in [
12310            (
12311                "invalid role name",
12312                role_versions(&[("Tool_Provider", "v1")]),
12313            ),
12314            ("invalid version", role_versions(&[("tool-provider", "v0")])),
12315            ("nine entries", nine),
12316        ] {
12317            // A refused open answers at once; one that reached the module
12318            // would wait for its bind ack and trip this timeout.
12319            let responses = tokio::time::timeout(
12320                Duration::from_secs(1),
12321                handler.handle_control_frame(
12322                    &route_ctx(ConnectionId::new(148)).0,
12323                    route_open_frame_with_role_versions(
12324                        404,
12325                        "aft",
12326                        unique_project_root("role-versions-malformed"),
12327                        Some(malformed),
12328                    ),
12329                ),
12330            )
12331            .await
12332            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12333            .unwrap();
12334            assert_eq!(responses.len(), 1, "{label}");
12335            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12336            let error = parse_error(&responses[0]);
12337            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12338            assert_eq!(
12339                error["detail"]["field"], "role_versions",
12340                "{label}: {error}"
12341            );
12342            assert!(
12343                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12344                "{label}: a malformed declaration is terminal"
12345            );
12346            assert!(
12347                module_rx.try_recv().is_err(),
12348                "{label}: the module must never see a bind"
12349            );
12350        }
12351    }
12352
12353    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
12354    /// consumer can tell this daemon forwards the field from one that would
12355    /// drop it.
12356    #[tokio::test]
12357    async fn route_role_versions_capability_is_advertised() {
12358        let handler = ControlHandler::new(Arc::new(Registry::default()));
12359        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12360        let ack = hello_via_sink(
12361            &handler,
12362            &ctx,
12363            &mut rx,
12364            hello_frame("m", PROTOCOL_VERSION, 1),
12365        )
12366        .await;
12367        let ack = parse_ack(&ack);
12368        assert!(
12369            ack.subc_capabilities
12370                .iter()
12371                .any(|c| c == "route-role-versions/v1"),
12372            "{:?}",
12373            ack.subc_capabilities
12374        );
12375
12376        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12377        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12378        let reply = handler
12379            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12380            .await
12381            .unwrap()
12382            .pop()
12383            .unwrap();
12384        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12385            serde_json::from_slice(&reply.body).unwrap()
12386        else {
12387            panic!("not a server.describe reply");
12388        };
12389        assert!(
12390            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12391            "{capabilities:?}"
12392        );
12393    }
12394
12395    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12396    /// catalog entry without `events` (the module declares none) from a daemon
12397    /// that never reports declared events.
12398    #[tokio::test]
12399    async fn catalog_events_capability_is_advertised() {
12400        let handler = ControlHandler::new(Arc::new(Registry::default()));
12401        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12402        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12403        let reply = handler
12404            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12405            .await
12406            .unwrap()
12407            .pop()
12408            .unwrap();
12409        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12410            serde_json::from_slice(&reply.body).unwrap()
12411        else {
12412            panic!("not a server.describe reply");
12413        };
12414        assert!(
12415            capabilities
12416                .iter()
12417                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12418            "{capabilities:?}"
12419        );
12420    }
12421
12422    #[tokio::test]
12423    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12424        let handler = ControlHandler::new(Arc::new(Registry::default()));
12425        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12426        let ack = hello_via_sink(
12427            &handler,
12428            &module,
12429            &mut module_rx,
12430            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12431        )
12432        .await;
12433        let ack = parse_ack(&ack);
12434        assert!(
12435            ack.subc_capabilities
12436                .iter()
12437                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12438            "{:?}",
12439            ack.subc_capabilities
12440        );
12441        assert!(ack
12442            .subc_capabilities
12443            .iter()
12444            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12445
12446        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12447        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12448        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12449        let reply = handler
12450            .handle_control_frame(&client, frame)
12451            .await
12452            .unwrap()
12453            .pop()
12454            .unwrap();
12455        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12456            serde_json::from_slice(&reply.body).unwrap()
12457        else {
12458            panic!("not a server.describe reply");
12459        };
12460        assert!(
12461            capabilities
12462                .iter()
12463                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12464            "{capabilities:?}"
12465        );
12466        assert!(capabilities
12467            .iter()
12468            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12469    }
12470
12471    #[tokio::test]
12472    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12473        let handler = ControlHandler::new(Arc::new(Registry::default()));
12474        let (module, mut rx) = route_ctx(ConnectionId::new(156));
12475        let ack = hello_via_sink(
12476            &handler,
12477            &module,
12478            &mut rx,
12479            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12480        )
12481        .await;
12482        assert!(parse_ack(&ack)
12483            .subc_capabilities
12484            .iter()
12485            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12486        assert!(parse_ack(&ack)
12487            .subc_capabilities
12488            .iter()
12489            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12490        let (client, _rx) = route_ctx(ConnectionId::new(157));
12491        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12492        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12493        let reply = handler
12494            .handle_control_frame(&client, frame)
12495            .await
12496            .unwrap()
12497            .pop()
12498            .unwrap();
12499        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12500            serde_json::from_slice(&reply.body).unwrap()
12501        else {
12502            panic!("server.describe reply")
12503        };
12504        assert!(capabilities
12505            .iter()
12506            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12507        assert!(capabilities
12508            .iter()
12509            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12510    }
12511
12512    #[tokio::test]
12513    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12514        let handler = ControlHandler::new(Arc::new(Registry::default()));
12515        let (module, mut rx) = route_ctx(ConnectionId::new(158));
12516        let ack = hello_via_sink(
12517            &handler,
12518            &module,
12519            &mut rx,
12520            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12521        )
12522        .await;
12523        assert!(parse_ack(&ack)
12524            .subc_capabilities
12525            .iter()
12526            .any(|cap| cap == "scope-workdir/v1"));
12527        let (client, _rx) = route_ctx(ConnectionId::new(159));
12528        let frame = Frame::build(
12529            FrameType::Request,
12530            control_flags(),
12531            0,
12532            0,
12533            5,
12534            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12535        )
12536        .unwrap();
12537        let reply = handler
12538            .handle_control_frame(&client, frame)
12539            .await
12540            .unwrap()
12541            .remove(0);
12542        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12543            serde_json::from_slice(&reply.body).unwrap()
12544        else {
12545            panic!("server.describe reply")
12546        };
12547        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12548    }
12549
12550    #[tokio::test]
12551    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12552        let handler = ControlHandler::new(Arc::new(Registry::default()));
12553        let (module, mut rx) = route_ctx(ConnectionId::new(154));
12554        let ack = hello_via_sink(
12555            &handler,
12556            &module,
12557            &mut rx,
12558            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12559        )
12560        .await;
12561        assert!(parse_ack(&ack)
12562            .subc_capabilities
12563            .iter()
12564            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12565        let (client, _rx) = route_ctx(ConnectionId::new(155));
12566        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12567        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12568        let reply = handler
12569            .handle_control_frame(&client, frame)
12570            .await
12571            .unwrap()
12572            .pop()
12573            .unwrap();
12574        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12575            serde_json::from_slice(&reply.body).unwrap()
12576        else {
12577            panic!("not a server.describe reply")
12578        };
12579        assert!(capabilities
12580            .iter()
12581            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12582    }
12583
12584    #[tokio::test]
12585    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12586        let registry = Arc::new(Registry::default());
12587        let forwarding = Arc::new(ForwardingTable::default());
12588        let handler =
12589            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12590                .with_health_probe_timeout(Duration::from_secs(5));
12591        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12592        hello_via_sink(
12593            &handler,
12594            &module_ctx,
12595            &mut module_rx,
12596            non_routable_hello_frame_with_control_ops(
12597                "mcp",
12598                300,
12599                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12600            ),
12601        )
12602        .await;
12603        assert!(registry
12604            .get_module("mcp")
12605            .unwrap()
12606            .unwrap()
12607            .manifest
12608            .provides
12609            .is_empty());
12610
12611        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12612        let route_response = handler
12613            .handle_control_frame(
12614                &route_client_ctx,
12615                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12616            )
12617            .await
12618            .unwrap();
12619        assert_eq!(route_response[0].header.ty, FrameType::Error);
12620        assert_eq!(
12621            parse_error(&route_response[0])["code"],
12622            "target_unavailable"
12623        );
12624        assert!(parse_error(&route_response[0])["message"]
12625            .as_str()
12626            .unwrap()
12627            .contains("does not provide the requested target"));
12628        assert!(module_rx.try_recv().is_err());
12629
12630        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12631        let health_handler = handler.clone();
12632        let health_task = tokio::spawn(async move {
12633            health_handler
12634                .handle_control_frame(
12635                    &health_client_ctx,
12636                    supervisor_health_probe_frame(302, "mcp"),
12637                )
12638                .await
12639                .unwrap()
12640        });
12641        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12642            .await
12643            .unwrap()
12644            .unwrap();
12645        assert_eq!(
12646            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12647            ModuleControlRequest::HealthCheck {}
12648        );
12649        handler
12650            .handle_control_frame(
12651                &module_ctx,
12652                health_response(health_frame.header.corr, HealthStatus::Ok),
12653            )
12654            .await
12655            .unwrap();
12656        let health_response = health_task.await.unwrap();
12657        assert_eq!(health_response[0].header.ty, FrameType::Response);
12658        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12659            ClientControlResponse::SupervisorHealthProbe {
12660                module_id, status, ..
12661            } => {
12662                assert_eq!(module_id, "mcp");
12663                assert_eq!(status, HealthStatus::Ok);
12664            }
12665            other => panic!("unexpected health response: {other:?}"),
12666        }
12667
12668        // Exercise the forwarding cleanup path directly while leaving the registry
12669        // advertisement in place. If cleanup leaves a stale control sink behind,
12670        // the next probe will enqueue onto it and wait for the long probe timeout
12671        // instead of returning an immediate no-connection error.
12672        forwarding
12673            .cleanup_connection(module_ctx.connection_id)
12674            .unwrap();
12675        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12676        let cleanup_response = tokio::time::timeout(
12677            Duration::from_millis(200),
12678            handler.handle_control_frame(
12679                &cleanup_probe_ctx,
12680                supervisor_health_probe_frame(303, "mcp"),
12681            ),
12682        )
12683        .await
12684        .expect("probe should fail immediately when the control lane is gone")
12685        .unwrap();
12686        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12687        assert_eq!(
12688            parse_error(&cleanup_response[0])["code"],
12689            "target_unavailable"
12690        );
12691        assert!(parse_error(&cleanup_response[0])["message"]
12692            .as_str()
12693            .unwrap()
12694            .contains("no module connection"));
12695
12696        handler
12697            .cleanup_connection(module_ctx.connection_id)
12698            .unwrap();
12699    }
12700
12701    #[tokio::test]
12702    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12703        let registry = Arc::new(Registry::default());
12704        let supervisor_handle = SupervisorHandle::new();
12705        let supervisor =
12706            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12707                .with_handle(supervisor_handle.clone())
12708                .with_connection_file_path(
12709                    std::env::temp_dir()
12710                        .join(format!("subc-route-open-warming-{}", std::process::id())),
12711                );
12712        let module = supervisor
12713            .supervise_configured(
12714                ModuleSpec {
12715                    module_id: "warming".to_string(),
12716                    program: fake_aft_stub_path(),
12717                    args: Vec::new(),
12718                    env: Vec::new(),
12719                    reserved: false,
12720                    reserved_prefixes: Vec::new(),
12721                    protocol: ModuleProtocol::Subc,
12722                    overlap: Default::default(),
12723                },
12724                true,
12725            )
12726            .unwrap();
12727        assert_eq!(module.state().unwrap(), ModuleState::Running);
12728
12729        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12730        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12731        let response = handler
12732            .handle_control_frame(
12733                &ctx,
12734                route_open_frame(304, "warming", unique_project_root("warming")),
12735            )
12736            .await
12737            .unwrap();
12738        module.stop().await.unwrap();
12739
12740        assert_eq!(response[0].header.ty, FrameType::Error);
12741        let error = parse_error(&response[0]);
12742        assert_eq!(error["code"], "module_warming");
12743        assert!(error["message"]
12744            .as_str()
12745            .unwrap()
12746            .contains("state=running, enabled=true, live=false"));
12747    }
12748
12749    #[test]
12750    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12751        let handler = ControlHandler::new(Arc::new(Registry::default()));
12752        let capture = EventCapture::default();
12753        let _subscriber =
12754            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12755        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12756        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12757        let pending = (0..limit).collect::<Vec<_>>();
12758        let response = handler
12759            .route_open_capacity_refusal(
12760                &ctx,
12761                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12762                "busy",
12763                pending.len(),
12764                limit,
12765            )
12766            .unwrap();
12767        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12768        let event = capture
12769            .events()
12770            .into_iter()
12771            .find(|event| {
12772                event.target == "control"
12773                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12774            })
12775            .expect("connection admission refusal event");
12776        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12777        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12778        // Refused before `handle_route_open` parses the request, so the caller
12779        // fields are read from the frame itself.
12780        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
12781        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
12782    }
12783
12784    #[test]
12785    fn route_open_target_cap_logs_admission_reason_and_capacity() {
12786        let handler = ControlHandler::new(Arc::new(Registry::default()));
12787        let capture = EventCapture::default();
12788        let _subscriber =
12789            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12790        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12791        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12792        let guards = (0..limit)
12793            .map(|_| {
12794                handler
12795                    .route_bind_concurrency
12796                    .try_admit("busy", limit)
12797                    .unwrap()
12798            })
12799            .collect::<Vec<_>>();
12800        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12801            Err(in_flight) => in_flight,
12802            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12803        };
12804        let response = handler
12805            .route_open_target_capacity_refusal(
12806                &ctx,
12807                &RouteOpenCaller::unknown(),
12808                &route_open_frame(397, "busy", unique_project_root("target-cap")),
12809                "busy",
12810                in_flight,
12811            )
12812            .unwrap();
12813        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12814        let event = capture
12815            .events()
12816            .into_iter()
12817            .find(|event| {
12818                event.target == "control"
12819                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12820            })
12821            .expect("target admission refusal event");
12822        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12823        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12824        drop(guards);
12825    }
12826
12827    #[cfg(unix)]
12828    #[tokio::test(start_paused = true)]
12829    async fn supervisor_list_shows_a_failed_modules_next_retry() {
12830        let registry = Arc::new(Registry::default());
12831        let handle = SupervisorHandle::new();
12832        let supervisor =
12833            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
12834                .with_handle(handle.clone());
12835        let module = supervisor
12836            .spawn(crate::ModuleSpec {
12837                module_id: "retry-list".into(),
12838                program: PathBuf::from("/bin/sh"),
12839                args: vec!["-c".into(), "exit 17".into()],
12840                env: vec![],
12841                reserved: false,
12842                reserved_prefixes: vec![],
12843                protocol: ModuleProtocol::None,
12844                overlap: Default::default(),
12845            })
12846            .unwrap();
12847        let deadline = std::time::Instant::now() + Duration::from_secs(10);
12848        while module.state().unwrap() != crate::ModuleState::Failed {
12849            assert!(
12850                std::time::Instant::now() < deadline,
12851                "waiting for budget failure"
12852            );
12853            tokio::task::yield_now().await;
12854        }
12855        let handler = ControlHandler::new(registry).with_supervisor(handle);
12856        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12857        let replies = handler.handle_supervisor_list(frame).await.unwrap();
12858        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12859        assert_eq!(body["modules"][0]["state"], "failed");
12860        let retry = body["modules"][0]["next_retry_at_ms"]
12861            .as_u64()
12862            .expect("scheduled retry on the wire");
12863        let now = std::time::SystemTime::now()
12864            .duration_since(std::time::UNIX_EPOCH)
12865            .unwrap()
12866            .as_millis() as u64;
12867        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
12868        module.set_enabled(false).await.unwrap();
12869        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12870        let replies = handler.handle_supervisor_list(frame).await.unwrap();
12871        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12872        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
12873    }
12874
12875    #[tokio::test]
12876    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12877        let registry = Arc::new(Registry::default());
12878        let handle = SupervisorHandle::new();
12879        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12880            .with_handle(handle.clone());
12881        supervisor
12882            .supervise_configured(
12883                crate::ModuleSpec {
12884                    module_id: "drift".into(),
12885                    program: PathBuf::from("disabled-module"),
12886                    args: Vec::new(),
12887                    env: Vec::new(),
12888                    reserved: false,
12889                    reserved_prefixes: Vec::new(),
12890                    protocol: ModuleProtocol::Subc,
12891                    overlap: Default::default(),
12892                },
12893                false,
12894            )
12895            .unwrap();
12896        let handler = ControlHandler::new(registry).with_supervisor(handle);
12897        for configured in [None, Some(false), Some(true)] {
12898            handler.capability_evaluator.configure(
12899                configured.map(|enabled| ("drift".to_string(), enabled)),
12900                BTreeMap::new(),
12901            );
12902            let frame =
12903                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12904            let replies = handler
12905                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12906                .await
12907                .unwrap();
12908            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12909            assert!(body.get("configured_enabled").is_none());
12910            assert_eq!(body["op"], "supervisor.ack");
12911            let frame =
12912                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12913            let replies = handler.handle_supervisor_list(frame).await.unwrap();
12914            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12915            assert_eq!(
12916                body["modules"][0].get("configured_enabled"),
12917                configured.map(Value::Bool).as_ref()
12918            );
12919        }
12920    }
12921
12922    #[tokio::test(flavor = "current_thread")]
12923    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12924        let handler = ControlHandler::new(Arc::new(Registry::default()));
12925        let capture = EventCapture::default();
12926        let _subscriber =
12927            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12928        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12929        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12930
12931        let _ = handler
12932            .handle_client_control_request(
12933                &ctx,
12934                frame,
12935                ClientControlRequest::SupervisorSetEnabled {
12936                    module_id: "broca".to_string(),
12937                    enabled: false,
12938                },
12939                &mut None,
12940            )
12941            .await
12942            .unwrap();
12943
12944        let events = capture
12945            .events()
12946            .into_iter()
12947            .filter(|event| {
12948                event.target == "control"
12949                    && event.fields.get("message").map(String::as_str)
12950                        == Some("supervisor request received")
12951            })
12952            .collect::<Vec<_>>();
12953        assert_eq!(events.len(), 1, "one supervisor request log line");
12954        let event = &events[0];
12955        assert_eq!(event.level, tracing::Level::INFO);
12956        assert_eq!(
12957            event.fields.get("op"),
12958            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12959        );
12960        assert_eq!(
12961            event.fields.get("module_id"),
12962            Some(&"\"broca\"".to_string())
12963        );
12964        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12965        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12966        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12967    }
12968
12969    #[tokio::test(flavor = "current_thread")]
12970    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12971        let registry = Arc::new(Registry::default());
12972        let module_connection = ConnectionId::new(303);
12973        registry
12974            .register_with_control_ops(
12975                manifest("aft", PROTOCOL_VERSION),
12976                PROTOCOL_VERSION,
12977                module_connection,
12978                module_baseline_control_ops(),
12979            )
12980            .unwrap();
12981        let handler = ControlHandler::new(registry);
12982        let capture = EventCapture::default();
12983        let _subscriber =
12984            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12985        let (ctx, _rx) = route_ctx(module_connection);
12986        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12987
12988        let _ = handler
12989            .handle_client_control_request(
12990                &ctx,
12991                frame,
12992                ClientControlRequest::SupervisorSetEnabled {
12993                    module_id: "broca".to_string(),
12994                    enabled: false,
12995                },
12996                &mut None,
12997            )
12998            .await
12999            .unwrap();
13000
13001        let callers = capture
13002            .events()
13003            .into_iter()
13004            .filter(|event| {
13005                event.fields.get("message").map(String::as_str)
13006                    == Some("supervisor request received")
13007            })
13008            .map(|event| event.fields.get("caller").cloned())
13009            .collect::<Vec<_>>();
13010        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13011    }
13012
13013    #[tokio::test(flavor = "current_thread")]
13014    async fn supervisor_list_does_not_log_request_received() {
13015        let handler = ControlHandler::new(Arc::new(Registry::default()));
13016        let capture = EventCapture::default();
13017        let _subscriber =
13018            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13019        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13020        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13021
13022        handler
13023            .handle_client_control_request(
13024                &ctx,
13025                frame,
13026                ClientControlRequest::SupervisorList {},
13027                &mut None,
13028            )
13029            .await
13030            .unwrap();
13031
13032        assert!(capture.events().into_iter().all(|event| {
13033            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13034        }));
13035    }
13036
13037    #[tokio::test(flavor = "current_thread")]
13038    async fn supervisor_rescan_preview_does_not_log_request_received() {
13039        let handler = ControlHandler::new(Arc::new(Registry::default()));
13040        let capture = EventCapture::default();
13041        let _subscriber =
13042            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13043        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13044        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13045
13046        handler
13047            .handle_client_control_request(
13048                &ctx,
13049                frame,
13050                ClientControlRequest::SupervisorRescan { preview: true },
13051                &mut None,
13052            )
13053            .await
13054            .unwrap();
13055
13056        assert!(capture.events().into_iter().all(|event| {
13057            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13058        }));
13059    }
13060
13061    struct ApproveOperator;
13062    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13063        fn prompt(
13064            &self,
13065            _: &str,
13066            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13067        ) -> crate::operator_confirm::ProviderResult {
13068            crate::operator_confirm::ProviderResult::Approved
13069        }
13070    }
13071
13072    #[tokio::test]
13073    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13074        assert_eq!(
13075            ControlHandler::default().privileged_operator_prompt,
13076            PrivilegedOperatorPrompt::Log
13077        );
13078        for (initial, changed, running) in [
13079            ("log", "enforce", PrivilegedOperatorPrompt::Log),
13080            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13081        ] {
13082            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13083            let path = dir.join("subc.jsonc");
13084            std::fs::write(
13085                &path,
13086                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13087            )
13088            .unwrap();
13089            let config = crate::daemon_config::load(&path).unwrap().unwrap();
13090            let registry = Arc::new(Registry::default());
13091            let handle = SupervisorHandle::new();
13092            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13093                .with_handle(handle.clone());
13094            let handler = ControlHandler::new(registry)
13095                .with_supervisor(handle)
13096                .with_privileged_operator_prompt(config.privileged_operator_prompt)
13097                .with_supervisor_rescan(supervisor, &path, config.port);
13098            handler.forwarding.operator_confirms().configure(
13099                Arc::new(ApproveOperator),
13100                crate::operator_confirm::OperatorLimits::default(),
13101            );
13102            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13103
13104            for (disk_mode, needs_restart) in [
13105                (Some(initial), false),
13106                (Some(changed), true),
13107                (Some(changed), true),
13108                (None, running == PrivilegedOperatorPrompt::Enforce),
13109                (Some(initial), false),
13110            ] {
13111                let mut doc = json!({"version":1});
13112                if let Some(mode) = disk_mode {
13113                    doc["privileged_operator_prompt"] = json!(mode);
13114                }
13115                std::fs::write(&path, doc.to_string()).unwrap();
13116                for preview in [true, false] {
13117                    let request = ClientControlRequest::SupervisorRescan { preview };
13118                    let frame = Frame::build(
13119                        FrameType::Request,
13120                        control_flags(),
13121                        0,
13122                        0,
13123                        1,
13124                        serde_json::to_vec(&request).unwrap(),
13125                    )
13126                    .unwrap();
13127                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13128                    assert_eq!(frames.len(), 1);
13129                    let ClientControlResponse::SupervisorRescan { result } =
13130                        serde_json::from_slice(&frames[0].body).unwrap()
13131                    else {
13132                        panic!("expected a rescan response");
13133                    };
13134                    let expected = if needs_restart {
13135                        vec!["privileged_operator_prompt"]
13136                    } else {
13137                        Vec::new()
13138                    };
13139                    assert_eq!(result.restart_required, expected);
13140                    assert_eq!(handler.privileged_operator_prompt, running);
13141                    assert_eq!(
13142                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13143                        running
13144                    );
13145                }
13146            }
13147        }
13148    }
13149
13150    #[tokio::test]
13151    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13152        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13153        let path = dir.join("subc.jsonc");
13154        let registry = Arc::new(Registry::default());
13155        let handle = SupervisorHandle::new();
13156        let supervisor =
13157            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13158        let handler = ControlHandler::new(registry)
13159            .with_supervisor(handle)
13160            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13161            .with_supervisor_rescan(supervisor, &path, None);
13162        handler.forwarding.operator_confirms().configure(
13163            Arc::new(ApproveOperator),
13164            crate::operator_confirm::OperatorLimits::default(),
13165        );
13166        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13167        for preview in [true, false] {
13168            std::fs::write(
13169                &path,
13170                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13171            )
13172            .unwrap();
13173            let request = ClientControlRequest::SupervisorRescan { preview };
13174            let frame = Frame::build(
13175                FrameType::Request,
13176                control_flags(),
13177                0,
13178                0,
13179                1,
13180                serde_json::to_vec(&request).unwrap(),
13181            )
13182            .unwrap();
13183            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13184            assert_eq!(frames.len(), 1);
13185            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13186            assert_eq!(
13187                handler.privileged_operator_prompt,
13188                PrivilegedOperatorPrompt::Enforce
13189            );
13190        }
13191    }
13192
13193    /// One wire code has several senders, so the refusal line names the check
13194    /// that refused. This drives the shared refusal path for ordinary refusals
13195    /// with an unregistered
13196    /// target and requires the branch label on the event.
13197    #[tokio::test]
13198    async fn route_open_refusal_names_the_check_that_refused() {
13199        let handler = ControlHandler::new(Arc::new(Registry::default()));
13200        let capture = EventCapture::default();
13201        let _subscriber =
13202            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13203        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13204        let response = handler
13205            .handle_control_frame(
13206                &ctx,
13207                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13208            )
13209            .await
13210            .unwrap();
13211
13212        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13213        let event = capture
13214            .events()
13215            .into_iter()
13216            .find(|event| {
13217                event.target == "control"
13218                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13219            })
13220            .expect("route.open refusal event");
13221        assert_eq!(
13222            event.fields.get("reason"),
13223            Some(&"\"not_registered\"".to_string())
13224        );
13225    }
13226
13227    #[tokio::test]
13228    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13229        let registry = Arc::new(Registry::default());
13230        let supervisor_handle = SupervisorHandle::new();
13231        let supervisor =
13232            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13233                .with_handle(supervisor_handle.clone())
13234                .with_connection_file_path(std::env::temp_dir().join(format!(
13235                    "subc-route-open-refusal-info-{}",
13236                    std::process::id()
13237                )));
13238        let module = supervisor
13239            .supervise_configured(
13240                ModuleSpec {
13241                    module_id: "warming".to_string(),
13242                    program: fake_aft_stub_path(),
13243                    args: Vec::new(),
13244                    env: Vec::new(),
13245                    reserved: false,
13246                    reserved_prefixes: Vec::new(),
13247                    protocol: ModuleProtocol::Subc,
13248                    overlap: Default::default(),
13249                },
13250                true,
13251            )
13252            .unwrap();
13253        assert_eq!(module.state().unwrap(), ModuleState::Running);
13254
13255        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13256        assert!(handler
13257            .counters()
13258            .snapshot()
13259            .get("route_open_refused_by_code")
13260            .is_none());
13261        let capture = EventCapture::default();
13262        let _subscriber =
13263            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13264        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13265        let response = handler
13266            .handle_control_frame(
13267                &ctx,
13268                route_open_frame(394, "warming", unique_project_root("refusal-info")),
13269            )
13270            .await
13271            .unwrap();
13272        module.stop().await.unwrap();
13273
13274        assert_eq!(parse_error(&response[0])["code"], "module_warming");
13275        let event = capture
13276            .events()
13277            .into_iter()
13278            .find(|event| {
13279                event.target == "control"
13280                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13281            })
13282            .expect("route.open refusal event");
13283        assert_eq!(
13284            event.fields.get("module_id"),
13285            Some(&"\"warming\"".to_string())
13286        );
13287        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13288        assert_eq!(
13289            event.fields.get("reason"),
13290            Some(&"\"supervised_not_registered\"".to_string())
13291        );
13292        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13293        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13294        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13295        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13296        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13297        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13298        assert_eq!(
13299            handler.counters().snapshot()["route_open_refused_by_code"],
13300            json!({ "module_warming": 1 })
13301        );
13302    }
13303
13304    const OUTAGE_START: &str = "route.open refusing module: not serving";
13305    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13306
13307    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13308        capture
13309            .events()
13310            .into_iter()
13311            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13312            .collect()
13313    }
13314
13315    fn supervise_stub(
13316        registry: &Arc<Registry>,
13317        module_id: &str,
13318        enabled: bool,
13319    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13320        let supervisor_handle = SupervisorHandle::new();
13321        let supervisor =
13322            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13323                .with_handle(supervisor_handle.clone())
13324                .with_connection_file_path(std::env::temp_dir().join(format!(
13325                    "subc-route-outage-{module_id}-{}",
13326                    std::process::id()
13327                )));
13328        let module = supervisor
13329            .supervise_configured(
13330                ModuleSpec {
13331                    module_id: module_id.to_string(),
13332                    program: fake_aft_stub_path(),
13333                    args: Vec::new(),
13334                    env: Vec::new(),
13335                    reserved: false,
13336                    reserved_prefixes: Vec::new(),
13337                    protocol: ModuleProtocol::Subc,
13338                    overlap: Default::default(),
13339                },
13340                enabled,
13341            )
13342            .unwrap();
13343        (supervisor_handle, module)
13344    }
13345
13346    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13347        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13348            module_id: module_id.to_string(),
13349            drain_timeout_ms: Some(50),
13350        })
13351        .unwrap();
13352        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13353    }
13354
13355    /// Two handlers built over one forwarding table must share one outage
13356    /// tracker; separate trackers would each log their own opening line for
13357    /// the same outage.
13358    #[test]
13359    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13360        let registry = Arc::new(Registry::default());
13361        let forwarding = Arc::new(ForwardingTable::default());
13362        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13363        let second = ControlHandler::with_forwarding(registry, forwarding);
13364        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13365    }
13366
13367    /// A client can name any module id it likes. Refusing an unknown one,
13368    /// however often, must not create outage state or outage lines, or the
13369    /// tracker would be a memory sink any client could fill.
13370    #[tokio::test(flavor = "current_thread")]
13371    async fn route_open_unknown_module_refusals_add_no_outage_state() {
13372        let handler = ControlHandler::new(Arc::new(Registry::default()));
13373        let capture = EventCapture::default();
13374        let _subscriber =
13375            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13376        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13377        for corr in 0..8 {
13378            let response = handler
13379                .handle_control_frame(
13380                    &ctx,
13381                    route_open_frame(
13382                        380 + corr,
13383                        &format!("nobody-{corr}"),
13384                        unique_project_root("outage-unknown"),
13385                    ),
13386                )
13387                .await
13388                .unwrap();
13389            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13390        }
13391
13392        assert_eq!(handler.route_outages.tracked_module_count(), 0);
13393        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13394        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13395    }
13396
13397    /// Drives the refusal path end to end: a supervised module that served
13398    /// before and stopped being registered with no instruction to stop is a
13399    /// WARN, and the same module refused after an operator `supervisor.restart`
13400    /// is an INFO.
13401    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13402    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13403        let registry = Arc::new(Registry::default());
13404        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
13405        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13406        let capture = EventCapture::default();
13407        let _subscriber =
13408            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13409        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13410        // The stub never registers, so pretend it served once: otherwise every
13411        // refusal would fall in its startup window.
13412        handler.route_outages.record_accepted("outage-restart");
13413
13414        let response = handler
13415            .handle_control_frame(
13416                &ctx,
13417                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13418            )
13419            .await
13420            .unwrap();
13421        assert_eq!(response[0].header.ty, FrameType::Error);
13422        let starts = outage_lines(&capture, OUTAGE_START);
13423        assert_eq!(starts.len(), 1, "{starts:?}");
13424        assert_eq!(starts[0].level, tracing::Level::WARN);
13425        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13426        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13427        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13428        handler.route_outages.record_accepted("outage-restart");
13429        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13430
13431        let restart = handler
13432            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13433            .await
13434            .unwrap();
13435        assert_eq!(
13436            restart[0].header.ty,
13437            FrameType::Response,
13438            "{:?}",
13439            parse_error(&restart[0])
13440        );
13441        handler
13442            .handle_control_frame(
13443                &ctx,
13444                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13445            )
13446            .await
13447            .unwrap();
13448        module.stop().await.unwrap();
13449
13450        let starts = outage_lines(&capture, OUTAGE_START);
13451        assert_eq!(starts.len(), 2, "{starts:?}");
13452        assert_eq!(starts[1].level, tracing::Level::INFO);
13453        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13454    }
13455
13456    /// A restart refused before it touched the module (here: the module is
13457    /// disabled) must clear its operator mark, so the next real outage is
13458    /// still reported as a warning.
13459    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13460    async fn failed_operator_restart_leaves_no_operator_mark() {
13461        let registry = Arc::new(Registry::default());
13462        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
13463        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13464        let capture = EventCapture::default();
13465        let _subscriber =
13466            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13467        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13468        handler.route_outages.record_accepted("outage-disabled");
13469
13470        let restart = handler
13471            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13472            .await
13473            .unwrap();
13474        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13475        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13476
13477        handler
13478            .handle_control_frame(
13479                &ctx,
13480                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13481            )
13482            .await
13483            .unwrap();
13484        let starts = outage_lines(&capture, OUTAGE_START);
13485        assert_eq!(starts.len(), 1, "{starts:?}");
13486        assert_eq!(starts[0].level, tracing::Level::WARN);
13487    }
13488
13489    #[tokio::test(flavor = "current_thread")]
13490    async fn route_open_unknown_module_escapes_target_module_id() {
13491        let handler = ControlHandler::new(Arc::new(Registry::default()));
13492        let capture = EventCapture::default();
13493        let _subscriber =
13494            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13495        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13496        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13497        let response = handler
13498            .handle_control_frame(
13499                &ctx,
13500                route_open_frame(
13501                    395,
13502                    hostile_module_id,
13503                    unique_project_root("hostile-target-module-id"),
13504                ),
13505            )
13506            .await
13507            .unwrap();
13508
13509        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13510        let event = capture
13511            .events()
13512            .into_iter()
13513            .find(|event| {
13514                event.target == "control"
13515                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13516            })
13517            .expect("route.open unknown-module refusal event");
13518        let logged = event.fields.get("module_id").expect("module_id field");
13519        assert!(!logged.bytes().any(|byte| byte < 0x20));
13520        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13521    }
13522
13523    #[tokio::test(flavor = "current_thread")]
13524    async fn route_open_module_rejection_uses_daemon_counter_key() {
13525        let registry = Arc::new(Registry::default());
13526        let forwarding = Arc::new(ForwardingTable::default());
13527        let handler =
13528            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13529        let module_connection = ConnectionId::new(95);
13530        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13531        hello_via_sink(
13532            &handler,
13533            &module_ctx,
13534            &mut module_rx,
13535            hello_frame("aft", PROTOCOL_VERSION, 395),
13536        )
13537        .await;
13538
13539        let client_connection = ConnectionId::new(96);
13540        let (client_ctx, _client_rx) = route_ctx(client_connection);
13541        let capture = EventCapture::default();
13542        let _subscriber =
13543            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13544        let (route_task, bind) = relay_route_open(
13545            &handler,
13546            client_connection,
13547            &client_ctx.egress,
13548            &mut module_rx,
13549            396,
13550            "aft",
13551            "hostile-module-code",
13552        )
13553        .await;
13554        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13555        let rejection = Frame::build(
13556            FrameType::Error,
13557            control_flags(),
13558            0,
13559            0,
13560            bind.header.corr,
13561            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13562        )
13563        .unwrap();
13564        handler
13565            .handle_control_frame(&module_ctx, rejection)
13566            .await
13567            .unwrap();
13568
13569        let response = route_task.await.unwrap();
13570        assert_eq!(parse_error(&response[0])["code"], hostile_code);
13571        let counters = handler.counters().snapshot();
13572        assert_eq!(
13573            counters["route_open_refused_by_code"],
13574            json!({ "module_rejected": 1 })
13575        );
13576        assert!(counters["route_open_refused_by_code"]
13577            .get(hostile_code)
13578            .is_none());
13579
13580        let event = capture
13581            .events()
13582            .into_iter()
13583            .find(|event| {
13584                event.target == "control"
13585                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13586            })
13587            .expect("route.open module-rejection refusal event");
13588        let logged = event.fields.get("module_code").expect("module_code field");
13589        assert!(!logged.bytes().any(|byte| byte < 0x20));
13590        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13591    }
13592
13593    #[tokio::test]
13594    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13595        let registry = Arc::new(Registry::default());
13596        let supervisor_handle = SupervisorHandle::new();
13597        let missing_program = std::env::temp_dir().join(format!(
13598            "subc-route-open-missing-program-{}",
13599            std::process::id()
13600        ));
13601        let supervisor =
13602            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13603                .with_handle(supervisor_handle.clone());
13604        let module = supervisor
13605            .supervise_configured(
13606                ModuleSpec {
13607                    module_id: "failed".to_string(),
13608                    program: missing_program,
13609                    args: Vec::new(),
13610                    env: Vec::new(),
13611                    reserved: false,
13612                    reserved_prefixes: Vec::new(),
13613                    protocol: ModuleProtocol::Subc,
13614                    overlap: Default::default(),
13615                },
13616                true,
13617            )
13618            .unwrap();
13619        assert_eq!(module.state().unwrap(), ModuleState::Failed);
13620
13621        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13622        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13623        let response = handler
13624            .handle_control_frame(
13625                &ctx,
13626                route_open_frame(305, "failed", unique_project_root("failed")),
13627            )
13628            .await
13629            .unwrap();
13630
13631        assert_eq!(response[0].header.ty, FrameType::Error);
13632        let error = parse_error(&response[0]);
13633        assert_eq!(error["code"], "target_unavailable");
13634        assert!(error["message"]
13635            .as_str()
13636            .unwrap()
13637            .contains("state=failed, enabled=true, live=false"));
13638    }
13639
13640    #[tokio::test]
13641    async fn route_open_role_mismatch_remains_target_unavailable() {
13642        let registry = Arc::new(Registry::default());
13643        let handler = ControlHandler::new(Arc::clone(&registry));
13644        handler
13645            .handle_control(
13646                ConnectionId::new(41),
13647                non_routable_hello_frame_with_control_ops("health-only", 306, None),
13648            )
13649            .unwrap();
13650
13651        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13652        let response = handler
13653            .handle_control_frame(
13654                &ctx,
13655                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13656            )
13657            .await
13658            .unwrap();
13659
13660        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13661        assert!(parse_error(&response[0])["message"]
13662            .as_str()
13663            .unwrap()
13664            .contains("does not provide the requested target"));
13665    }
13666
13667    #[tokio::test]
13668    async fn route_open_inactive_registration_remains_target_unavailable() {
13669        let registry = Arc::new(Registry::default());
13670        let handler = ControlHandler::new(Arc::clone(&registry));
13671        handler
13672            .handle_control(
13673                ConnectionId::new(43),
13674                hello_frame("inactive", PROTOCOL_VERSION, 308),
13675            )
13676            .unwrap();
13677        assert!(registry
13678            .set_module_state_for_test("inactive", ChannelState::Closed)
13679            .unwrap());
13680
13681        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13682        let response = handler
13683            .handle_control_frame(
13684                &ctx,
13685                route_open_frame(309, "inactive", unique_project_root("inactive")),
13686            )
13687            .await
13688            .unwrap();
13689
13690        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13691        assert!(parse_error(&response[0])["message"]
13692            .as_str()
13693            .unwrap()
13694            .contains("is not active"));
13695    }
13696
13697    #[tokio::test]
13698    async fn late_health_reply_is_recorded_through_the_module_response_path() {
13699        let registry = Arc::new(Registry::default());
13700        let forwarding = Arc::new(ForwardingTable::default());
13701        let supervisor_handle = SupervisorHandle::new();
13702        let supervisor =
13703            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
13704                .with_forwarding(Arc::clone(&forwarding))
13705                .with_handle(supervisor_handle.clone());
13706        let module = supervisor
13707            .supervise_configured(
13708                crate::ModuleSpec {
13709                    module_id: "late-health-response".to_string(),
13710                    program: PathBuf::from("disabled-module"),
13711                    args: Vec::new(),
13712                    env: Vec::new(),
13713                    reserved: false,
13714                    reserved_prefixes: Vec::new(),
13715                    protocol: ModuleProtocol::Subc,
13716                    overlap: Default::default(),
13717                },
13718                false,
13719            )
13720            .unwrap();
13721        let handler =
13722            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13723                .with_supervisor(supervisor_handle);
13724        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13725        handler
13726            .handle_control_frame(
13727                &module_ctx,
13728                hello_frame_with_control_ops(
13729                    "late-health-response",
13730                    PROTOCOL_VERSION,
13731                    7,
13732                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13733                ),
13734            )
13735            .await
13736            .unwrap();
13737        let probe_started_at = Instant::now() - Duration::from_millis(80);
13738        let pending = forwarding
13739            .begin_health_probe_rpc_for(
13740                "late-health-response",
13741                MODULE_CONTROL_OP_HEALTH_CHECK,
13742                probe_started_at,
13743                Instant::now() - Duration::from_millis(1),
13744            )
13745            .unwrap();
13746        assert!(forwarding
13747            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13748            .unwrap());
13749
13750        let responses = handler
13751            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13752            .await
13753            .unwrap();
13754
13755        assert!(responses.is_empty());
13756        let health = module.status().unwrap().health;
13757        assert_eq!(health.late_answer_count, 1);
13758        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13759    }
13760
13761    #[tokio::test]
13762    async fn health_probe_timeout_and_module_death_are_typed() {
13763        let registry = Arc::new(Registry::default());
13764        let forwarding = Arc::new(ForwardingTable::default());
13765        let handler =
13766            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13767                .with_health_probe_timeout(Duration::from_millis(50));
13768        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13769        hello_via_sink(
13770            &handler,
13771            &module_ctx,
13772            &mut module_rx,
13773            hello_frame_with_control_ops(
13774                "aft",
13775                PROTOCOL_VERSION,
13776                7,
13777                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13778            ),
13779        )
13780        .await;
13781
13782        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13783        let responses = handler
13784            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13785            .await
13786            .unwrap();
13787        assert_eq!(responses[0].header.ty, FrameType::Error);
13788        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13789        let _ = module_rx.try_recv();
13790
13791        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13792        let health_handler = handler.clone();
13793        let death_task = tokio::spawn(async move {
13794            health_handler
13795                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13796                .await
13797                .unwrap()
13798        });
13799        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13800            .await
13801            .unwrap()
13802            .unwrap();
13803        handler
13804            .cleanup_connection(module_ctx.connection_id)
13805            .unwrap();
13806        let responses = death_task.await.unwrap();
13807        assert_eq!(responses[0].header.ty, FrameType::Error);
13808        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13809    }
13810
13811    #[test]
13812    fn hello_requires_exact_protocol_version() {
13813        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13814            let registry = Arc::new(Registry::default());
13815            let handler = ControlHandler::new(Arc::clone(&registry));
13816            let responses = handler
13817                .handle_control(
13818                    ConnectionId::new(connection),
13819                    hello_frame("aft", offered, 9),
13820                )
13821                .unwrap();
13822
13823            assert_eq!(responses.len(), 1);
13824            assert_eq!(responses[0].header.ty, FrameType::Error);
13825            let error = parse_error(&responses[0]);
13826            assert_eq!(error["code"], "version_unsupported");
13827            assert!(registry.get_module("aft").unwrap().is_none());
13828            assert_eq!(registry.active_registration_count().unwrap(), 0);
13829        }
13830    }
13831
13832    #[test]
13833    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13834        let registry = Arc::new(Registry::default());
13835        let forwarding = Arc::new(ForwardingTable::default());
13836        let handler =
13837            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13838        let module_connection = ConnectionId::new(301);
13839        let registration = registry
13840            .register_with_control_ops(
13841                manifest("aft-push", PROTOCOL_VERSION),
13842                PROTOCOL_VERSION,
13843                module_connection,
13844                module_baseline_control_ops(),
13845            )
13846            .unwrap();
13847        let (module_tx, _module_rx) = mpsc::channel(8);
13848        let endpoint = forwarding
13849            .register_module_connection(
13850                module_connection,
13851                "aft-push".to_string(),
13852                PROTOCOL_VERSION,
13853                manifest_concurrency(&registration.manifest),
13854                FrameSink::new(module_tx),
13855            )
13856            .unwrap();
13857
13858        // A push op this version does not know is ignored (forward-compat), not errored.
13859        let unknown = Frame::build(
13860            FrameType::Push,
13861            control_flags(),
13862            0,
13863            0,
13864            5,
13865            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13866        )
13867        .unwrap();
13868        let out = handler.handle_status_update(endpoint, unknown).unwrap();
13869        assert!(
13870            out.is_empty(),
13871            "unknown push op must be ignored, got {out:?}"
13872        );
13873
13874        // A malformed body for a KNOWN op is a real error worth surfacing.
13875        let malformed = Frame::build(
13876            FrameType::Push,
13877            control_flags(),
13878            0,
13879            0,
13880            6,
13881            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13882        )
13883        .unwrap();
13884        let out = handler.handle_status_update(endpoint, malformed).unwrap();
13885        assert_eq!(out.len(), 1);
13886        assert_eq!(out[0].header.ty, FrameType::Error);
13887        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13888    }
13889
13890    #[test]
13891    fn hello_rejected_when_connection_already_owns_client_routes() {
13892        let registry = Arc::new(Registry::default());
13893        let forwarding = Arc::new(ForwardingTable::default());
13894        let handler =
13895            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13896        // Commits a client route on connection 202 (bound to a module on conn 101).
13897        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
13898        let client_connection = ConnectionId::new(202);
13899
13900        // That same connection now tries to register as a module: rejected, so one
13901        // connection never holds both client-route and module-endpoint state.
13902        let responses = handler
13903            .handle_control(
13904                client_connection,
13905                hello_frame("aft-second", PROTOCOL_VERSION, 9),
13906            )
13907            .unwrap();
13908        assert_eq!(responses[0].header.ty, FrameType::Error);
13909        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13910        assert!(registry.get_module("aft-second").unwrap().is_none());
13911    }
13912
13913    #[tokio::test]
13914    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13915        let registry = Arc::new(Registry::default());
13916        let forwarding = Arc::new(ForwardingTable::default());
13917        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13918        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13919        hello_via_sink(
13920            &handler,
13921            &module_ctx,
13922            &mut module_rx,
13923            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13924        )
13925        .await;
13926        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13927        let pending = forwarding
13928            .begin_route_bind_relay_for_test(
13929                client_ctx.connection_id,
13930                client_ctx.egress.clone(),
13931                2,
13932                "alpha",
13933            )
13934            .unwrap();
13935        forwarding
13936            .complete_pending_relay(
13937                module_ctx.connection_id,
13938                pending.corr,
13939                RouteBindRelayOutcome::Accepted,
13940            )
13941            .unwrap();
13942        client_rx.try_recv().unwrap();
13943        for module_id in ["beta", "alpha"] {
13944            let replies = handler
13945                .handle_control_frame(
13946                    &module_ctx,
13947                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13948                )
13949                .await
13950                .unwrap();
13951            assert_eq!(replies.len(), 1, "second HELLO must be refused");
13952            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13953        }
13954        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13955        assert!(registry.get_module("beta").unwrap().is_none());
13956        assert!(matches!(
13957            forwarding
13958                .lookup_data_route(
13959                    client_ctx.connection_id,
13960                    pending.client_channel,
13961                    pending.client_epoch,
13962                )
13963                .unwrap(),
13964            DataRoute::Client(DataRouteState::Bound(_))
13965        ));
13966        assert!(handler
13967            .hello_launch_nonces
13968            .lock()
13969            .unwrap()
13970            .presented(module_ctx.connection_id, Some("alpha-nonce")));
13971        assert!(module_rx.try_recv().is_err());
13972    }
13973
13974    #[test]
13975    fn reserved_module_hello_requires_matching_launch_nonce() {
13976        let registry = Arc::new(Registry::default());
13977        let supervisor = SupervisorHandle::new();
13978        // The supervisor recorded the nonce it injected when it spawned the reserved
13979        // module; the HELLO verifier checks against the same shared handle.
13980        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13981        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13982
13983        // A HELLO with NO nonce is rejected.
13984        let no_nonce = handler
13985            .handle_control(
13986                ConnectionId::new(1),
13987                hello_frame("vault", PROTOCOL_VERSION, 1),
13988            )
13989            .unwrap();
13990        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13991        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13992        assert!(registry.get_module("vault").unwrap().is_none());
13993
13994        // A HELLO with the WRONG nonce is rejected.
13995        let wrong = handler
13996            .handle_control(
13997                ConnectionId::new(2),
13998                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13999            )
14000            .unwrap();
14001        assert_eq!(wrong[0].header.ty, FrameType::Error);
14002        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14003        assert!(registry.get_module("vault").unwrap().is_none());
14004
14005        // A HELLO with the CORRECT nonce registers.
14006        let ok = handler
14007            .handle_control(
14008                ConnectionId::new(3),
14009                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14010            )
14011            .unwrap();
14012        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14013        assert!(registry.get_module("vault").unwrap().is_some());
14014    }
14015
14016    #[test]
14017    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14018        let registry = Arc::new(Registry::default());
14019        let supervisor = SupervisorHandle::new();
14020        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14021        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14022        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14023
14024        let squat = handler
14025            .handle_control(
14026                ConnectionId::new(1),
14027                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14028            )
14029            .unwrap();
14030        assert_eq!(squat[0].header.ty, FrameType::Error);
14031        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14032        assert!(parse_error(&squat[0])["message"]
14033            .as_str()
14034            .unwrap()
14035            .contains("fed:"));
14036
14037        let accepted_peer = handler
14038            .handle_control(
14039                ConnectionId::new(2),
14040                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14041            )
14042            .unwrap();
14043        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14044
14045        let accepted_short = handler
14046            .handle_control(
14047                ConnectionId::new(3),
14048                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14049            )
14050            .unwrap();
14051        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14052
14053        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14054            let response = handler
14055                .handle_control(
14056                    ConnectionId::new(conn),
14057                    hello_frame(module_id, PROTOCOL_VERSION, conn),
14058                )
14059                .unwrap();
14060            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14061        }
14062    }
14063
14064    #[test]
14065    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14066        let registry = Arc::new(Registry::default());
14067        let supervisor = SupervisorHandle::new();
14068        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14069        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14070        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14071        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14072
14073        let owner_nonce = handler
14074            .handle_control(
14075                ConnectionId::new(1),
14076                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14077            )
14078            .unwrap();
14079        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14080        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14081        assert!(registry.get_module("fed:special").unwrap().is_none());
14082
14083        let exact_nonce = handler
14084            .handle_control(
14085                ConnectionId::new(2),
14086                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14087            )
14088            .unwrap();
14089        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14090        assert!(registry.get_module("fed:special").unwrap().is_some());
14091    }
14092
14093    #[test]
14094    fn non_reserved_module_ignores_launch_nonce() {
14095        let registry = Arc::new(Registry::default());
14096        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
14097        // registration succeeds whether a spawned process echoes a nonce or not.
14098        let handler = ControlHandler::new(Arc::clone(&registry));
14099        let no_nonce = handler
14100            .handle_control(
14101                ConnectionId::new(1),
14102                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14103            )
14104            .unwrap();
14105        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14106        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14107
14108        let echoed_nonce = handler
14109            .handle_control(
14110                ConnectionId::new(2),
14111                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14112            )
14113            .unwrap();
14114        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14115        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14116    }
14117
14118    #[test]
14119    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14120        let handler = ControlHandler::default();
14121        let conn = ConnectionId::new(1);
14122        let malformed = Frame::build(
14123            FrameType::Hello,
14124            control_flags(),
14125            0,
14126            0,
14127            3,
14128            b"{not json".to_vec(),
14129        )
14130        .unwrap();
14131
14132        let error = handler.handle_control(conn, malformed).unwrap();
14133        assert_eq!(error[0].header.ty, FrameType::Error);
14134        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14135
14136        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14137        let pong = handler.handle_control(conn, ping).unwrap();
14138        assert_eq!(pong[0].header.ty, FrameType::Pong);
14139        assert_eq!(pong[0].header.corr, 4);
14140    }
14141
14142    #[test]
14143    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14144        let registry = Arc::new(Registry::default());
14145        let handler = ControlHandler::new(Arc::clone(&registry));
14146
14147        handler
14148            .handle_control(
14149                ConnectionId::new(1),
14150                hello_frame("aft", PROTOCOL_VERSION, 1),
14151            )
14152            .unwrap();
14153        let duplicate = handler
14154            .handle_control(
14155                ConnectionId::new(2),
14156                hello_frame("aft", PROTOCOL_VERSION, 2),
14157            )
14158            .unwrap();
14159
14160        assert_eq!(duplicate[0].header.ty, FrameType::Error);
14161        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14162        let registration = registry.get_module("aft").unwrap().unwrap();
14163        assert_eq!(registration.connection_id, ConnectionId::new(1));
14164    }
14165
14166    #[test]
14167    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14168        let registry = Arc::new(Registry::default());
14169        let forwarding = Arc::new(ForwardingTable::default());
14170        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14171        let handler =
14172            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14173                .with_process_liveness(process_liveness);
14174        let (ctx, route_channel, route_epoch) =
14175            bind_liveness_route(&registry, &forwarding, "aft-dead");
14176        let responses = handler
14177            .handle_route_poll(
14178                &ctx,
14179                route_poll_frame(41, PollKind::Liveness, route_channel),
14180                route_channel,
14181                route_epoch,
14182                PollKind::Liveness,
14183            )
14184            .unwrap();
14185
14186        assert_eq!(responses.len(), 1);
14187        assert_eq!(responses[0].header.ty, FrameType::Response);
14188        assert_route_poll_liveness(&responses[0], false);
14189    }
14190
14191    #[test]
14192    fn liveness_poll_without_process_source_uses_bound_route() {
14193        let registry = Arc::new(Registry::default());
14194        let forwarding = Arc::new(ForwardingTable::default());
14195        let handler =
14196            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14197        let (ctx, route_channel, route_epoch) =
14198            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
14199        let responses = handler
14200            .handle_route_poll(
14201                &ctx,
14202                route_poll_frame(42, PollKind::Liveness, route_channel),
14203                route_channel,
14204                route_epoch,
14205                PollKind::Liveness,
14206            )
14207            .unwrap();
14208
14209        assert_route_poll_liveness(&responses[0], true);
14210    }
14211
14212    #[test]
14213    fn liveness_poll_untracked_process_source_uses_bound_route() {
14214        let registry = Arc::new(Registry::default());
14215        let forwarding = Arc::new(ForwardingTable::default());
14216        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14217        let handler =
14218            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14219                .with_process_liveness(process_liveness);
14220        let (ctx, route_channel, route_epoch) =
14221            bind_liveness_route(&registry, &forwarding, "aft-untracked");
14222        let responses = handler
14223            .handle_route_poll(
14224                &ctx,
14225                route_poll_frame(43, PollKind::Liveness, route_channel),
14226                route_channel,
14227                route_epoch,
14228                PollKind::Liveness,
14229            )
14230            .unwrap();
14231
14232        assert_route_poll_liveness(&responses[0], true);
14233    }
14234
14235    #[tokio::test]
14236    async fn unknown_op_returns_unknown_control_op() {
14237        let handler = ControlHandler::default();
14238        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14239        let request = Frame::build(
14240            FrameType::Request,
14241            control_flags(),
14242            0,
14243            0,
14244            55,
14245            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14246        )
14247        .unwrap();
14248
14249        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14250
14251        assert_eq!(response.len(), 1);
14252        assert_eq!(response[0].header.ty, FrameType::Error);
14253        assert_eq!(response[0].header.corr, 55);
14254        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14255    }
14256
14257    #[tokio::test]
14258    async fn supervisor_provenance_rejects_unknown_exact_module() {
14259        let handler = ControlHandler::default();
14260        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14261        let request = Frame::build(
14262            FrameType::Request,
14263            control_flags(),
14264            0,
14265            0,
14266            57,
14267            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14268        )
14269        .unwrap();
14270
14271        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14272
14273        assert_eq!(response.len(), 1);
14274        assert_eq!(response[0].header.ty, FrameType::Error);
14275        assert_eq!(response[0].header.corr, 57);
14276        let error = parse_error(&response[0]);
14277        assert_eq!(error["code"], "unknown_module");
14278        assert_eq!(error["message"], "module_id 'missing' is not supervised");
14279    }
14280
14281    #[test]
14282    fn provenance_probe_override_keeps_handler_tests_deterministic() {
14283        let expected = subc_control::RunningImageAgreement::Unavailable {
14284            reason: subc_control::RunningImageUnavailableReason::HashFailed,
14285        };
14286        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14287        assert_eq!(handler.provenance_probe_override, Some(expected));
14288    }
14289
14290    #[test]
14291    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14292        let verdict = reload_verdict(
14293            std::path::Path::new("/bin/new"),
14294            Some(std::path::Path::new("/bin/old")),
14295            subc_control::RunningImageAgreement::Unavailable {
14296                reason: subc_control::RunningImageUnavailableReason::HashFailed,
14297            },
14298        );
14299        assert!(matches!(
14300            verdict.path,
14301            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14302                if configured == std::path::Path::new("/bin/new")
14303                    && spawned_from == std::path::Path::new("/bin/old")
14304        ));
14305    }
14306
14307    #[test]
14308    fn launch_spec_without_running_digest_is_unavailable_not_match() {
14309        use subc_control::{
14310            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14311        };
14312        let configured = crate::supervise::ModuleSpec {
14313            module_id: "legacy".into(),
14314            program: "/module".into(),
14315            args: Vec::new(),
14316            env: Vec::new(),
14317            reserved: false,
14318            reserved_prefixes: Vec::new(),
14319            protocol: subc_control::ModuleProtocol::Subc,
14320            overlap: Default::default(),
14321        };
14322        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14323        assert_eq!(
14324            launch_spec_agreement(&configured, false, Some(&digest)),
14325            Agreement::Unavailable {
14326                reason: Reason::NotRunning
14327            }
14328        );
14329        assert_eq!(
14330            launch_spec_agreement(&configured, true, None),
14331            Agreement::Unavailable {
14332                reason: Reason::SpawnedSpecUnavailable
14333            }
14334        );
14335        assert_eq!(
14336            launch_spec_agreement(&configured, true, Some(&digest)),
14337            Agreement::Match
14338        );
14339    }
14340
14341    #[cfg(windows)]
14342    #[test]
14343    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14344        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14345        let configured = dir.join("module.exe");
14346        std::fs::write(&configured, b"executable path fixture").unwrap();
14347        let spawned = std::fs::canonicalize(&configured).unwrap();
14348        assert_ne!(
14349            configured, spawned,
14350            "fixture must exercise different path spellings"
14351        );
14352        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14353        let lower_case = configured.to_string_lossy().to_lowercase();
14354        for spelling in [
14355            &configured,
14356            Path::new(&forward_slashes),
14357            Path::new(&lower_case),
14358        ] {
14359            let verdict = reload_verdict(
14360                spelling,
14361                Some(&spawned),
14362                subc_control::RunningImageAgreement::Unavailable {
14363                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14364                },
14365            );
14366            assert_eq!(
14367                verdict.path,
14368                ReloadPathAgreement::Match,
14369                "configured {spelling:?}, spawned {spawned:?}"
14370            );
14371        }
14372    }
14373
14374    #[cfg(windows)]
14375    #[test]
14376    fn windows_reload_path_keeps_different_existing_program_pending() {
14377        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14378        let old = dir.join("old.exe");
14379        let new = dir.join("new.exe");
14380        std::fs::write(&old, b"same executable content").unwrap();
14381        std::fs::copy(&old, &new).unwrap();
14382        let spawned = std::fs::canonicalize(&old).unwrap();
14383        let image = subc_control::RunningImageAgreement::Unavailable {
14384            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14385        };
14386        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14387        assert!(
14388            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14389            if configured == new && spawned_from == spawned)
14390        );
14391        let missing = dir.join("missing.exe");
14392        assert!(matches!(
14393            reload_verdict(&missing, Some(&spawned), image).path,
14394            ReloadPathAgreement::Mismatch { .. }
14395        ));
14396    }
14397
14398    #[test]
14399    fn reload_verdict_detects_replaced_image_at_same_path() {
14400        let image = subc_control::RunningImageAgreement::Mismatch {
14401            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14402                digest: "old".into(),
14403            },
14404            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14405                digest: "new".into(),
14406            },
14407        };
14408        let verdict = reload_verdict(
14409            std::path::Path::new("/bin/same"),
14410            Some(std::path::Path::new("/bin/same")),
14411            image.clone(),
14412        );
14413        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14414        assert_eq!(verdict.image, image);
14415    }
14416
14417    #[test]
14418    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14419        let image = subc_control::RunningImageAgreement::Unavailable {
14420            reason: subc_control::RunningImageUnavailableReason::NotRunning,
14421        };
14422        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14423        assert_eq!(
14424            verdict.path,
14425            subc_control::ReloadPathAgreement::Unavailable {
14426                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14427            }
14428        );
14429        assert_eq!(verdict.image, image);
14430
14431        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14432            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14433        };
14434        let verdict = reload_verdict(
14435            std::path::Path::new("/bin/same"),
14436            Some(std::path::Path::new("/bin/same")),
14437            unconfirmed.clone(),
14438        );
14439        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14440        assert_eq!(verdict.image, unconfirmed);
14441    }
14442
14443    #[test]
14444    fn reload_verdict_preserves_each_image_unavailability_reason() {
14445        use subc_control::RunningImageUnavailableReason as Reason;
14446
14447        for reason in [
14448            Reason::NotRunning,
14449            Reason::UnsupportedPlatform,
14450            Reason::RunningExecutableUnreadable,
14451            Reason::SpawnedPathUnreadable,
14452            Reason::HashFailed,
14453            Reason::ProcessIdentityUnconfirmed,
14454            Reason::Unknown("future_probe_reason".to_string()),
14455        ] {
14456            let image = subc_control::RunningImageAgreement::Unavailable {
14457                reason: reason.clone(),
14458            };
14459            let verdict = reload_verdict(
14460                std::path::Path::new("/bin/same"),
14461                Some(std::path::Path::new("/bin/same")),
14462                image.clone(),
14463            );
14464            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14465            assert_eq!(verdict.image, image, "{reason:?}");
14466        }
14467    }
14468
14469    #[tokio::test]
14470    async fn malformed_control_bodies_return_invalid_control_body() {
14471        let handler = ControlHandler::default();
14472        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14473
14474        for (corr, body) in [
14475            (56, br#"{"route_channel":1}"#.as_slice()),
14476            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14477            (
14478                58,
14479                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14480            ),
14481        ] {
14482            let request = Frame::build(
14483                FrameType::Request,
14484                control_flags(),
14485                0,
14486                0,
14487                corr,
14488                body.to_vec(),
14489            )
14490            .unwrap();
14491            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14492
14493            assert_eq!(response.len(), 1);
14494            assert_eq!(response[0].header.ty, FrameType::Error);
14495            assert_eq!(response[0].header.corr, corr);
14496            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14497        }
14498    }
14499
14500    #[tokio::test]
14501    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14502        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14503        let registry = Arc::new(Registry::default());
14504        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14505        let router = Router::with_control_handler(Arc::clone(&control));
14506        let connection = router.begin_connection();
14507        let (ctx, mut rx) = route_ctx(connection.id());
14508
14509        router
14510            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14511            .await
14512            .unwrap();
14513        let response = rx.recv().await.unwrap();
14514        let ack = parse_ack(&response);
14515        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14516        let channel = 1;
14517
14518        let goodbye =
14519            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14520        router.route_for_connection(&ctx, goodbye).await.unwrap();
14521        assert!(rx.try_recv().is_err());
14522        assert!(registry.get_module("aft").unwrap().is_none());
14523
14524        router
14525            .route_for_connection(&ctx, channel_request(channel, 13))
14526            .await
14527            .unwrap();
14528        let error_frame = rx.recv().await.unwrap();
14529        assert_eq!(error_frame.header.ty, FrameType::Error);
14530        assert_eq!(error_frame.header.channel, channel);
14531        let captured = crate::router::test_log::captured_logs(&logs);
14532        assert_eq!(
14533            captured
14534                .lines()
14535                .filter(|line| {
14536                    line.contains("module_id=aft")
14537                        && line.contains("reason=explicit_goodbye")
14538                        && line.contains("module registration ended")
14539                })
14540                .count(),
14541            1,
14542            "unexpected GOODBYE registry log: {captured}"
14543        );
14544    }
14545
14546    #[tokio::test]
14547    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14548        let registry = Arc::new(Registry::default());
14549        let handler = ControlHandler::new(registry).with_capability_config(
14550            [("prov".to_string(), true), ("cons".to_string(), true)],
14551            BTreeMap::new(),
14552        );
14553        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14554        register_capability_manifest(
14555            &handler,
14556            &provider_ctx,
14557            &mut provider_rx,
14558            capability_manifest("prov", &["thing/v1"], &[]),
14559            1,
14560        )
14561        .await;
14562        let mut consumer = capability_manifest("cons", &[], &[]);
14563        consumer.capabilities.as_mut().unwrap().requires.push(
14564            subc_protocol::manifest::CapabilityRequirement {
14565                capability: "thing/v1".to_string(),
14566                need: subc_protocol::manifest::CapabilityNeed::Required,
14567            },
14568        );
14569        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14570        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14571        assert_eq!(
14572            handler.capability_evaluator.verdict("cons", "thing/v1"),
14573            Some(CapabilityVerdict::Provided)
14574        );
14575        let (mut client_rx, _) = open_route_for_capability_test(
14576            &handler,
14577            &provider_ctx,
14578            &mut provider_rx,
14579            703,
14580            3,
14581            "prov",
14582            None,
14583        )
14584        .await;
14585        let goodbye =
14586            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14587        handler
14588            .handle_control_frame(&provider_ctx, goodbye)
14589            .await
14590            .unwrap();
14591        assert_eq!(
14592            handler.capability_evaluator.verdict("cons", "thing/v1"),
14593            Some(CapabilityVerdict::NeverProvided)
14594        );
14595        let closed = client_rx
14596            .try_recv()
14597            .expect("GOODBYE pushes route.closed before route GOODBYE");
14598        assert!(
14599            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14600            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14601        );
14602        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14603        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14604    }
14605
14606    struct RouteClosedFixture {
14607        handler: ControlHandler,
14608        #[cfg(unix)]
14609        supervisor: Supervisor,
14610        module_ctx: RouteCtx,
14611        #[cfg(unix)]
14612        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14613        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14614        route: Arc<crate::forwarding::RouteBinding>,
14615    }
14616
14617    async fn route_closed_fixture() -> RouteClosedFixture {
14618        let registry = Arc::new(Registry::default());
14619        let handler = ControlHandler::new(Arc::clone(&registry));
14620        #[cfg(unix)]
14621        let supervisor = Supervisor::new(registry, RestartPolicy::default())
14622            .with_forwarding(Arc::clone(&handler.forwarding));
14623        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14624        register_capability_manifest(
14625            &handler,
14626            &module_ctx,
14627            &mut module_rx,
14628            capability_manifest("prov", &[], &[]),
14629            1,
14630        )
14631        .await;
14632        let (client_rx, bind) = open_route_for_capability_test(
14633            &handler,
14634            &module_ctx,
14635            &mut module_rx,
14636            702,
14637            2,
14638            "prov",
14639            None,
14640        )
14641        .await;
14642        let ModuleControlRequest::RouteBind {
14643            route_channel: channel,
14644            epoch: route_epoch,
14645            ..
14646        } = bind
14647        else {
14648            panic!("expected route.bind");
14649        };
14650        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14651            handler
14652                .forwarding
14653                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14654                .unwrap()
14655        else {
14656            panic!("the acknowledged route must be bound");
14657        };
14658        RouteClosedFixture {
14659            handler,
14660            #[cfg(unix)]
14661            supervisor,
14662            module_ctx,
14663            #[cfg(unix)]
14664            module_rx,
14665            client_rx,
14666            route,
14667        }
14668    }
14669
14670    impl RouteClosedFixture {
14671        #[cfg(unix)]
14672        async fn cut(&mut self) {
14673            let notices = async {
14674                let mut draining = self.module_rx.recv().await.unwrap();
14675                assert!(matches!(
14676                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14677                        &draining.body
14678                    )
14679                    .unwrap(),
14680                    subc_protocol::session::ModuleControlCommand::Draining {
14681                        reason: RouteCloseReason::Restart,
14682                        ..
14683                    }
14684                ));
14685                draining.flushed.take().unwrap().send(()).unwrap();
14686                let mut closing = self.client_rx.recv().await.unwrap();
14687                assert!(matches!(
14688                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14689                    ClientControlPush::RouteClosing {
14690                        reason: RouteCloseReason::Restart,
14691                        ..
14692                    }
14693                ));
14694                closing.flushed.take().unwrap().send(()).unwrap();
14695            };
14696            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14697            result.unwrap();
14698        }
14699
14700        fn disconnect(&mut self) -> ClientControlPush {
14701            self.handler
14702                .cleanup_connection(self.module_ctx.connection_id)
14703                .unwrap();
14704            let closed = self.client_rx.try_recv().unwrap();
14705            assert_eq!(closed.header.ty, FrameType::Push);
14706            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14707            assert!(matches!(
14708                &push,
14709                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14710                    if channels == &[self.route.client_channel]
14711            ));
14712            assert_eq!(
14713                self.client_rx.try_recv().unwrap().header.ty,
14714                FrameType::Goodbye
14715            );
14716            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14717            push
14718        }
14719    }
14720
14721    #[cfg(unix)]
14722    #[tokio::test(start_paused = true)]
14723    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14724        let mut fixture = route_closed_fixture().await;
14725        fixture.cut().await;
14726        assert!(matches!(
14727            fixture.disconnect(),
14728            ClientControlPush::RouteClosed {
14729                reason: RouteCloseReason::Restart,
14730                drained: true,
14731                ..
14732            }
14733        ));
14734    }
14735
14736    #[cfg(unix)]
14737    #[tokio::test(start_paused = true)]
14738    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14739        let mut fixture = route_closed_fixture().await;
14740        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14741        let started = tokio::time::Instant::now();
14742        fixture.cut().await;
14743        assert!(started.elapsed() >= Duration::from_secs(2));
14744        assert_eq!(fixture.route.flow.in_flight(), 1);
14745        assert!(matches!(
14746            fixture.disconnect(),
14747            ClientControlPush::RouteClosed {
14748                reason: RouteCloseReason::Restart,
14749                drained: false,
14750                ..
14751            }
14752        ));
14753    }
14754
14755    #[tokio::test]
14756    async fn module_connection_crash_reports_crash_and_not_drained() {
14757        let mut fixture = route_closed_fixture().await;
14758        // The server calls cleanup_connection on module EOF. Without a planned
14759        // daemon shutdown, that disconnect must remain a crash with no prior
14760        // route.closing notice, even when the route has no in-flight calls.
14761        assert!(matches!(
14762            fixture.disconnect(),
14763            ClientControlPush::RouteClosed {
14764                reason: RouteCloseReason::Crash,
14765                drained: false,
14766                ..
14767            }
14768        ));
14769    }
14770
14771    #[tokio::test]
14772    async fn dropping_router_connection_releases_registration() {
14773        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14774        let registry = Arc::new(Registry::default());
14775        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14776        let router = Router::with_control_handler(Arc::clone(&control));
14777        let connection = router.begin_connection();
14778        let connection_id = connection.id();
14779        let (ctx, mut rx) = route_ctx(connection_id);
14780
14781        router
14782            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14783            .await
14784            .unwrap();
14785        let response = rx.recv().await.unwrap();
14786        let ack = parse_ack(&response);
14787        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14788        assert!(registry.get_module("aft").unwrap().is_some());
14789
14790        drop(connection);
14791
14792        assert!(registry.get_module("aft").unwrap().is_none());
14793        assert_eq!(registry.active_registration_count().unwrap(), 0);
14794
14795        control
14796            .cleanup_connection(ConnectionId::new(u64::MAX))
14797            .unwrap();
14798        let captured = crate::router::test_log::captured_logs(&logs);
14799        println!("captured registry lifecycle logs:\n{captured}");
14800        let events: Vec<_> = captured
14801            .lines()
14802            .filter(|line| {
14803                line.contains("module registered module_id=aft ")
14804                    || line.contains("module registration ended")
14805            })
14806            .collect();
14807        assert_eq!(
14808            events.len(),
14809            2,
14810            "unexpected registry lifecycle logs: {captured}"
14811        );
14812        assert!(events[0].contains("module registered module_id=aft "));
14813        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14814        assert!(events[1].contains(&format!(
14815            "module_id=aft connection_id={}",
14816            connection_id.get()
14817        )));
14818        assert!(events[1].contains("reason=connection_closed"));
14819        assert!(events[1].contains("module registration ended"));
14820        assert_eq!(
14821            captured
14822                .lines()
14823                .filter(|line| line.contains("module registered module_id=aft "))
14824                .count(),
14825            1,
14826            "legacy registration admission line must appear once: {captured}"
14827        );
14828    }
14829
14830    #[tokio::test]
14831    async fn hello_registration_keeps_legacy_module_registered_line_once() {
14832        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14833        let registry = Arc::new(Registry::default());
14834        let control = ControlHandler::new(Arc::clone(&registry));
14835        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14836        hello_via_sink(
14837            &control,
14838            &ctx,
14839            &mut rx,
14840            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14841        )
14842        .await;
14843
14844        let captured = crate::router::test_log::captured_logs(&logs);
14845        let admissions: Vec<_> = captured
14846            .lines()
14847            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14848            .collect();
14849        assert_eq!(
14850            admissions.len(),
14851            1,
14852            "legacy admission line must remain exactly once: {captured}"
14853        );
14854        assert!(admissions[0].contains("routable_provider=true"));
14855        assert!(admissions[0].contains("connection_id=777"));
14856    }
14857
14858    fn capability_manifest(
14859        module_id: &str,
14860        provides: &[&str],
14861        must_never_reach: &[&str],
14862    ) -> ModuleManifest {
14863        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14864        manifest.capabilities = Some(CapabilityDeclarations {
14865            provides: provides
14866                .iter()
14867                .map(|capability| (*capability).to_string())
14868                .collect(),
14869            requires: Vec::new(),
14870            must_never_reach: must_never_reach
14871                .iter()
14872                .map(|capability| (*capability).to_string())
14873                .collect(),
14874        });
14875        manifest
14876    }
14877
14878    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14879        Frame::build(
14880            FrameType::Hello,
14881            control_flags(),
14882            0,
14883            0,
14884            corr,
14885            serde_json::to_vec(&ModuleHelloBody {
14886                protocol_ver: manifest.protocol_ver,
14887                manifest,
14888                control_ops: None,
14889                launch_nonce: None,
14890            })
14891            .expect("capability test HELLO serializes"),
14892        )
14893        .expect("capability test HELLO frame builds")
14894    }
14895
14896    fn catalog_update_with_capabilities_frame(
14897        corr: u64,
14898        capabilities: CapabilityDeclarations,
14899    ) -> Frame {
14900        Frame::build(
14901            FrameType::Request,
14902            control_flags(),
14903            0,
14904            0,
14905            corr,
14906            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14907                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14908                capabilities: Some(capabilities),
14909                ready: None,
14910            })
14911            .expect("capability catalog.update serializes"),
14912        )
14913        .expect("capability catalog.update frame builds")
14914    }
14915
14916    async fn register_capability_manifest(
14917        handler: &ControlHandler,
14918        ctx: &RouteCtx,
14919        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14920        manifest: ModuleManifest,
14921        corr: u64,
14922    ) {
14923        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14924    }
14925
14926    async fn open_route_for_capability_test(
14927        handler: &ControlHandler,
14928        target_ctx: &RouteCtx,
14929        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14930        client_connection_id: u64,
14931        corr: u64,
14932        target_module_id: &str,
14933        consumer_identity: Option<ConsumerIdentity>,
14934    ) -> (
14935        mpsc::Receiver<crate::router::OutboundFrame>,
14936        ModuleControlRequest,
14937    ) {
14938        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14939        let route_handler = handler.clone();
14940        let target_module_id = target_module_id.to_string();
14941        let route_task = tokio::spawn(async move {
14942            route_handler
14943                .handle_control_frame(
14944                    &client_ctx,
14945                    route_open_frame_with_admission_facts(
14946                        corr,
14947                        &target_module_id,
14948                        unique_project_root("admission-facts"),
14949                        consumer_identity,
14950                        None,
14951                    ),
14952                )
14953                .await
14954                .expect("capability test route.open succeeds")
14955        });
14956        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14957            .await
14958            .expect("capability test route.open must reach route.bind")
14959            .expect("target control receiver stays open");
14960        let bind_request: ModuleControlRequest =
14961            serde_json::from_slice(&bind.body).expect("route.bind decodes");
14962        handler
14963            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14964            .await
14965            .expect("capability test route.bind ACK succeeds");
14966        assert!(route_task.await.expect("route.open task joins").is_empty());
14967        let opened = client_rx
14968            .recv()
14969            .await
14970            .expect("successful route.open publishes a response");
14971        assert!(matches!(
14972            serde_json::from_slice::<ClientControlResponse>(&opened.body),
14973            Ok(ClientControlResponse::RouteOpen { .. })
14974        ));
14975        (client_rx, bind_request)
14976    }
14977
14978    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
14979        assert_eq!(frame.header.ty, FrameType::Push);
14980        assert_eq!(frame.header.channel, 0);
14981        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14982            .expect("route.closed control push decodes");
14983        let ClientControlPush::RouteClosed { channels, .. } = &push else {
14984            panic!("expected route.closed");
14985        };
14986        assert_eq!(channels.len(), 1, "exactly one violating route closed");
14987        let closed_channel = channels[0];
14988        let channels = channels.clone();
14989        assert_eq!(
14990            push,
14991            ClientControlPush::RouteClosed {
14992                module_id: target_module_id.to_string(),
14993                channels,
14994                reason: RouteCloseReason::CapabilityDenied,
14995                drained: false,
14996                abandoned: 0,
14997                excluded_subscriptions: 0,
14998                terminal: Some(false),
14999            }
15000        );
15001        closed_channel
15002    }
15003
15004    #[tokio::test]
15005    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15006        let registry = Arc::new(Registry::default());
15007        let forwarding = Arc::new(ForwardingTable::default());
15008        let supervisor = SupervisorHandle::new();
15009        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15010        let handler =
15011            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15012                .with_supervisor(supervisor);
15013        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15014        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15015        register_capability_manifest(
15016            &handler,
15017            &target_ctx,
15018            &mut target_rx,
15019            capability_manifest("target", &["credentials-provider/v1"], &[]),
15020            1,
15021        )
15022        .await;
15023        register_capability_manifest(
15024            &handler,
15025            &opener_ctx,
15026            &mut opener_rx,
15027            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15028            2,
15029        )
15030        .await;
15031
15032        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15033        let replies = handler
15034            .handle_control_frame(
15035                &client_ctx,
15036                route_open_frame_with_admission_facts(
15037                    3,
15038                    "target",
15039                    unique_project_root("admission-facts"),
15040                    Some(ConsumerIdentity {
15041                        module_id: "opener".to_string(),
15042                        launch_nonce: "opener-nonce".to_string(),
15043                    }),
15044                    None,
15045                ),
15046            )
15047            .await
15048            .expect("denied route.open returns a typed frame");
15049        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15050        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15051        assert!(
15052            target_rx.try_recv().is_err(),
15053            "forbidden route.open must not relay route.bind"
15054        );
15055    }
15056
15057    #[tokio::test]
15058    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15059        let registry = Arc::new(Registry::default());
15060        let forwarding = Arc::new(ForwardingTable::default());
15061        let supervisor = SupervisorHandle::new();
15062        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15063        let handler =
15064            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15065                .with_supervisor(supervisor);
15066        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15067        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15068        register_capability_manifest(
15069            &handler,
15070            &target_ctx,
15071            &mut target_rx,
15072            capability_manifest("target", &["credentials-provider/v1"], &[]),
15073            1,
15074        )
15075        .await;
15076        register_capability_manifest(
15077            &handler,
15078            &old_opener_ctx,
15079            &mut old_opener_rx,
15080            capability_manifest("opener", &[], &[]),
15081            2,
15082        )
15083        .await;
15084        let (mut client_rx, _) = open_route_for_capability_test(
15085            &handler,
15086            &target_ctx,
15087            &mut target_rx,
15088            712,
15089            3,
15090            "target",
15091            Some(ConsumerIdentity {
15092                module_id: "opener".to_string(),
15093                launch_nonce: "opener-nonce".to_string(),
15094            }),
15095        )
15096        .await;
15097        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15098
15099        handler
15100            .cleanup_connection(old_opener_ctx.connection_id)
15101            .expect("old opener registration cleans up");
15102        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15103        register_capability_manifest(
15104            &handler,
15105            &new_opener_ctx,
15106            &mut new_opener_rx,
15107            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15108            4,
15109        )
15110        .await;
15111
15112        let closed_channel = assert_capability_denied_push(
15113            client_rx
15114                .try_recv()
15115                .expect("HELLO deny addition must emit route.closed")
15116                .frame,
15117            "target",
15118        );
15119        let goodbye = client_rx
15120            .try_recv()
15121            .expect("HELLO deny addition must send the client a route GOODBYE")
15122            .frame;
15123        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15124        assert_eq!(goodbye.header.channel, closed_channel);
15125        assert_ne!(goodbye.header.channel, 0);
15126        assert_ne!(goodbye.header.epoch, 0);
15127        assert!(client_rx.try_recv().is_err());
15128        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15129        assert!(matches!(
15130            target_rx.try_recv(),
15131            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15132        ));
15133    }
15134
15135    #[tokio::test]
15136    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15137        let registry = Arc::new(Registry::default());
15138        let forwarding = Arc::new(ForwardingTable::default());
15139        let supervisor = SupervisorHandle::new();
15140        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15141        let handler =
15142            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15143                .with_supervisor(supervisor);
15144        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15145        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15146        register_capability_manifest(
15147            &handler,
15148            &target_ctx,
15149            &mut target_rx,
15150            capability_manifest("target", &[], &[]),
15151            1,
15152        )
15153        .await;
15154        register_capability_manifest(
15155            &handler,
15156            &opener_ctx,
15157            &mut opener_rx,
15158            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15159            2,
15160        )
15161        .await;
15162        let (mut client_rx, _) = open_route_for_capability_test(
15163            &handler,
15164            &target_ctx,
15165            &mut target_rx,
15166            722,
15167            3,
15168            "target",
15169            Some(ConsumerIdentity {
15170                module_id: "opener".to_string(),
15171                launch_nonce: "opener-nonce".to_string(),
15172            }),
15173        )
15174        .await;
15175        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15176
15177        let replies = handler
15178            .handle_control_frame(
15179                &target_ctx,
15180                catalog_update_with_capabilities_frame(
15181                    4,
15182                    CapabilityDeclarations {
15183                        provides: vec!["credentials-provider/v1".to_string()],
15184                        requires: Vec::new(),
15185                        must_never_reach: Vec::new(),
15186                    },
15187                ),
15188            )
15189            .await
15190            .expect("claim catalog.update succeeds");
15191        assert!(matches!(
15192            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15193            Ok(ModuleControlResponseToModule::CatalogUpdate {})
15194        ));
15195        let closed_channel = assert_capability_denied_push(
15196            client_rx
15197                .try_recv()
15198                .expect("claim addition must emit route.closed")
15199                .frame,
15200            "target",
15201        );
15202        let goodbye = client_rx
15203            .try_recv()
15204            .expect("claim addition must send the client a route GOODBYE")
15205            .frame;
15206        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15207        assert_eq!(goodbye.header.channel, closed_channel);
15208        assert_ne!(goodbye.header.channel, 0);
15209        assert_ne!(goodbye.header.epoch, 0);
15210        assert!(client_rx.try_recv().is_err());
15211        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15212        assert!(matches!(
15213            target_rx.try_recv(),
15214            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15215        ));
15216    }
15217
15218    #[tokio::test]
15219    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15220        let registry = Arc::new(Registry::default());
15221        let forwarding = Arc::new(ForwardingTable::default());
15222        let supervisor = SupervisorHandle::new();
15223        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15224        let handler =
15225            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15226                .with_supervisor(supervisor);
15227        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15228        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15229        register_capability_manifest(
15230            &handler,
15231            &target_ctx,
15232            &mut target_rx,
15233            capability_manifest("target", &["credentials-provider/v1"], &[]),
15234            1,
15235        )
15236        .await;
15237        register_capability_manifest(
15238            &handler,
15239            &opener_ctx,
15240            &mut opener_rx,
15241            capability_manifest("opener", &[], &[]),
15242            2,
15243        )
15244        .await;
15245        let (mut client_rx, _) = open_route_for_capability_test(
15246            &handler,
15247            &target_ctx,
15248            &mut target_rx,
15249            732,
15250            3,
15251            "target",
15252            Some(ConsumerIdentity {
15253                module_id: "opener".to_string(),
15254                launch_nonce: "opener-nonce".to_string(),
15255            }),
15256        )
15257        .await;
15258        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15259
15260        handler
15261            .handle_control_frame(
15262                &target_ctx,
15263                catalog_update_with_capabilities_frame(
15264                    4,
15265                    CapabilityDeclarations {
15266                        provides: Vec::new(),
15267                        requires: Vec::new(),
15268                        must_never_reach: Vec::new(),
15269                    },
15270                ),
15271            )
15272            .await
15273            .expect("claim removal catalog.update succeeds");
15274        assert_eq!(
15275            forwarding.active_binding_count().unwrap(),
15276            1,
15277            "removing an attested target claim must leave the route census unchanged"
15278        );
15279        assert!(
15280            client_rx.try_recv().is_err(),
15281            "claim removal must not emit route.closed or a client route GOODBYE"
15282        );
15283        assert!(
15284            target_rx.try_recv().is_err(),
15285            "claim removal must not send the target a route GOODBYE"
15286        );
15287    }
15288
15289    /// A direct client may open a route to a denied capability provider; this
15290    /// policy applies only to attested supervised module origins, not to direct clients.
15291    #[tokio::test]
15292    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15293        let registry = Arc::new(Registry::default());
15294        let forwarding = Arc::new(ForwardingTable::default());
15295        let supervisor = SupervisorHandle::new();
15296        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15297        let handler =
15298            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15299                .with_supervisor(supervisor);
15300        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15301        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15302        register_capability_manifest(
15303            &handler,
15304            &target_ctx,
15305            &mut target_rx,
15306            capability_manifest("target", &["credentials-provider/v1"], &[]),
15307            1,
15308        )
15309        .await;
15310        register_capability_manifest(
15311            &handler,
15312            &opener_ctx,
15313            &mut opener_rx,
15314            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15315            2,
15316        )
15317        .await;
15318
15319        let (_client_rx, bind) = open_route_for_capability_test(
15320            &handler,
15321            &target_ctx,
15322            &mut target_rx,
15323            742,
15324            3,
15325            "target",
15326            None,
15327        )
15328        .await;
15329        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15330            panic!("direct scope-honesty route must bind");
15331        };
15332        assert_eq!(principal, Some(Principal::Direct));
15333        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15334    }
15335
15336    /// A module that denies a capability receives no self-route exemption when it
15337    /// also attestedly provides that capability.
15338    #[tokio::test]
15339    async fn must_never_reach_self_route_is_capability_forbidden() {
15340        let registry = Arc::new(Registry::default());
15341        let forwarding = Arc::new(ForwardingTable::default());
15342        let supervisor = SupervisorHandle::new();
15343        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15344        let handler =
15345            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15346                .with_supervisor(supervisor);
15347        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15348        register_capability_manifest(
15349            &handler,
15350            &self_ctx,
15351            &mut self_rx,
15352            capability_manifest(
15353                "self-provider",
15354                &["credentials-provider/v1"],
15355                &["credentials-provider/v1"],
15356            ),
15357            1,
15358        )
15359        .await;
15360
15361        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15362        let replies = handler
15363            .handle_control_frame(
15364                &client_ctx,
15365                route_open_frame_with_admission_facts(
15366                    2,
15367                    "self-provider",
15368                    unique_project_root("admission-facts"),
15369                    Some(ConsumerIdentity {
15370                        module_id: "self-provider".to_string(),
15371                        launch_nonce: "self-nonce".to_string(),
15372                    }),
15373                    None,
15374                ),
15375            )
15376            .await
15377            .expect("self-route refusal returns a typed frame");
15378        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15379        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15380        assert!(
15381            self_rx.try_recv().is_err(),
15382            "self denial must not relay route.bind"
15383        );
15384    }
15385
15386    #[test]
15387    fn unsupported_channel_zero_frame_returns_error() {
15388        let handler = ControlHandler::default();
15389        let request = Frame::build(
15390            FrameType::Request,
15391            control_flags(),
15392            0,
15393            0,
15394            21,
15395            b"opaque".to_vec(),
15396        )
15397        .unwrap();
15398
15399        let response = handler
15400            .handle_control(ConnectionId::new(1), request)
15401            .unwrap();
15402
15403        assert_eq!(response[0].header.ty, FrameType::Error);
15404        assert_eq!(
15405            parse_error(&response[0])["code"],
15406            "unsupported_control_frame"
15407        );
15408    }
15409
15410    /// Blue/green swap at the control-plane boundary. The supervisor that opens
15411    /// a swap is not wired yet, so the candidate is registered here directly
15412    /// into the registry and forwarding candidate slots, the way the swap's
15413    /// HELLO admission will.
15414    mod swap {
15415        use super::*;
15416
15417        const INCUMBENT: ConnectionId = ConnectionId::new(30);
15418        const CANDIDATE: ConnectionId = ConnectionId::new(40);
15419
15420        struct Swap {
15421            registry: Arc<Registry>,
15422            forwarding: Arc<ForwardingTable>,
15423            handler: ControlHandler,
15424            incumbent_ctx: RouteCtx,
15425            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15426            candidate_ctx: RouteCtx,
15427            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15428        }
15429
15430        async fn swap_with_incumbent() -> Swap {
15431            let registry = Arc::new(Registry::default());
15432            let forwarding = Arc::new(ForwardingTable::default());
15433            let handler =
15434                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15435            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15436            hello_via_sink(
15437                &handler,
15438                &incumbent_ctx,
15439                &mut incumbent_rx,
15440                hello_frame("aft", PROTOCOL_VERSION, 7),
15441            )
15442            .await;
15443            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15444            Swap {
15445                registry,
15446                forwarding,
15447                handler,
15448                incumbent_ctx,
15449                incumbent_rx,
15450                candidate_ctx,
15451                candidate_rx,
15452            }
15453        }
15454
15455        fn register_candidate(swap: &Swap, ready: Option<bool>) {
15456            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15457            candidate_manifest.ready = ready;
15458            let registration = swap
15459                .registry
15460                .register_candidate_with_control_ops(
15461                    candidate_manifest,
15462                    PROTOCOL_VERSION,
15463                    CANDIDATE,
15464                    module_baseline_control_ops(),
15465                )
15466                .unwrap();
15467            swap.forwarding
15468                .register_candidate_module_connection(
15469                    CANDIDATE,
15470                    "aft".to_string(),
15471                    PROTOCOL_VERSION,
15472                    manifest_concurrency(&registration.manifest),
15473                    swap.candidate_ctx.egress.clone(),
15474                )
15475                .unwrap();
15476        }
15477
15478        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15479            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15480            swap.registry.promote_candidate("aft").unwrap().unwrap();
15481            cutover.incumbent.unwrap()
15482        }
15483
15484        /// A swap candidate's HELLO reserves its candidate registry entry
15485        /// before installing its candidate forwarding endpoint. The swap
15486        /// supervisor promotes a ready candidate as soon as it reads it as
15487        /// registered, and promotion cuts forwarding over to the candidate
15488        /// endpoint, so the lookup the supervisor reads
15489        /// (`Registry::published_registration`) must not return the candidate
15490        /// until that endpoint exists. Otherwise the cutover finds no candidate
15491        /// endpoint and the swap fails as if the candidate had disconnected.
15492        #[tokio::test]
15493        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15494            let mut swap = swap_with_incumbent().await;
15495            let hold = Arc::new(std::sync::Barrier::new(2));
15496            let handler = swap
15497                .handler
15498                .clone()
15499                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15500            handler
15501                .supervisor
15502                .open_swap("aft", "candidate-nonce".to_string());
15503            let hello = {
15504                let handler = handler.clone();
15505                let candidate_ctx = swap.candidate_ctx.clone();
15506                std::thread::spawn(move || {
15507                    handler
15508                        .handle_hello(
15509                            candidate_ctx.connection_id,
15510                            Some(candidate_ctx.egress.clone()),
15511                            hello_frame_with_nonce(
15512                                "aft",
15513                                PROTOCOL_VERSION,
15514                                8,
15515                                Some("candidate-nonce"),
15516                            ),
15517                        )
15518                        .unwrap()
15519                })
15520            };
15521
15522            hold.wait();
15523            // Parked: the candidate slot is claimed but not published, and
15524            // forwarding has no candidate endpoint.
15525            assert!(swap
15526                .registry
15527                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15528                .unwrap()
15529                .is_some());
15530            assert!(swap
15531                .registry
15532                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15533                .unwrap()
15534                .is_none());
15535            assert!(swap
15536                .forwarding
15537                .module_endpoint_for_connection(CANDIDATE)
15538                .unwrap()
15539                .is_none());
15540
15541            hold.wait();
15542            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15543            assert_eq!(
15544                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15545                FrameType::HelloAck
15546            );
15547            let published = swap
15548                .registry
15549                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15550                .unwrap()
15551                .expect("candidate is published once its HELLO completes");
15552            assert_eq!(published.connection_id, CANDIDATE);
15553            assert!(swap
15554                .forwarding
15555                .module_endpoint_for_connection(CANDIDATE)
15556                .unwrap()
15557                .is_some());
15558        }
15559
15560        fn keyed_total(counters: &Value, key: &str) -> u64 {
15561            counters[key]
15562                .as_object()
15563                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15564                .unwrap_or(0)
15565        }
15566
15567        #[tokio::test]
15568        async fn replacement_logs_old_end_and_new_admission_once() {
15569            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15570            let mut swap = swap_with_incumbent().await;
15571            swap.handler
15572                .supervisor
15573                .open_swap("aft", "candidate-nonce".to_string());
15574            hello_via_sink(
15575                &swap.handler,
15576                &swap.candidate_ctx,
15577                &mut swap.candidate_rx,
15578                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15579            )
15580            .await;
15581            cutover(&swap);
15582            swap.handler.cleanup_connection(INCUMBENT).unwrap();
15583
15584            let captured = crate::router::test_log::captured_logs(&logs);
15585            println!("captured replacement registry logs:\n{captured}");
15586            let new_admissions: Vec<_> = captured
15587                .lines()
15588                .filter(|line| {
15589                    line.contains("connection_id=40")
15590                        && line.contains("swap candidate registered; not routable until cutover")
15591                })
15592                .collect();
15593            assert_eq!(
15594                new_admissions.len(),
15595                1,
15596                "unexpected admission logs: {captured}"
15597            );
15598            assert!(new_admissions[0].contains("module_id=aft"));
15599            assert!(new_admissions[0].contains("connection_id=40"));
15600            assert!(
15601                new_admissions[0].contains("swap candidate registered; not routable until cutover")
15602            );
15603            assert!(new_admissions[0].contains("ready=true"));
15604
15605            let old_admissions: Vec<_> = captured
15606                .lines()
15607                .filter(|line| {
15608                    line.contains("module registered module_id=aft ")
15609                        && line.contains("connection_id=30")
15610                })
15611                .collect();
15612            assert_eq!(
15613                old_admissions.len(),
15614                1,
15615                "unexpected incumbent admission: {captured}"
15616            );
15617
15618            let old_ends: Vec<_> = captured
15619                .lines()
15620                .filter(|line| {
15621                    line.contains("connection_id=30") && line.contains("module registration ended")
15622                })
15623                .collect();
15624            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15625            assert!(old_ends[0].contains("module_id=aft"));
15626            assert!(old_ends[0].contains("connection_id=30"));
15627            assert!(old_ends[0].contains("reason=replaced"));
15628            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15629            assert_eq!(
15630                captured
15631                    .lines()
15632                    .filter(|line| line.contains("module registration promoted"))
15633                    .count(),
15634                1,
15635                "unexpected promotion logs: {captured}"
15636            );
15637        }
15638
15639        /// An ack from the incumbent for a bind it was sent before cutover,
15640        /// arriving before the incumbent is drained. The incumbent is the live
15641        /// connection carrying every other client's routes, so the ack must
15642        /// not end it: the waiting client is told to retry, the reservation is
15643        /// given back, and the incumbent is told to drop just that binding.
15644        #[tokio::test]
15645        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15646            let mut swap = swap_with_incumbent().await;
15647            let handler = swap.handler.clone();
15648
15649            // A co-tenant route, bound on the incumbent before the swap.
15650            let cotenant = ConnectionId::new(31);
15651            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15652            let (cotenant_task, cotenant_bind) = relay_route_open(
15653                &handler,
15654                cotenant,
15655                &cotenant_ctx.egress,
15656                &mut swap.incumbent_rx,
15657                100,
15658                "aft",
15659                "swap-cotenant",
15660            )
15661            .await;
15662            handler
15663                .handle_control_frame(
15664                    &swap.incumbent_ctx,
15665                    route_bind_ack(cotenant_bind.header.corr),
15666                )
15667                .await
15668                .unwrap();
15669            assert!(cotenant_task.await.unwrap().is_empty());
15670            let (cotenant_channel, cotenant_epoch) =
15671                published_route(&cotenant_rx.recv().await.unwrap());
15672
15673            // A second route.open, relayed to the incumbent and not yet acked.
15674            let caller = ConnectionId::new(32);
15675            let (caller_ctx, mut caller_rx) = route_ctx(caller);
15676            let (caller_task, caller_bind) = relay_route_open(
15677                &handler,
15678                caller,
15679                &caller_ctx.egress,
15680                &mut swap.incumbent_rx,
15681                101,
15682                "aft",
15683                "swap-caller",
15684            )
15685            .await;
15686            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15687
15688            register_candidate(&swap, None);
15689            cutover(&swap);
15690
15691            // The incumbent acks after promotion and before any drain.
15692            let ack = handler
15693                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15694                .await;
15695            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15696            if module_loop_error.is_some() {
15697                // What the connection loop does with an untranslated router
15698                // error: end the connection, releasing every route on it.
15699                handler.cleanup_connection(INCUMBENT).unwrap();
15700            }
15701
15702            // 1. The incumbent's other routes survive.
15703            assert!(
15704                cotenant_rx.try_recv().is_err(),
15705                "the co-tenant route on the incumbent was torn down by one late ack: \
15706                 {module_loop_error:?}"
15707            );
15708            assert!(matches!(
15709                swap.forwarding
15710                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15711                    .unwrap(),
15712                DataRoute::Client(DataRouteState::Bound(_))
15713            ));
15714            assert_eq!(module_loop_error, None);
15715            assert!(swap
15716                .registry
15717                .get_module_by_connection(INCUMBENT)
15718                .unwrap()
15719                .is_some());
15720
15721            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
15722            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15723                .await
15724                .expect("the incumbent is told to drop the abandoned binding")
15725                .unwrap()
15726                .frame;
15727            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15728            assert_eq!(goodbye.header.channel, abandoned_channel);
15729            assert_eq!(goodbye.header.epoch, abandoned_epoch);
15730            assert!(swap.incumbent_rx.try_recv().is_err());
15731
15732            // 3. The waiting client gets a retryable refusal and no route.
15733            let response = caller_task.await.unwrap();
15734            assert_eq!(response.len(), 1);
15735            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15736            assert!(caller_rx.try_recv().is_err());
15737
15738            // 4. The reservation pair is given back, and the pending bind
15739            //    settled exactly once: one accepted open (the co-tenant) and one
15740            //    refused open (the caller), nothing counted twice.
15741            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15742            let counters = handler.counters().snapshot();
15743            assert_eq!(
15744                keyed_total(&counters, "route_open_accepted_by_principal"),
15745                1
15746            );
15747            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15748            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15749        }
15750
15751        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
15752        /// id would resolve to the promoted candidate and every new route.open
15753        /// would be refused as reloading, leaving neither process routable.
15754        #[tokio::test]
15755        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15756            let mut swap = swap_with_incumbent().await;
15757            register_candidate(&swap, None);
15758            let incumbent = cutover(&swap);
15759            swap.forwarding
15760                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15761                .unwrap()
15762                .expect("the incumbent is still registered");
15763
15764            let client = ConnectionId::new(33);
15765            let (client_ctx, mut client_rx) = route_ctx(client);
15766            let route_handler = swap.handler.clone();
15767            let open_ctx = RouteCtx {
15768                connection_id: client,
15769                egress: client_ctx.egress.clone(),
15770            };
15771            let mut route_task = tokio::spawn(async move {
15772                route_handler
15773                    .handle_control_frame(
15774                        &open_ctx,
15775                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15776                    )
15777                    .await
15778                    .unwrap()
15779            });
15780            let bind = tokio::select! {
15781                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15782                response = &mut route_task => {
15783                    let response = response.unwrap();
15784                    panic!(
15785                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
15786                        parse_error(&response[0])["code"]
15787                    );
15788                }
15789            };
15790            swap.handler
15791                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15792                .await
15793                .unwrap();
15794            assert!(route_task.await.unwrap().is_empty());
15795            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15796            match swap
15797                .forwarding
15798                .lookup_data_route(client, channel, epoch)
15799                .unwrap()
15800            {
15801                DataRoute::Client(DataRouteState::Bound(route)) => {
15802                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15803                }
15804                other => panic!("expected a bound route on the candidate, got {other:?}"),
15805            }
15806            assert!(swap.incumbent_rx.try_recv().is_err());
15807        }
15808
15809        /// A candidate declares itself ready with `catalog.update` on its own
15810        /// connection. If the connection-keyed registry lookups searched only the
15811        /// active slot, this would answer `not_registered` and the candidate
15812        /// would never become ready.
15813        #[tokio::test]
15814        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15815            let swap = swap_with_incumbent().await;
15816            register_candidate(&swap, Some(false));
15817            let update = Frame::build(
15818                FrameType::Request,
15819                control_flags(),
15820                0,
15821                0,
15822                55,
15823                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15824                    provides: manifest("aft", PROTOCOL_VERSION).provides,
15825                    capabilities: None,
15826                    ready: Some(true),
15827                })
15828                .unwrap(),
15829            )
15830            .unwrap();
15831
15832            let replies = swap
15833                .handler
15834                .handle_control_frame(&swap.candidate_ctx, update)
15835                .await
15836                .unwrap();
15837
15838            assert_eq!(replies.len(), 1);
15839            assert_eq!(
15840                replies[0].header.ty,
15841                FrameType::Response,
15842                "candidate catalog.update was refused: {:?}",
15843                serde_json::from_slice::<Value>(&replies[0].body).ok()
15844            );
15845            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15846            assert_eq!(
15847                swap.registry
15848                    .get_module("aft")
15849                    .unwrap()
15850                    .unwrap()
15851                    .connection_id,
15852                INCUMBENT
15853            );
15854        }
15855    }
15856
15857    /// The HELLO gate while the supervisor has a swap open: only the nonce it
15858    /// minted for the candidate admits a second process, into the candidate
15859    /// slot, and that check runs ahead of the reserved-module gate.
15860    mod swap_admission {
15861        use super::*;
15862
15863        const INCUMBENT_NONCE: &str = "incumbent-nonce";
15864        const CANDIDATE_NONCE: &str = "candidate-nonce";
15865
15866        fn handler_with_incumbent(
15867            module_id: &str,
15868            reserved: bool,
15869        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15870            let registry = Arc::new(Registry::default());
15871            let supervisor = SupervisorHandle::new();
15872            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15873            if reserved {
15874                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15875            }
15876            let handler =
15877                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
15878            let incumbent = handler
15879                .handle_control(
15880                    ConnectionId::new(1),
15881                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15882                )
15883                .unwrap();
15884            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15885            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15886            (registry, supervisor, handler)
15887        }
15888
15889        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
15890        /// admits every nonce, so while a swap is open the swap gate is the only
15891        /// thing between a key-holder and the candidate slot. A nonce the
15892        /// supervisor did not mint, or none at all, is refused, and neither the
15893        /// incumbent's registration nor the candidate slot moves.
15894        #[test]
15895        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15896            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15897
15898            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15899                let replies = handler
15900                    .handle_control(
15901                        ConnectionId::new(connection),
15902                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15903                    )
15904                    .unwrap();
15905                assert_eq!(replies[0].header.ty, FrameType::Error);
15906                assert_eq!(
15907                    parse_error(&replies[0])["code"],
15908                    "swap_token_invalid",
15909                    "nonce {nonce:?}"
15910                );
15911            }
15912            assert!(registry.get_candidate("aft").unwrap().is_none());
15913            assert_eq!(
15914                registry.get_module("aft").unwrap().unwrap().connection_id,
15915                ConnectionId::new(1)
15916            );
15917
15918            // Control: the minted token is admitted, into the candidate slot,
15919            // and only once.
15920            let admitted = handler
15921                .handle_control(
15922                    ConnectionId::new(4),
15923                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15924                )
15925                .unwrap();
15926            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15927            assert_eq!(
15928                registry
15929                    .get_candidate("aft")
15930                    .unwrap()
15931                    .unwrap()
15932                    .connection_id,
15933                ConnectionId::new(4)
15934            );
15935            assert_eq!(
15936                registry.get_module("aft").unwrap().unwrap().connection_id,
15937                ConnectionId::new(1),
15938                "the candidate must not take the active slot"
15939            );
15940            let replayed = handler
15941                .handle_control(
15942                    ConnectionId::new(5),
15943                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15944                )
15945                .unwrap();
15946            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15947
15948            // The case only this gate covers: the incumbent has died mid-swap,
15949            // so its duplicate refusal is gone too, and without the gate a
15950            // key-holder would take the id's ACTIVE slot.
15951            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15952            let squatter = handler
15953                .handle_control(
15954                    ConnectionId::new(6),
15955                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15956                )
15957                .unwrap();
15958            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15959            assert!(
15960                registry.get_module("aft").unwrap().is_none(),
15961                "a squatter took the active slot of an id being swapped"
15962            );
15963        }
15964
15965        /// Design mutation arm (iii). A reserved module's candidate presents a
15966        /// nonce the reserved gate has never seen (that gate holds the
15967        /// incumbent's), so the swap gate must run first or the candidate is
15968        /// refused `reserved_module` and a reserved module can never be swapped.
15969        #[test]
15970        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15971            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15972
15973            let replies = handler
15974                .handle_control(
15975                    ConnectionId::new(2),
15976                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15977                )
15978                .unwrap();
15979
15980            assert_eq!(
15981                replies[0].header.ty,
15982                FrameType::HelloAck,
15983                "reserved candidate refused: {:?}",
15984                serde_json::from_slice::<Value>(&replies[0].body).ok()
15985            );
15986            assert_eq!(
15987                registry
15988                    .get_candidate("vault")
15989                    .unwrap()
15990                    .unwrap()
15991                    .connection_id,
15992                ConnectionId::new(2)
15993            );
15994        }
15995
15996        /// With no swap open the gate is inert: the incumbent's reserved gate
15997        /// and duplicate refusal behave exactly as before.
15998        #[test]
15999        fn without_an_open_swap_the_ordinary_gates_decide() {
16000            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16001            supervisor.close_swap("vault");
16002
16003            let candidate = handler
16004                .handle_control(
16005                    ConnectionId::new(2),
16006                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16007                )
16008                .unwrap();
16009            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16010            let duplicate = handler
16011                .handle_control(
16012                    ConnectionId::new(3),
16013                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16014                )
16015                .unwrap();
16016            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16017            assert!(registry.get_candidate("vault").unwrap().is_none());
16018        }
16019    }
16020
16021    /// `scope.sync` and `scope.describe` through the real control handler: who
16022    /// may sync is decided by the registration and launch nonce of the module
16023    /// connection, never by the request body.
16024    mod scopes {
16025        mod idle_tests {
16026            use super::*;
16027            include!("scope_idle_tests.rs");
16028        }
16029        mod route_open_timings {
16030            use super::*;
16031            include!("route_open_timing_tests.rs");
16032        }
16033        mod route_open_refusal_log {
16034            use super::*;
16035            include!("route_open_refusal_log_tests.rs");
16036        }
16037        mod activity {
16038            include!("activity_scope_tests.rs");
16039        }
16040        mod delta {
16041            include!("scope_handler_delta_tests.rs");
16042        }
16043        mod call_modes {
16044            include!("scope_call_mode_tests.rs");
16045        }
16046        #[tokio::test]
16047        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16048            let rig = rig().await;
16049            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16050                "s".into(),
16051                1,
16052                "tool".into(),
16053            ))
16054            .unwrap();
16055            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16056            let (direct, _rx) = wide_ctx(999);
16057            for (ctx, expected) in [
16058                (&rig.owner, FrameType::Response),
16059                (&direct, FrameType::Error),
16060            ] {
16061                let frame =
16062                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16063                        .unwrap();
16064                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16065                assert_eq!(replies[0].header.ty, expected);
16066                if expected == FrameType::Response {
16067                    let reply: subc_control::ScopeServedByReply =
16068                        serde_json::from_slice(&replies[0].body).unwrap();
16069                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16070                } else {
16071                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16072                }
16073            }
16074            let reply = call(
16075                &rig.handler,
16076                &rig.owner,
16077                &ModuleControlRequestFromModule::ScopeDescribe {
16078                    owner: Principal::Reserved {
16079                        module_id: OWNER.into(),
16080                    },
16081                    scope_ref: "s".into(),
16082                },
16083            )
16084            .await;
16085            assert!(matches!(
16086                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16087                ModuleControlResponseToModule::ScopeDescribe { .. }
16088            ));
16089        }
16090
16091        use subc_protocol::scope::{
16092            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16093            ScopeStatus,
16094        };
16095
16096        use super::*;
16097
16098        const OWNER: &str = "prefrontal-core";
16099
16100        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16101            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16102        }
16103
16104        async fn call(
16105            handler: &ControlHandler,
16106            ctx: &RouteCtx,
16107            request: &ModuleControlRequestFromModule,
16108        ) -> Frame {
16109            let body = serde_json::to_vec(request).unwrap();
16110            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16111            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16112            assert_eq!(replies.len(), 1, "{replies:?}");
16113            replies.pop().unwrap()
16114        }
16115
16116        async fn sync(
16117            handler: &ControlHandler,
16118            ctx: &RouteCtx,
16119            generation: u64,
16120            scopes: Vec<ScopeRecord>,
16121        ) -> Result<ModuleControlResponseToModule, String> {
16122            let reply = call(
16123                handler,
16124                ctx,
16125                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16126            )
16127            .await;
16128            match reply.header.ty {
16129                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16130                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16131            }
16132        }
16133
16134        async fn describe(
16135            handler: &ControlHandler,
16136            ctx: &RouteCtx,
16137            owner: &str,
16138            scope_ref: &str,
16139        ) -> ModuleControlResponseToModule {
16140            let reply = call(
16141                handler,
16142                ctx,
16143                &ModuleControlRequestFromModule::ScopeDescribe {
16144                    owner: Principal::Reserved {
16145                        module_id: owner.to_string(),
16146                    },
16147                    scope_ref: scope_ref.to_string(),
16148                },
16149            )
16150            .await;
16151            assert_eq!(
16152                reply.header.ty,
16153                FrameType::Response,
16154                "{:?}",
16155                parse_error(&reply)
16156            );
16157            serde_json::from_slice(&reply.body).unwrap()
16158        }
16159
16160        #[tokio::test]
16161        async fn scope_describe_never_echoes_bind_proof_or_hash() {
16162            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16163            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16164            let mut rig = rig().await;
16165            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16166                .await;
16167            let _bound = rig
16168                .bound(
16169                    Some(AFT),
16170                    PLEXUS,
16171                    Some(
16172                        rig.selector("s", Some(1))
16173                            .with_bind_proof(Some(PROOF.into())),
16174                    ),
16175                )
16176                .await;
16177            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16178            let output = serde_json::to_string(&reply).unwrap();
16179            assert!(output.contains("\"status\":\"live\""));
16180            assert!(!output.contains(PROOF));
16181            assert!(!output.contains(HASH));
16182        }
16183
16184        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
16185        async fn module(
16186            handler: &ControlHandler,
16187            connection: u64,
16188            module_id: &str,
16189            nonce: Option<&str>,
16190        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16191            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16192            hello_via_sink(
16193                handler,
16194                &ctx,
16195                &mut rx,
16196                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16197            )
16198            .await;
16199            (ctx, rx)
16200        }
16201
16202        /// `direct` and every other client connection has no registration, so
16203        /// it can neither sync nor own a scope.
16204        #[tokio::test]
16205        async fn a_client_connection_cannot_sync_or_describe() {
16206            let handler = ControlHandler::new(Arc::new(Registry::default()));
16207            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16208            for request in [
16209                ModuleControlRequestFromModule::ScopeSync {
16210                    generation: 1,
16211                    scopes: vec![head("s", 1)],
16212                },
16213                ModuleControlRequestFromModule::ScopeDescribe {
16214                    owner: Principal::Direct,
16215                    scope_ref: "s".to_string(),
16216                },
16217            ] {
16218                let reply = call(&handler, &ctx, &request).await;
16219                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16220            }
16221            assert!(
16222                !handler
16223                    .scopes
16224                    .read()
16225                    .unwrap()
16226                    .describe(
16227                        &Principal::Reserved {
16228                            module_id: OWNER.to_string()
16229                        },
16230                        "s"
16231                    )
16232                    .owner_synced
16233            );
16234        }
16235
16236        /// A module the supervisor did not spawn registers without a launch
16237        /// nonce, so it is never an owner's current launch.
16238        #[tokio::test]
16239        async fn a_module_without_a_supervised_launch_cannot_sync() {
16240            let handler = ControlHandler::new(Arc::new(Registry::default()));
16241            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16242            assert_eq!(
16243                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16244                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16245            );
16246        }
16247
16248        #[tokio::test]
16249        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16250            let supervisor = SupervisorHandle::new();
16251            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16252            let handler = ControlHandler::new(Arc::new(Registry::default()))
16253                .with_supervisor(supervisor.clone());
16254            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16255            sync(&handler, &incumbent, 1, vec![head("s", 1)])
16256                .await
16257                .expect("the current launch syncs");
16258
16259            // A swap candidate registers with the swap token and is refused
16260            // while the incumbent keeps syncing.
16261            supervisor.open_swap(OWNER, "n2".to_string());
16262            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16263            assert_eq!(
16264                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16265                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16266            );
16267            sync(&handler, &incumbent, 2, vec![head("s", 1)])
16268                .await
16269                .expect("the serving owner syncs during the swap");
16270
16271            // The swap fails and is rolled back. The candidate never held sync
16272            // authority, and still cannot sync.
16273            supervisor.close_swap(OWNER);
16274            assert_eq!(
16275                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16276                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16277            );
16278            sync(&handler, &incumbent, 3, vec![head("s", 1)])
16279                .await
16280                .expect("the serving owner syncs after the rollback");
16281            handler.cleanup_connection(candidate.connection_id).unwrap();
16282
16283            // A swap that cuts over. Promotion records the candidate's nonce as
16284            // the module's spawn nonce, which is what `set_spawn_nonce` does
16285            // here; the promoted connection then takes authority at any
16286            // generation and the superseded incumbent is refused.
16287            supervisor.open_swap(OWNER, "n3".to_string());
16288            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16289            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16290            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16291                .await
16292                .expect("the promoted launch takes authority");
16293            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16294                panic!("unexpected reply {reply:?}");
16295            };
16296            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16297            assert_eq!(
16298                sync(&handler, &incumbent, 4, Vec::new()).await,
16299                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16300            );
16301        }
16302
16303        /// Authority dies with its connection: the cleanup path releases it,
16304        /// so the owner's next connection takes it at any generation.
16305        #[tokio::test]
16306        async fn closing_the_authority_connection_frees_sync_authority() {
16307            let supervisor = SupervisorHandle::new();
16308            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16309            let handler = ControlHandler::new(Arc::new(Registry::default()))
16310                .with_supervisor(supervisor.clone());
16311            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16312            sync(&handler, &first, 10, vec![head("s", 1)])
16313                .await
16314                .unwrap();
16315            handler.cleanup_connection(first.connection_id).unwrap();
16316
16317            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16318            sync(&handler, &second, 1, vec![head("s", 1)])
16319                .await
16320                .expect("the next connection takes the released authority");
16321        }
16322
16323        #[tokio::test]
16324        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16325            let supervisor = SupervisorHandle::new();
16326            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16327            let handler =
16328                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16329            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16330            sync(&handler, &first, 10, vec![head("s", 1)])
16331                .await
16332                .unwrap();
16333            handler
16334                .handle_control_frame(
16335                    &first,
16336                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16337                )
16338                .await
16339                .unwrap();
16340            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16341            sync(&handler, &second, 1, vec![head("s", 1)])
16342                .await
16343                .expect("GOODBYE releases authority even if the old socket remains open");
16344        }
16345
16346        #[tokio::test]
16347        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16348            let registry = Arc::new(Registry::default());
16349            let supervisor_handle = SupervisorHandle::new();
16350            let supervisor =
16351                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16352                    .with_handle(supervisor_handle.clone())
16353                    .with_daemon_incarnation("incarnation-7".to_string());
16354            // Configured with enabled: false, so the supervisor lists the
16355            // module without spawning a process for it.
16356            supervisor
16357                .supervise_configured(
16358                    ModuleSpec {
16359                        module_id: OWNER.to_string(),
16360                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16361                        args: Vec::new(),
16362                        env: Vec::new(),
16363                        reserved: false,
16364                        reserved_prefixes: Vec::new(),
16365                        protocol: ModuleProtocol::Subc,
16366                        overlap: Default::default(),
16367                    },
16368                    false,
16369                )
16370                .unwrap();
16371            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16372            let handler =
16373                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
16374            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16375
16376            // Configured but not yet synced: a reader waits for the owner.
16377            let ModuleControlResponseToModule::ScopeDescribe {
16378                status,
16379                daemon_incarnation,
16380                owner_synced,
16381                owner_configured,
16382                scope,
16383                ..
16384            } = describe(&handler, &reader, OWNER, "s").await
16385            else {
16386                panic!("not a describe reply");
16387            };
16388            assert_eq!(status, ScopeStatus::NotLive);
16389            assert_eq!(daemon_incarnation, "incarnation-7");
16390            assert!(!owner_synced);
16391            assert!(owner_configured);
16392            assert!(scope.is_none());
16393
16394            // Not a supervised module: the owner will never sync, and a reader
16395            // refuses rather than waits.
16396            let ModuleControlResponseToModule::ScopeDescribe {
16397                status,
16398                owner_configured,
16399                ..
16400            } = describe(&handler, &reader, "ghost", "s").await
16401            else {
16402                panic!("not a describe reply");
16403            };
16404            assert_eq!(status, ScopeStatus::NotLive);
16405            assert!(!owner_configured);
16406
16407            // Live, with the stamp fields and the computed owner_authorized.
16408            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16409            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16410            let ModuleControlResponseToModule::ScopeDescribe {
16411                status,
16412                scope_epoch,
16413                owner_synced,
16414                scope,
16415                ..
16416            } = describe(&handler, &reader, OWNER, "s").await
16417            else {
16418                panic!("not a describe reply");
16419            };
16420            assert_eq!(status, ScopeStatus::Live);
16421            assert_eq!(scope_epoch, Some(4));
16422            assert!(owner_synced);
16423            let stamp = scope.expect("a live scope carries its stamp");
16424            assert!(
16425                stamp.owner_authorized,
16426                "prefrontal-core is the default authority"
16427            );
16428            assert_eq!(stamp.kind, ScopeKind::Head);
16429        }
16430
16431        #[tokio::test]
16432        async fn scope_authority_owners_decides_owner_authorized() {
16433            let supervisor = SupervisorHandle::new();
16434            supervisor.set_spawn_nonce("broca", "b1".to_string());
16435            let handler = ControlHandler::new(Arc::new(Registry::default()))
16436                .with_supervisor(supervisor)
16437                .with_scope_authority_owners(vec!["broca".to_string()]);
16438            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16439            let mut gated = head("s", 1);
16440            gated.attributes.agent_id = Some("agent".to_string());
16441            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16442            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16443                describe(&handler, &broca, "broca", "s").await
16444            else {
16445                panic!("not a describe reply");
16446            };
16447            assert!(scope.unwrap().owner_authorized);
16448        }
16449
16450        /// With route admission, the stamp, the commit re-check and drains in
16451        /// place, the feature is advertised: the module ops in HELLO_ACK, and
16452        /// `scopes/v1` in HELLO_ACK and `server.describe`.
16453        #[tokio::test]
16454        async fn scope_ops_and_the_scopes_capability_are_advertised() {
16455            let handler = ControlHandler::new(Arc::new(Registry::default()));
16456            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16457            let ack = hello_via_sink(
16458                &handler,
16459                &ctx,
16460                &mut rx,
16461                hello_frame("m", PROTOCOL_VERSION, 1),
16462            )
16463            .await;
16464            let ack = parse_ack(&ack);
16465            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16466                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16467            }
16468            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16469
16470            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16471            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16472            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16473            let reply = handler
16474                .handle_control_frame(&client, frame)
16475                .await
16476                .unwrap()
16477                .pop()
16478                .unwrap();
16479            let ClientControlResponse::ServerDescribe { capabilities, .. } =
16480                serde_json::from_slice(&reply.body).unwrap()
16481            else {
16482                panic!("not a server.describe reply");
16483            };
16484            assert!(
16485                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16486                "{capabilities:?}"
16487            );
16488        }
16489
16490        // ---- route admission, stamps, commit re-check and drains ----------
16491
16492        const PLEXUS: &str = "plexus";
16493        const OTHER: &str = "other";
16494        const AFT: &str = "aft";
16495        const BROCA: &str = "broca";
16496        const MAGIC: &str = "magic-context";
16497
16498        fn nonce(module_id: &str) -> String {
16499            format!("nonce-{module_id}")
16500        }
16501
16502        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16503            let (tx, rx) = mpsc::channel(64);
16504            (
16505                RouteCtx {
16506                    connection_id: ConnectionId::new(connection),
16507                    egress: FrameSink::new(tx),
16508                },
16509                rx,
16510            )
16511        }
16512
16513        /// A daemon with a configured owner (prefrontal-core) registered on its
16514        /// own module connection, two routable targets (plexus, other), and
16515        /// launch nonces minted for the modules that open routes as carriers.
16516        struct Rig {
16517            handler: ControlHandler,
16518            forwarding: Arc<ForwardingTable>,
16519            owner: RouteCtx,
16520            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16521            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16522            generation: u64,
16523            next_connection: u64,
16524            _supervisor: Supervisor,
16525        }
16526
16527        async fn rig() -> Rig {
16528            rig_with_flow_support(true).await
16529        }
16530
16531        async fn rig_with_flow_support(flow_support: bool) -> Rig {
16532            rig_with_scope_capabilities(flow_support, false).await
16533        }
16534
16535        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16536            let mut provides = Vec::new();
16537            if flow_support {
16538                provides.push("flow-scopes/v1");
16539            }
16540            if run_support {
16541                provides.push("agent-run-scopes/v1");
16542            }
16543            rig_with_target_capabilities(&provides).await
16544        }
16545
16546        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16547            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16548            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16549            // Decoder versions alone cannot admit scoped calls: only a capability
16550            // declaration promises the required run or flow behaviour.
16551            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16552            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16553            Frame::build(
16554                FrameType::Hello,
16555                control_flags(),
16556                0,
16557                0,
16558                connection,
16559                serde_json::to_vec(&body).unwrap(),
16560            )
16561            .unwrap()
16562        }
16563
16564        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16565            let registry = Arc::new(Registry::default());
16566            let forwarding = Arc::new(ForwardingTable::default());
16567            let supervisor_handle = SupervisorHandle::new();
16568            let supervisor =
16569                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16570                    .with_handle(supervisor_handle.clone());
16571            supervisor
16572                .supervise_configured(
16573                    ModuleSpec {
16574                        module_id: OWNER.to_string(),
16575                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16576                        args: Vec::new(),
16577                        env: Vec::new(),
16578                        reserved: false,
16579                        reserved_prefixes: Vec::new(),
16580                        protocol: ModuleProtocol::Subc,
16581                        overlap: Default::default(),
16582                    },
16583                    false,
16584                )
16585                .unwrap();
16586            for module_id in [OWNER, AFT, BROCA, MAGIC] {
16587                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16588            }
16589            let handler =
16590                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16591                    .with_supervisor(supervisor_handle);
16592            let (owner, mut owner_rx) = wide_ctx(1);
16593            hello_via_sink(
16594                &handler,
16595                &owner,
16596                &mut owner_rx,
16597                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16598            )
16599            .await;
16600            let mut modules = BTreeMap::new();
16601            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16602                let (ctx, mut rx) = wide_ctx(connection);
16603                let hello = hello_with_capabilities(module_id, connection, provides);
16604                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16605                modules.insert(module_id.to_string(), (ctx, rx));
16606            }
16607            Rig {
16608                handler,
16609                forwarding,
16610                owner,
16611                _owner_rx: owner_rx,
16612                modules,
16613                generation: 0,
16614                next_connection: 100,
16615                _supervisor: supervisor,
16616            }
16617        }
16618
16619        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16620            ScopeCarrier::new(Principal::Reserved {
16621                module_id: module_id.to_string(),
16622            })
16623            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16624        }
16625
16626        /// The scope most tests open under: aft carries to any module, broca
16627        /// only to plexus and other, and the owner delegates as agent-1.
16628        fn session(scope_epoch: u64) -> ScopeRecord {
16629            let mut record = head("s", scope_epoch);
16630            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16631            record.attributes.agent_id = Some("agent-1".to_string());
16632            record.attributes.delegates = true;
16633            record
16634        }
16635
16636        impl Rig {
16637            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16638                self.generation += 1;
16639                sync(&self.handler, &self.owner, self.generation, scopes)
16640                    .await
16641                    .expect("the owner's sync is accepted");
16642            }
16643
16644            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16645                ScopeSelector::new(
16646                    Principal::Reserved {
16647                        module_id: OWNER.to_string(),
16648                    },
16649                    scope_ref.to_string(),
16650                    scope_epoch,
16651                )
16652            }
16653
16654            fn open_frame(
16655                &mut self,
16656                opener: Option<&str>,
16657                target: &str,
16658                scope: Option<ScopeSelector>,
16659            ) -> (
16660                RouteCtx,
16661                mpsc::Receiver<crate::router::OutboundFrame>,
16662                Frame,
16663            ) {
16664                self.next_connection += 1;
16665                let (ctx, rx) = wide_ctx(self.next_connection);
16666                let root = unique_project_root("scoped-open");
16667                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16668                    target: RouteTarget::ToolProvider {
16669                        module_id: target.to_string(),
16670                    },
16671                    identity: BindIdentity::new(
16672                        root.path().to_path_buf(),
16673                        "unit".to_string(),
16674                        "session".to_string(),
16675                    ),
16676                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
16677                        module_id: module_id.to_string(),
16678                        launch_nonce: nonce(module_id),
16679                    }),
16680                    consumer_capabilities: None,
16681                    role_versions: None,
16682                    admission_facts: None,
16683                    scope,
16684                })
16685                .unwrap();
16686                let frame = Frame::build(
16687                    FrameType::Request,
16688                    control_flags(),
16689                    0,
16690                    0,
16691                    self.next_connection,
16692                    body,
16693                )
16694                .unwrap();
16695                (ctx, rx, frame)
16696            }
16697
16698            /// Open and expect a refusal before anything is relayed.
16699            async fn refused(
16700                &mut self,
16701                opener: Option<&str>,
16702                target: &str,
16703                scope: Option<ScopeSelector>,
16704            ) -> String {
16705                self.refusal_body(opener, target, scope).await["code"]
16706                    .as_str()
16707                    .unwrap()
16708                    .to_string()
16709            }
16710
16711            async fn refusal_body(
16712                &mut self,
16713                opener: Option<&str>,
16714                target: &str,
16715                scope: Option<ScopeSelector>,
16716            ) -> Value {
16717                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16718                let replies = tokio::time::timeout(
16719                    Duration::from_secs(2),
16720                    self.handler.handle_control_frame(&ctx, frame),
16721                )
16722                .await
16723                .expect("the open must be refused before waiting for a bind ack")
16724                .unwrap();
16725                assert_eq!(replies.len(), 1, "{replies:?}");
16726                assert_eq!(replies[0].header.ty, FrameType::Error);
16727                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16728                assert!(
16729                    module_rx.try_recv().is_err(),
16730                    "a refused open relays nothing"
16731                );
16732                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16733                parse_error(&replies[0])
16734            }
16735
16736            /// Start an open and return its task and the bind the target got.
16737            async fn relayed(
16738                &mut self,
16739                opener: Option<&str>,
16740                target: &str,
16741                scope: Option<ScopeSelector>,
16742            ) -> Relayed {
16743                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16744                let handler = self.handler.clone();
16745                let task_ctx = ctx.clone();
16746                let task = tokio::spawn(async move {
16747                    handler
16748                        .handle_control_frame(&task_ctx, frame)
16749                        .await
16750                        .unwrap()
16751                });
16752                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16753                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16754                    .await
16755                    .expect("the target receives the relayed route.bind")
16756                    .unwrap()
16757                    .frame;
16758                Relayed {
16759                    target: target.to_string(),
16760                    client: ctx,
16761                    client_rx: rx,
16762                    task,
16763                    bind,
16764                }
16765            }
16766
16767            async fn ack(&self, relayed: &Relayed) {
16768                let (module, _) = &self.modules[&relayed.target];
16769                self.handler
16770                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16771                    .await
16772                    .unwrap();
16773            }
16774
16775            /// Open, ack and return the bound route.
16776            async fn bound(
16777                &mut self,
16778                opener: Option<&str>,
16779                target: &str,
16780                scope: Option<ScopeSelector>,
16781            ) -> Bound {
16782                let relayed = self.relayed(opener, target, scope).await;
16783                self.ack(&relayed).await;
16784                let Relayed {
16785                    target,
16786                    client,
16787                    mut client_rx,
16788                    task,
16789                    bind,
16790                } = relayed;
16791                assert!(
16792                    task.await.unwrap().is_empty(),
16793                    "the open is answered by commit"
16794                );
16795                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16796                Bound {
16797                    target,
16798                    client,
16799                    client_rx,
16800                    channel,
16801                    epoch,
16802                    bind,
16803                }
16804            }
16805
16806            fn live(&self, route: &Bound) -> bool {
16807                matches!(
16808                    self.forwarding
16809                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16810                        .unwrap(),
16811                    DataRoute::Client(DataRouteState::Bound(_))
16812                )
16813            }
16814        }
16815
16816        struct Relayed {
16817            target: String,
16818            client: RouteCtx,
16819            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16820            task: tokio::task::JoinHandle<Vec<Frame>>,
16821            bind: Frame,
16822        }
16823
16824        struct Bound {
16825            target: String,
16826            client: RouteCtx,
16827            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16828            channel: u16,
16829            epoch: u32,
16830            bind: Frame,
16831        }
16832
16833        impl Bound {
16834            /// The reason of the `route.closed` this client was sent, after
16835            /// checking it also got a GOODBYE on exactly this route.
16836            fn closed_reason(&mut self) -> RouteCloseReason {
16837                let mut reason = None;
16838                let mut goodbye = false;
16839                while let Ok(outbound) = self.client_rx.try_recv() {
16840                    let frame = outbound.frame;
16841                    match frame.header.ty {
16842                        FrameType::Goodbye => {
16843                            assert_eq!(
16844                                (frame.header.channel, frame.header.epoch),
16845                                (self.channel, self.epoch)
16846                            );
16847                            goodbye = true;
16848                        }
16849                        FrameType::Push => {
16850                            let ClientControlPush::RouteClosed {
16851                                reason: r,
16852                                module_id,
16853                                ..
16854                            } = serde_json::from_slice(&frame.body).unwrap()
16855                            else {
16856                                panic!("unexpected push");
16857                            };
16858                            assert_eq!(module_id, self.target);
16859                            reason = Some(r);
16860                        }
16861                        other => panic!("unexpected frame {other:?}"),
16862                    }
16863                }
16864                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16865                reason.expect("the client is told why the route closed")
16866            }
16867
16868            fn untouched(&mut self) -> bool {
16869                self.client_rx.try_recv().is_err()
16870            }
16871
16872            fn stamp(&self) -> Option<ScopeStamp> {
16873                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16874                    ModuleControlRequest::RouteBind { scope, .. } => scope,
16875                    other => panic!("expected a route.bind, got {other:?}"),
16876                }
16877            }
16878        }
16879
16880        #[tokio::test]
16881        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16882        ) {
16883            let mut rig = rig().await;
16884            let mut record = session(1);
16885            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16886            record.child_owners = vec![Principal::Reserved {
16887                module_id: MAGIC.to_string(),
16888            }];
16889            rig.sync(vec![record]).await;
16890            let scope = || Some(rig_selector("s", Some(1)));
16891
16892            // Admitted: the owner, a bare carrier to any module, a targeted
16893            // carrier to its listed module.
16894            rig.bound(Some(OWNER), PLEXUS, scope()).await;
16895            rig.bound(Some(AFT), OTHER, scope()).await;
16896            rig.bound(Some(BROCA), PLEXUS, scope()).await;
16897
16898            // Refused scope_not_carrier: a targeted carrier to an unlisted
16899            // module, a module that is not listed at all (a child owner is not
16900            // a carrier), and a direct key-holder.
16901            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16902                assert_eq!(
16903                    rig.refused(opener, target, scope()).await,
16904                    error_codes::SCOPE_NOT_CARRIER,
16905                    "{opener:?} -> {target}"
16906                );
16907            }
16908        }
16909
16910        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16911            ScopeSelector::new(
16912                Principal::Reserved {
16913                    module_id: OWNER.to_string(),
16914                },
16915                scope_ref.to_string(),
16916                scope_epoch,
16917            )
16918        }
16919
16920        #[tokio::test]
16921        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16922            let mut rig = rig().await;
16923            rig.sync(vec![session(1)]).await;
16924            for opener in [OWNER, AFT] {
16925                assert_eq!(
16926                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16927                        .await,
16928                    error_codes::SCOPE_EPOCH_REQUIRED,
16929                    "{opener}"
16930                );
16931            }
16932        }
16933
16934        #[tokio::test]
16935        async fn admission_separates_not_synced_not_live_and_ended() {
16936            let mut rig = rig().await;
16937            // Before the configured owner's first sync: retryable.
16938            let code = rig
16939                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16940                .await;
16941            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16942            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16943
16944            // An owner that is not configured will never sync: terminal.
16945            let ghost = ScopeSelector::new(
16946                Principal::Reserved {
16947                    module_id: "ghost".to_string(),
16948                },
16949                "s".to_string(),
16950                Some(1),
16951            );
16952            assert_eq!(
16953                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16954                error_codes::SCOPE_NOT_LIVE
16955            );
16956
16957            rig.sync(vec![session(2)]).await;
16958            assert_eq!(
16959                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16960                    .await,
16961                error_codes::SCOPE_NOT_LIVE
16962            );
16963            for epoch in [1, 3] {
16964                assert_eq!(
16965                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16966                        .await,
16967                    error_codes::SCOPE_ENDED,
16968                    "epoch {epoch}"
16969                );
16970            }
16971            // Control: the live epoch is admitted.
16972            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16973                .await;
16974        }
16975
16976        #[tokio::test]
16977        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16978            let mut rig = rig().await;
16979            rig.sync(vec![session(1)]).await;
16980            let route = rig
16981                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16982                .await;
16983            let stamp = route.stamp().expect("a scoped bind carries the stamp");
16984            assert_eq!(stamp.scope_ref, "s");
16985            assert_eq!(stamp.scope_epoch, 1);
16986            assert_eq!(stamp.kind, ScopeKind::Head);
16987            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16988            assert!(stamp.attributes.delegates);
16989            assert!(stamp.owner_authorized);
16990            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16991            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16992
16993            // broca owns a scope of its own on its own module connection; it is
16994            // not in scope_authority_owners, so its stamp is not authorized.
16995            let (broca, mut broca_rx) = wide_ctx(50);
16996            hello_via_sink(
16997                &rig.handler,
16998                &broca,
16999                &mut broca_rx,
17000                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17001            )
17002            .await;
17003            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17004                .await
17005                .unwrap();
17006            let own = ScopeSelector::new(
17007                Principal::Reserved {
17008                    module_id: BROCA.to_string(),
17009                },
17010                "b".to_string(),
17011                Some(1),
17012            );
17013            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17014            assert!(!route.stamp().unwrap().owner_authorized);
17015        }
17016
17017        #[tokio::test]
17018        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17019            let mut rig = rig().await;
17020            let mut record = head("s", 1);
17021            record.carriers = vec![carrier(AFT, None)];
17022            let flow_id = "Flow:run-7/step_2!~";
17023            record.attributes.flow_id = Some(flow_id.to_string());
17024            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17025                .await
17026                .unwrap();
17027            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17028                panic!("not a sync reply");
17029            };
17030            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17031            let route = rig
17032                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17033                .await;
17034            assert!(rig.live(&route), "the stamped bind committed");
17035            let stamp = route.stamp().expect("a flow scope carries a stamp");
17036            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17037            assert_eq!(stamp.attributes.agent_id, None);
17038            assert!(!stamp.attributes.delegates);
17039            assert!(stamp.owner_authorized);
17040            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17041            assert_eq!(unscoped.stamp(), None);
17042        }
17043
17044        #[tokio::test]
17045        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17046            let mut rig = rig_with_flow_support(false).await;
17047            let mut record = session(1);
17048            record.attributes.flow_id = Some("flow:7".to_string());
17049            rig.sync(vec![record]).await;
17050            for opener in [OWNER, AFT] {
17051                let body = rig
17052                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17053                    .await;
17054                assert_eq!(body["code"], "target_flow_unsupported");
17055                let message = body["message"].as_str().unwrap();
17056                for required in [PLEXUS, "flow-scopes/v1"] {
17057                    assert!(message.contains(required), "{message}");
17058                }
17059            }
17060        }
17061
17062        #[tokio::test]
17063        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17064            let mut rig = rig_with_flow_support(true).await;
17065            let mut record = session(1);
17066            record.attributes.flow_id = Some("flow:7".to_string());
17067            rig.sync(vec![record]).await;
17068            for opener in [OWNER, AFT] {
17069                let route = rig
17070                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17071                    .await;
17072                assert!(rig.live(&route));
17073                assert_eq!(
17074                    route.stamp().unwrap().attributes.flow_id.as_deref(),
17075                    Some("flow:7")
17076                );
17077            }
17078        }
17079
17080        #[tokio::test]
17081        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17082            use std::future::Future;
17083
17084            let mut rig = rig_with_flow_support(true).await;
17085            let mut record = session(1);
17086            record.attributes.flow_id = Some("flow:7".to_string());
17087            rig.sync(vec![record]).await;
17088            let (client, mut client_rx, frame) =
17089                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17090            // Hold the route-open response permit so admission sees the first
17091            // target but relay reservation cannot capture an endpoint yet.
17092            for _ in 0..64 {
17093                client.egress.try_send(route_bind_ack(1)).unwrap();
17094            }
17095            let handler = rig.handler.clone();
17096            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17097            std::future::poll_fn(|cx| {
17098                assert!(open.as_mut().poll(cx).is_pending());
17099                std::task::Poll::Ready(())
17100            })
17101            .await;
17102            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17103
17104            let old_connection = rig.modules[PLEXUS].0.connection_id;
17105            rig.handler.cleanup_connection(old_connection).unwrap();
17106            let (replacement, mut replacement_rx) = wide_ctx(200);
17107            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17108            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17109            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17110            let hello = Frame::build(
17111                FrameType::Hello,
17112                control_flags(),
17113                0,
17114                0,
17115                200,
17116                serde_json::to_vec(&body).unwrap(),
17117            )
17118            .unwrap();
17119            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17120
17121            client_rx.try_recv().unwrap();
17122            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17123                .await
17124                .expect("the replacement is refused without waiting for a bind ack")
17125                .unwrap();
17126            assert_eq!(replies.len(), 1);
17127            let body = parse_error(&replies[0]);
17128            assert_eq!(body["code"], "target_flow_unsupported");
17129            for required in [PLEXUS, "flow-scopes/v1"] {
17130                assert!(body["message"].as_str().unwrap().contains(required));
17131            }
17132            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17133            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17134            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17135            assert!(rig
17136                .handler
17137                .registry
17138                .get_module_by_connection(replacement.connection_id)
17139                .unwrap()
17140                .is_some());
17141        }
17142
17143        #[tokio::test]
17144        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17145            use std::future::Future;
17146            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17147            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17148            const NEXT_HASH: &str =
17149                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17150            let mut rig = rig().await;
17151            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17152            record.carriers.push(ScopeCarrier::new(Principal::Direct));
17153            rig.sync(vec![record.clone()]).await;
17154            let (client, mut rx, frame) = rig.open_frame(
17155                None,
17156                PLEXUS,
17157                Some(
17158                    rig.selector("s", Some(1))
17159                        .with_bind_proof(Some(PROOF.into())),
17160                ),
17161            );
17162            // Hold egress reservation while the scope owner changes its bind-proof
17163            // hash, so a proof valid at admission must be checked again at relay.
17164            for _ in 0..64 {
17165                client.egress.try_send(route_bind_ack(1)).unwrap();
17166            }
17167            let handler = rig.handler.clone();
17168            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17169            std::future::poll_fn(|cx| {
17170                assert!(open.as_mut().poll(cx).is_pending());
17171                std::task::Poll::Ready(())
17172            })
17173            .await;
17174            record.bind_proof_sha256 = Some(NEXT_HASH.into());
17175            rig.sync(vec![record]).await;
17176            rx.try_recv().unwrap();
17177            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17178                .await
17179                .unwrap()
17180                .unwrap();
17181            assert_eq!(
17182                parse_error(&replies[0])["code"],
17183                error_codes::SCOPE_BIND_PROOF_MISMATCH
17184            );
17185            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17186            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17187            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17188        }
17189
17190        #[tokio::test]
17191        async fn scope_bind_proof_reserved_openers_ignore_proof() {
17192            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17193            let mut rig = rig().await;
17194            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17195                .await;
17196            for opener in [OWNER, AFT] {
17197                for proof in [None, Some("malformed+proof=")] {
17198                    let route = rig
17199                        .bound(
17200                            Some(opener),
17201                            PLEXUS,
17202                            Some(
17203                                rig.selector("s", Some(1))
17204                                    .with_bind_proof(proof.map(str::to_string)),
17205                            ),
17206                        )
17207                        .await;
17208                    assert!(rig.live(&route));
17209                }
17210            }
17211        }
17212
17213        #[tokio::test]
17214        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17215            use std::future::Future;
17216            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17217            const NEXT_HASH: &str =
17218                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17219            let mut rig = rig().await;
17220            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17221                .await;
17222            let (client, mut rx, frame) =
17223                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17224            for _ in 0..64 {
17225                client.egress.try_send(route_bind_ack(1)).unwrap();
17226            }
17227            let handler = rig.handler.clone();
17228            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17229            std::future::poll_fn(|cx| {
17230                assert!(open.as_mut().poll(cx).is_pending());
17231                std::task::Poll::Ready(())
17232            })
17233            .await;
17234            rig.sync(vec![
17235                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17236            ])
17237            .await;
17238            rx.try_recv().unwrap();
17239            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17240            let bind = std::future::poll_fn(|cx| {
17241                assert!(open.as_mut().poll(cx).is_pending());
17242                module_rx.poll_recv(cx)
17243            })
17244            .await
17245            .unwrap()
17246            .frame;
17247            handler
17248                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17249                .await
17250                .unwrap();
17251            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17252                .await
17253                .unwrap()
17254                .unwrap();
17255            assert_eq!(replies.len(), 1);
17256            // Proof rechecking allowed the relay. Independent tag settlement
17257            // still asks the opener to retry after any same-epoch content change.
17258            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17259            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17260            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17261            let route = rig
17262                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17263                .await;
17264            assert!(rig.live(&route));
17265        }
17266
17267        #[tokio::test]
17268        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17269        {
17270            let mut rig = rig_with_flow_support(false).await;
17271            rig.sync(vec![session(1)]).await;
17272            let route = rig
17273                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17274                .await;
17275            assert!(rig.live(&route));
17276            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17277            let mut record = session(1);
17278            record.attributes.flow_id = Some("flow:7".to_string());
17279            rig.sync(vec![record]).await;
17280            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17281            assert!(rig.live(&unscoped));
17282            assert_eq!(unscoped.stamp(), None);
17283        }
17284
17285        #[tokio::test]
17286        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17287            let mut rig = rig().await;
17288            let mut record = session(1);
17289            record.attributes.flow_id = Some("flow:7".to_string());
17290            rig.sync(vec![record.clone()]).await;
17291            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17292            let mut owner_route = rig
17293                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17294                .await;
17295            let mut carrier_route = rig
17296                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17297                .await;
17298            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17299            rig.sync(vec![record.clone()]).await;
17300            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17301            assert!(rig.live(&owner_route) && owner_route.untouched());
17302            assert!(rig.live(&carrier_route) && carrier_route.untouched());
17303
17304            record.attributes.flow_id = Some("flow:8".to_string());
17305            rig.sync(vec![record]).await;
17306            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17307            let before = before.unwrap();
17308            assert_eq!(after.scope_epoch, before.scope_epoch);
17309            assert!(after.version > before.version);
17310            for route in [&mut owner_route, &mut carrier_route] {
17311                assert!(!rig.live(route));
17312                assert_eq!(
17313                    route.closed_reason(),
17314                    RouteCloseReason::ScopeDelegationChanged
17315                );
17316            }
17317            assert!(rig.live(&unscoped) && unscoped.untouched());
17318            // Each provider also receives a GOODBYE for its drained route;
17319            // consume it before expecting the next route.bind on that sink.
17320            for target in [PLEXUS, OTHER] {
17321                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17322                let goodbye = module_rx
17323                    .try_recv()
17324                    .expect("the provider sees the drain")
17325                    .frame;
17326                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17327                assert!(module_rx.try_recv().is_err());
17328            }
17329            let rebound = rig
17330                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17331                .await;
17332            assert_eq!(
17333                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17334                Some("flow:8")
17335            );
17336        }
17337
17338        /// The owner's sync lands between admission and the module's ack. The
17339        /// open is refused by name, the module's other routes stay up, and the
17340        /// reserved pair is released. Changed content is retryable; an ended
17341        /// scope is not.
17342        #[tokio::test]
17343        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17344            let mut rig = rig().await;
17345            rig.sync(vec![session(1)]).await;
17346            let mut cotenant = rig
17347                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17348                .await;
17349
17350            let mut changed = session(1);
17351            changed.child_owners.push(Principal::Reserved {
17352                module_id: MAGIC.to_string(),
17353            });
17354            let mut ended = None;
17355            for (code, next) in [
17356                (error_codes::SCOPE_CHANGED, vec![changed]),
17357                (error_codes::SCOPE_ENDED, Vec::new()),
17358            ] {
17359                let relayed = rig
17360                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17361                    .await;
17362                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17363                ended = Some(next.is_empty());
17364                rig.sync(next).await;
17365                rig.ack(&relayed).await;
17366                let replies = relayed.task.await.unwrap();
17367                assert_eq!(replies.len(), 1, "{replies:?}");
17368                assert_eq!(parse_error(&replies[0])["code"], code);
17369                assert_eq!(
17370                    subc_protocol::error_codes::is_retryable_route_open(code),
17371                    code == error_codes::SCOPE_CHANGED
17372                );
17373                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17374                // The module is told to drop just the binding it created.
17375                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17376                // Collected, because ending the scope also closes the co-tenant
17377                // route, whose GOODBYE comes first.
17378                let mut goodbyes = Vec::new();
17379                while let Ok(outbound) = plexus_rx.try_recv() {
17380                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17381                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17382                }
17383                assert!(
17384                    goodbyes.contains(&(bind_channel, bind_epoch)),
17385                    "{goodbyes:?}"
17386                );
17387                assert!(rig
17388                    .handler
17389                    .registry
17390                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17391                    .unwrap()
17392                    .is_some());
17393            }
17394            assert_eq!(ended, Some(true));
17395            // The co-tenant stayed up through the change, and closed only when
17396            // the scope ended, by the drain rule rather than by the commit.
17397            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17398        }
17399
17400        /// Each row of the drain table on one set of routes: the owner's, a
17401        /// bare carrier's, and a targeted carrier's to each of its targets.
17402        #[tokio::test]
17403        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17404            struct Case {
17405                name: &'static str,
17406                change: fn(&mut ScopeRecord),
17407                /// Closed routes by index: owner->plexus, aft->plexus,
17408                /// broca->plexus, broca->other.
17409                closed: [Option<RouteCloseReason>; 4],
17410            }
17411            use RouteCloseReason::*;
17412            let cases = [
17413                Case {
17414                    name: "a carrier entry removed",
17415                    change: |r| {
17416                        r.carriers.retain(|c| {
17417                            c.principal
17418                                != Principal::Reserved {
17419                                    module_id: AFT.to_string(),
17420                                }
17421                        })
17422                    },
17423                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17424                },
17425                Case {
17426                    name: "a target removed from a carrier",
17427                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17428                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
17429                },
17430                Case {
17431                    name: "a bare carrier narrowed to targets",
17432                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17433                    closed: [None, Some(ScopeCarrierRemoved), None, None],
17434                },
17435                Case {
17436                    name: "delegates turned off",
17437                    change: |r| r.attributes.delegates = false,
17438                    closed: [Some(ScopeDelegationChanged); 4],
17439                },
17440                Case {
17441                    name: "agent_id changed",
17442                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17443                    closed: [Some(ScopeDelegationChanged); 4],
17444                },
17445                Case {
17446                    name: "a carrier added, child owners changed, the record re-sent",
17447                    change: |r| {
17448                        r.carriers.push(carrier(MAGIC, None));
17449                        r.child_owners.push(Principal::Reserved {
17450                            module_id: MAGIC.to_string(),
17451                        });
17452                    },
17453                    closed: [None; 4],
17454                },
17455                Case {
17456                    name: "a target added",
17457                    change: |r| {
17458                        r.carriers[1]
17459                            .targets
17460                            .as_mut()
17461                            .unwrap()
17462                            .push("third".to_string())
17463                    },
17464                    closed: [None; 4],
17465                },
17466                Case {
17467                    name: "delegates turned on",
17468                    change: |r| r.attributes.delegates = true,
17469                    closed: [None; 4],
17470                },
17471            ];
17472            for case in cases {
17473                let mut rig = rig().await;
17474                rig.sync(vec![session(1)]).await;
17475                let scope = || Some(rig_selector("s", Some(1)));
17476                let mut routes = [
17477                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
17478                    rig.bound(Some(AFT), PLEXUS, scope()).await,
17479                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
17480                    rig.bound(Some(BROCA), OTHER, scope()).await,
17481                ];
17482                let mut record = session(1);
17483                (case.change)(&mut record);
17484                rig.sync(vec![record]).await;
17485                for (index, expected) in case.closed.iter().enumerate() {
17486                    let route = &mut routes[index];
17487                    match expected {
17488                        Some(reason) => {
17489                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
17490                            assert_eq!(
17491                                route.closed_reason(),
17492                                *reason,
17493                                "{}: route {index}",
17494                                case.name
17495                            );
17496                        }
17497                        None => {
17498                            assert!(rig.live(route), "{}: route {index} closed", case.name);
17499                            assert!(
17500                                route.untouched(),
17501                                "{}: route {index} was told something",
17502                                case.name
17503                            );
17504                        }
17505                    }
17506                }
17507            }
17508        }
17509
17510        #[tokio::test]
17511        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17512            // Removed, and replaced by a higher epoch.
17513            for next in [Vec::new(), vec![session(2)]] {
17514                let mut rig = rig().await;
17515                rig.sync(vec![session(1)]).await;
17516                let mut route = rig
17517                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17518                    .await;
17519                rig.sync(next).await;
17520                assert!(!rig.live(&route));
17521                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17522            }
17523
17524            // A child whose parent ends: its routes close as parent-ended, the
17525            // child stays live, and routes under the parent close as ended.
17526            let mut rig = rig().await;
17527            let mut child = session(1);
17528            child.scope_ref = "child".to_string();
17529            child.kind = ScopeKind::Worker;
17530            child.parent = Some(ScopeParent::new(
17531                Principal::Reserved {
17532                    module_id: OWNER.to_string(),
17533                },
17534                "s".to_string(),
17535                1,
17536            ));
17537            rig.sync(vec![session(1), child.clone()]).await;
17538            let mut child_route = rig
17539                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17540                .await;
17541            assert_eq!(
17542                child_route.stamp().unwrap().parent_state,
17543                Some(ParentState::Linked)
17544            );
17545            rig.sync(vec![child]).await;
17546            assert!(!rig.live(&child_route));
17547            assert_eq!(
17548                child_route.closed_reason(),
17549                RouteCloseReason::ScopeParentEnded
17550            );
17551        }
17552
17553        #[tokio::test]
17554        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17555        ) {
17556            let mut rig = rig().await;
17557            rig.sync(vec![session(1)]).await;
17558            let mut route = rig
17559                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17560                .await;
17561            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17562            rig.sync(vec![session(1)]).await;
17563            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17564            assert!(rig.live(&route) && route.untouched());
17565
17566            // A call in flight on the route when another carrier is added. A
17567            // forwarded REQUEST holds one credit on the route's flow until the
17568            // module answers; the router takes it exactly like this.
17569            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17570                .forwarding
17571                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17572                .unwrap()
17573            else {
17574                panic!("the route is bound");
17575            };
17576            binding.flow.acquire_tagged(9, false).await.unwrap();
17577            let mut widened = session(1);
17578            widened.carriers.push(carrier(MAGIC, None));
17579            rig.sync(vec![widened]).await;
17580            assert!(rig.live(&route) && route.untouched());
17581            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17582            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17583            // The call's credit is still held on an open flow, so its answer
17584            // will be delivered: closing the route would have closed the flow.
17585            assert_eq!(binding.flow.in_flight(), 1);
17586            binding
17587                .flow
17588                .acquire_tagged(10, false)
17589                .await
17590                .expect("the flow is still open");
17591        }
17592
17593        /// A swap's superseded endpoint keeps its routes until drained; ending
17594        /// the scope closes them there too.
17595        #[tokio::test]
17596        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17597            let mut rig = rig().await;
17598            rig.sync(vec![session(1)]).await;
17599            let mut on_incumbent = rig
17600                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17601                .await;
17602
17603            // Swap plexus: register a candidate and cut over, leaving the
17604            // incumbent superseded with the route still on it.
17605            let (candidate, _candidate_rx) = wide_ctx(9);
17606            let registration = rig
17607                .handler
17608                .registry
17609                .register_candidate_with_control_ops(
17610                    manifest(PLEXUS, PROTOCOL_VERSION),
17611                    PROTOCOL_VERSION,
17612                    candidate.connection_id,
17613                    module_baseline_control_ops(),
17614                )
17615                .unwrap();
17616            rig.forwarding
17617                .register_candidate_module_connection(
17618                    candidate.connection_id,
17619                    PLEXUS.to_string(),
17620                    PROTOCOL_VERSION,
17621                    manifest_concurrency(&registration.manifest),
17622                    candidate.egress.clone(),
17623                )
17624                .unwrap();
17625            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17626            rig.handler
17627                .registry
17628                .promote_candidate(PLEXUS)
17629                .unwrap()
17630                .unwrap();
17631            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17632
17633            rig.sync(Vec::new()).await;
17634            assert!(!rig.live(&on_incumbent));
17635            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17636            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17637            let goodbye = incumbent_rx
17638                .try_recv()
17639                .expect("the superseded endpoint is told")
17640                .frame;
17641            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17642        }
17643    }
17644}
17645
17646#[cfg(test)]
17647mod concurrency_default_exposure_tests {
17648    use super::*;
17649
17650    fn hello_body(role_json: &str) -> Vec<u8> {
17651        format!(
17652            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17653        )
17654        .into_bytes()
17655    }
17656
17657    fn manifest_from(body: &[u8]) -> ModuleManifest {
17658        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17659        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17660            .expect("manifest parses")
17661    }
17662
17663    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17664
17665    #[test]
17666    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17667        let body = hello_body(&format!(
17668            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17669        ));
17670        let manifest = manifest_from(&body);
17671        // Precondition: serde really resolved it to the default, so the typed
17672        // manifest alone cannot answer the question this probe exists for.
17673        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17674        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17675    }
17676
17677    #[test]
17678    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17679        let body = hello_body(&format!(
17680            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17681        ));
17682        let manifest = manifest_from(&body);
17683        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17684        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17685    }
17686
17687    #[test]
17688    fn non_management_roles_are_never_reported() {
17689        let body = hello_body(
17690            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17691        );
17692        let manifest = manifest_from(&body);
17693        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17694    }
17695}